#!/usr/bin/env bash
# ============================================================================
#  manual_download.sh  -  Tai THU CONG / BO SUNG du lieu dau vao CFSv2 GRIB2
#                         cho MOT chu ky, dung khi crontab bo lo hoac data thieu.
#
#  ==================  TOM TAT NHANH (hay dung nhat)  ======================
#     ./manual_download.sh -n 2026090100     # KIEM TRA ngay 01/09/2026 (khong tai)
#     ./manual_download.sh 2026090100        # TAI bo sung file con thieu (TU chon nguon)
#     ./manual_download.sh 2026 09 06        # dang YYYY MM DD [HH]
#     ./manual_download.sh --source s3 2025010100   # ep lay tu kho luu tru AWS
#  NGUON (mac dinh --source auto):
#     - NOMADS (prod) chi giu ~7 ngay gan nhat -> dung cho chu ky moi.
#     - Qua 7 ngay NOMADS se xoa; tool TU DONG chuyen sang kho luu tru AWS
#       (bucket 'noaa-cfs-pds', co du lieu tu 2018-10-31 -> hien tai).
#  ========================================================================
#
#  Cach dung:
#     ./manual_download.sh [-n] [--verify] [--source auto|nomads|s3] [YYYYMMDDHH]
#     ./manual_download.sh [-n] [--verify] [--source auto|nomads|s3] YYYY MM DD [HH]
#
#     -n, --dry-run       Chi KIEM TRA (so file thieu), KHONG tai.
#         --verify        Kiem tra ky kich thuoc file DA CO. Voi nguon S3 listing
#                         da kem kich thuoc chinh xac nen luon kiem (mien phi);
#                         voi NOMADS phai HEAD tung file (CHAM) nen mac dinh TAT.
#         --source SRC    auto (mac dinh) | nomads | s3.
#     -h, --help          In tro giup nay.
#     Khong co ngay       -> mac dinh HOM NAY 00Z (UTC).
#
#  Hanh vi (idempotent, an toan chay lai nhieu lan):
#     - Chi tai file flxf*/pgbf* .grb2 (dung cho clWRF + RegCM), bo qua .idx.
#     - File da co & DU (khop kich thuoc neu biet, neu khong thi chi can khac rong)
#                                          -> BO QUA.
#     - File thieu / rong / sai kich thuoc -> TAI (wget -c, resume).
#     - Cuoi cung bao cao ky vong vs thuc te + kiem tra tinh lien tuc 6h.
#
#  Ghi chu:
#     - Duong dan dich, MEMBER, INPUT_DIR... lay tu config/rcms.conf (dung chung
#       voi run_rcms.sh). Ghi vao $INPUT_DIR/<YYYYMMDD>/ (vi tri he cu, quyen nampq).
#     - URL co the ghi de: CFS_URL (NOMADS), CFS_S3_URL (kho AWS).
#     - Chi ho tro day du kiem tra lien tuc cho chu ky 00Z (Seasonal_FC). HH khac
#       van tai duoc nhung buoc validate lien tuc se bo qua (canh bao).
# ============================================================================

set -Eeuo pipefail

RCMS_HOME="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
export RCMS_HOME

# shellcheck source=lib/common.sh
source "$RCMS_HOME/lib/common.sh"
load_config

CFS_URL="${CFS_URL:-https://nomads.ncep.noaa.gov/pub/data/nccf/com/cfs/prod}"
CFS_S3_URL="${CFS_S3_URL:-https://noaa-cfs-pds.s3.amazonaws.com}"

usage() {
    cat <<'EOF'
manual_download.sh - Tai thu cong / bo sung du lieu CFSv2 GRIB2 cho mot chu ky.

Cach dung:
   ./manual_download.sh [-n] [--verify] [--source auto|nomads|s3] [YYYYMMDDHH]
   ./manual_download.sh [-n] [--verify] [--source auto|nomads|s3] YYYY MM DD [HH]

   -n, --dry-run     Chi KIEM TRA (so file thieu), KHONG tai.
       --verify      So ky kich thuoc file da co (NOMADS: HEAD tung file, CHAM).
       --source SRC  auto (mac dinh) | nomads | s3 (kho luu tru AWS).
   -h, --help        In tro giup nay.
   Khong co ngay     -> mac dinh HOM NAY 00Z (UTC).

Nguon:
   - NOMADS prod giu ~7 ngay gan nhat; qua do tool TU DONG dung kho AWS
     (noaa-cfs-pds, co du lieu tu 2018-10-31 -> hien tai).

Vi du:
   ./manual_download.sh -n 2026090100            # kiem tra ngay 01/09/2026
   ./manual_download.sh 2026 09 06               # tai bo sung ngay 06/09/2026
   ./manual_download.sh --source s3 2025010100   # lay ngay cu tu kho luu tru
EOF
    exit "${1:-0}"
}

# ---- Phan tich tham so ----------------------------------------------------
DRY_RUN=0
VERIFY=0
SOURCE=auto
args=()
while [[ $# -gt 0 ]]; do
    case "$1" in
        -n|--dry-run) DRY_RUN=1; shift ;;
        --verify)     VERIFY=1; shift ;;
        --source)     SOURCE="${2:-}"; shift 2 ;;
        --source=*)   SOURCE="${1#*=}"; shift ;;
        -h|--help)    usage 0 ;;
        --)           shift; while [[ $# -gt 0 ]]; do args+=("$1"); shift; done ;;
        -*)           die "tham so khong hop le: $1 (dung -h de xem tro giup)" ;;
        *)            args+=("$1"); shift ;;
    esac
done

case "$SOURCE" in auto|nomads|s3) ;; *) die "--source phai la auto|nomads|s3 (nhan: $SOURCE)";; esac

if [[ ${#args[@]} -eq 0 ]]; then
    YMDH="$(date -u +%Y%m%d)00"
elif [[ ${#args[@]} -eq 1 ]]; then
    [[ ${args[0]} =~ ^[0-9]{10}$ ]] || die "ngay phai la YYYYMMDDHH (10 chu so): ${args[0]}"
    YMDH="${args[0]}"
else
    YMDH="$(normalize_cycle "${args[@]}")"
fi

YMD="${YMDH:0:8}"
HH="${YMDH:8:2}"

# Thu muc dich: $INPUT_DIR = .../CFS_Ope/00 -> goc = .../CFS_Ope
LOCAL_ROOT="${INPUT_DIR%/*}"
LOCAL_DIR="$LOCAL_ROOT/$HH/$YMD"

NOMADS_DIR="$CFS_URL/cfs.$YMD/$HH/6hrly_grib_01"
S3_PREFIX="cfs.$YMD/$HH/6hrly_grib_01/"

command -v curl >/dev/null 2>&1 || die "can co 'curl' de liet ke/kiem tra remote"
command -v wget >/dev/null 2>&1 || die "can co 'wget' de tai file"

# ---- Liet ke: ghi ra stdout cac dong "<basename>\t<size|>" -----------------
# NOMADS: khong co kich thuoc -> cot size de trong.
list_nomads() {
    curl -sk "$NOMADS_DIR/" | cut -d'>' -f2 | cut -d'<' -f1 \
        | grep -E '^(flxf|pgbf)[0-9]{10}\.[0-9]{2}\.[0-9]{10}\.grb2$' \
        | sort -u | sed 's/$/\t/'
}

# S3 (ListObjectsV2, phan trang): co kich thuoc chinh xac tung file.
list_s3() {
    local token="" resp truncated
    while :; do
        if [[ -z $token ]]; then
            resp="$(curl -sk -G "$CFS_S3_URL/" \
                    --data-urlencode "list-type=2" \
                    --data-urlencode "prefix=$S3_PREFIX" \
                    --data-urlencode "max-keys=1000")"
        else
            resp="$(curl -sk -G "$CFS_S3_URL/" \
                    --data-urlencode "list-type=2" \
                    --data-urlencode "prefix=$S3_PREFIX" \
                    --data-urlencode "max-keys=1000" \
                    --data-urlencode "continuation-token=$token")"
        fi
        # Trong moi <Contents>: <Key> dung truoc <Size> -> ghep theo thu tu.
        printf '%s\n' "$resp" | grep -oE '<Key>[^<]+</Key>|<Size>[0-9]+</Size>' \
            | sed -E 's#<Key>(.*)</Key>#K\t\1#; s#<Size>([0-9]+)</Size>#S\t\1#' \
            | awk -F'\t' '$1=="K"{k=$2} $1=="S"{print k"\t"$2}' \
            | awk -F'\t' '{n=split($1,a,"/"); b=a[n];
                          if (b ~ /^(flxf|pgbf)[0-9]{10}\.[0-9]{2}\.[0-9]{10}\.grb2$/)
                              print b"\t"$2}'
        truncated="$(printf '%s\n' "$resp" | grep -oE '<IsTruncated>[^<]+' | sed 's/.*>//' | head -1)"
        token="$(printf '%s\n' "$resp" | grep -oE '<NextContinuationToken>[^<]+' | sed 's/.*>//' | head -1)"
        [[ $truncated == true && -n $token ]] || break
    done | sort -u
}

# ---- Chon nguon (auto: NOMADS truoc, khong co -> S3) -----------------------
TMPLIST="$(mktemp)"
trap 'rm -f "$TMPLIST"' EXIT

USED_SRC=""; URL_BASE=""
resolve_source() {
    case "$SOURCE" in
        nomads) USED_SRC=nomads ;;
        s3)     USED_SRC=s3 ;;
        auto)
            log "thu nguon NOMADS (prod, ~7 ngay)..."
            list_nomads > "$TMPLIST" || true
            if [[ -s $TMPLIST ]]; then USED_SRC=nomads
            else log "NOMADS khong co $YMDH -> chuyen sang kho luu tru AWS (S3)"; USED_SRC=s3; fi
            ;;
    esac
    if [[ $USED_SRC == nomads ]]; then
        URL_BASE="$NOMADS_DIR"
        # || true: bo qua rc gia do SIGPIPE (head -1) duoi pipefail; tinh day du
        # duoc kiem lai bang so file (n_remote) + validate_cycle_files o duoi.
        [[ -s $TMPLIST ]] || list_nomads > "$TMPLIST" || true
    else
        URL_BASE="$CFS_S3_URL/$S3_PREFIX"; URL_BASE="${URL_BASE%/}"
        log "liet ke tu kho AWS S3 (noaa-cfs-pds)..."
        list_s3 > "$TMPLIST" || true
    fi
}

log "Chu ky        : $YMDH  (init ${YMD} ${HH}Z)"
log "Thu muc dich  : $LOCAL_DIR"
[[ $DRY_RUN -eq 1 ]] && log "Che do        : DRY-RUN (chi kiem tra, khong tai)"

resolve_source
log "Nguon dung    : $USED_SRC ($URL_BASE)"

n_remote=$(wc -l < "$TMPLIST")
[[ $n_remote -gt 0 ]] || die "khong lay duoc danh sach file flxf/pgbf cho $YMDH tu nguon '$USED_SRC'. Thu --source s3 hoac kiem tra lai ngay/mang."
log "remote co $n_remote file flxf/pgbf .grb2"

# ---- Vong tai / bo qua ----------------------------------------------------
[[ $DRY_RUN -eq 1 ]] || mkdir -p "$LOCAL_DIR"

n_skip=0 n_get=0 n_fail=0 n_missing=0
while IFS=$'\t' read -r f size; do
    [[ -n $f ]] || continue
    local="$LOCAL_DIR/$f"

    if [[ -s $local ]]; then
        lsize="$(stat -c %s "$local")"
        if [[ -n $size ]]; then
            # Biet kich thuoc (S3, hoac NOMADS sau nay): so chinh xac.
            [[ $lsize == "$size" ]] && { n_skip=$((n_skip+1)); continue; }
            warn "kich thuoc khong khop ($lsize != $size) -> tai lai: $f"
        elif [[ $VERIFY -eq 1 ]]; then
            rsize="$(curl -sIk "$URL_BASE/$f" \
                     | awk 'tolower($1)=="content-length:"{v=$2} END{gsub(/\r/,"",v); print v}')"
            if [[ -z $rsize || $lsize == "$rsize" ]]; then n_skip=$((n_skip+1)); continue; fi
            warn "kich thuoc khong khop ($lsize != $rsize) -> tai lai: $f"
        else
            n_skip=$((n_skip+1)); continue
        fi
    fi

    if [[ $DRY_RUN -eq 1 ]]; then
        n_missing=$((n_missing+1)); log "  [THIEU] $f"; continue
    fi

    if wget --no-check-certificate -c -q "$URL_BASE/$f" -O "$local"; then
        if [[ -n $size && -s $local && "$(stat -c %s "$local")" != "$size" ]]; then
            n_fail=$((n_fail+1)); warn "tai xong nhung sai kich thuoc: $f"
        elif [[ -s $local ]]; then
            n_get=$((n_get+1))
        else
            n_fail=$((n_fail+1)); warn "tai ve rong: $f"
        fi
    else
        n_fail=$((n_fail+1)); warn "tai that bai: $f"
    fi
done < "$TMPLIST"

# ---- Bao cao --------------------------------------------------------------
echo "----------------------------------------------------------------------"
if [[ $DRY_RUN -eq 1 ]]; then
    n_present=$(( n_remote - n_missing ))
    log "KET QUA (dry-run, nguon=$USED_SRC): remote=$n_remote  da co du=$n_present  con thieu=$n_missing"
    [[ $n_missing -eq 0 ]] && log "=> Du lieu $YMDH DA DU." \
                           || log "=> Bo '-n' de tai $n_missing file con thieu."
    exit 0
fi

n_local_flxf=$(find "$LOCAL_DIR" -maxdepth 1 -name 'flxf*.grb2' -size +0c 2>/dev/null | wc -l)
n_local_pgbf=$(find "$LOCAL_DIR" -maxdepth 1 -name 'pgbf*.grb2' -size +0c 2>/dev/null | wc -l)
log "KET QUA (nguon=$USED_SRC): bo qua(da du)=$n_skip  tai moi=$n_get  loi=$n_fail"
log "Hien co tai dich: flxf=$n_local_flxf  pgbf=$n_local_pgbf  (remote=$n_remote)"

# Kiem tra tinh lien tuc 6h (chi cho chu ky 00Z, dung ham chung voi run_rcms.sh)
if [[ $HH == 00 ]] && [[ "$LOCAL_DIR" == "$INPUT_DIR/$YMD" ]]; then
    if resolve_cycle_dates "$YMD"; then
        if validate_cycle_files "$CYCLE_S_DATE" "$CYCLE_E_DATE"; then
            log "=> Du lieu $YMDH DAY DU va LIEN TUC, san sang chay RCM."
        else
            warn "=> Van con thieu/dut quang trong chuoi 6h - kiem tra canh bao o tren."
            exit 1
        fi
    fi
else
    warn "chu ky khong phai 00Z (hoac dich khac INPUT_DIR) -> bo qua buoc validate lien tuc."
fi

[[ $n_fail -eq 0 ]] || exit 1
exit 0