#!/bin/bash -l
# ============================================================================
#  jobs/run_cycle_nampq.sh  -  Cong cu CHAY LAI cac chu ky RCMs BI HONG (nampq)
#
#  KHONG dung cho chu ky hien tai (da co CRON lo). Chi de KHOI PHUC cac ngay
#  chu ky {01,06,11,16,21,26} CHUA HOAN TAT vi: HPC sap, thieu so lieu ini/bien,
#  dang chay bi ngat, hoac chua kip chay. Voi moi ngay, TU CHAN DOAN va xu ly:
#     - thieu so lieu (SL)  -> DOWNLOAD bo sung truoc (manual_download.sh), roi chay
#     - dang chay do dang   -> RESUME (giu WPS/Dataset/IC-BC)
#     - chua chay           -> CHAY MOI
#     - da tich phan xong    -> chi DONG GOI GD3
#  ( '--resume' cua run_rcms.sh an toan cho ca 3 truong hop: khi chua co wdir no
#    tu chay moi; khi wdir do dang no nop lai. )
#
#  CHE DO:
#     scan            Quet cac ngay chu ky gan day, IN trang thai + viec can lam
#                     (chi doc, khong thay doi gi).
#     fix  YYYY MM DD Khoi phuc MOT ngay: download(neu thieu) -> chay -> GD3.
#     fixall          Tu quet va khoi phuc TUNG ngay hong (bo qua chu ky = hom nay
#                     de nhuong cron; bo qua khi he thong dang ban).
#
#  Bien moi truong: RCMS_HOME (mac dinh /home/nampq/Auto_RCMs), LOOKBACK_MONTHS
#  (mac dinh 1 = thang nay + thang truoc), POLL_SECONDS (mac dinh 300).
#
#  KHONG xoa gi. Ghi ket qua vao cay thu muc cu cua nampq (qua run_rcms.sh /
#  manual_download.sh). Chay bang user nampq.
#
#  DUNG:  ./run_cycle_nampq.sh scan
#         ./run_cycle_nampq.sh fix 2026 09 06
#         nohup ./run_cycle_nampq.sh fixall >& ~/fixall.log &
# ============================================================================
set -euo pipefail

RCMS_HOME="${RCMS_HOME:-/home/nampq/Auto_RCMs}"
RCMS_CONF="${RCMS_CONF:-$RCMS_HOME/config/rcms.conf}"
RUN="$RCMS_HOME/run_rcms.sh"
DL="$RCMS_HOME/manual_download.sh"
LOOKBACK_MONTHS="${LOOKBACK_MONTHS:-1}"
POLL_SECONDS="${POLL_SECONDS:-300}"
CYCLE_DAYS=(01 06 11 16 21 26)

die(){ echo "ERROR: $*" >&2; exit 1; }
info(){ echo "[$(date '+%F %T')] $*"; }

[[ -x "$RUN" ]] || die "khong thay run_rcms.sh tai $RUN (dat RCMS_HOME dung ban deploy nampq)"
[[ -x "$DL"  ]] || die "khong thay manual_download.sh tai $DL"
[[ -r "$RCMS_CONF" ]] || die "khong doc duoc config $RCMS_CONF"

# --- Nap duong dan tu config (chi lay bien; set +u vi conf dung ${VAR:-...}) --
# shellcheck disable=SC1090
{ set +u; source "$RCMS_CONF"; set -u; }
: "${INPUT_DIR:?INPUT_DIR chua dinh nghia trong config}"
: "${CURRENT_ROOT:?CURRENT_ROOT chua dinh nghia}"
: "${CLWRF_DIR:?CLWRF_DIR chua dinh nghia}"
: "${REGCM_DIR:?REGCM_DIR chua dinh nghia}"
: "${FORECAST_DIR:?FORECAST_DIR chua dinh nghia}"

# --- Canh bao neu KHONG chay bang nampq --------------------------------------
if [[ "$(id -un)" != "nampq" ]]; then
    echo "CANH BAO: dang chay bang $(id -un), khong phai nampq;" >&2
    echo "          thao tac ghi (download/chay/GD3) se THIEU QUYEN. 'su - nampq' truoc." >&2
fi

# ---- Anh xa ngay -> so hieu thanh vien (giong lib/monthly.sh) ---------------
_mem_of_date(){ case "${1:6:2}" in
    01)echo 01;; 06)echo 02;; 11)echo 03;; 16)echo 04;; 21)echo 05;; 26)echo 06;; *)return 1;; esac; }

# ---- Danh sach chu ky ung vien: cac ngay {01..26} <= hom nay trong cua so ---
_candidates(){
    local today ym i
    today=$(date +%Y%m%d)
    for i in $(seq 0 "$LOOKBACK_MONTHS"); do
        ym=$(date -d "$(date +%Y-%m-01) -${i} month" +%Y%m) || continue
        for d in "${CYCLE_DAYS[@]}"; do
            (( 10#${ym}${d} <= 10#$today )) && echo "${ym}${d}00"
        done
    done | sort -u
}

# ---- GD3 da du 4 file cho chu ky? ------------------------------------------
_gd3_complete(){ local s=$1 mem y=${1:0:4} m=${1:4:2} f
    mem=$(_mem_of_date "$s") || return 1
    for f in "monthly/$y/$m/clWRF_$mem.nc" "monthly/$y/$m/RegCM_$mem.nc" \
             "daily/$y/$m/clWRF_$mem.nc"   "daily/$y/$m/RegCM_$mem.nc"; do
        [[ -s "$FORECAST_DIR/$f" ]] || return 1
    done; return 0; }

# ---- GD2 (hau xu ly) da xong tung mo hinh? (dung tin hieu GD3 doc) ----------
_clwrf_done(){ [[ -s "$CURRENT_ROOT/CLWRF/$1/u.nc" ]]; }
_regcm_done(){ [[ -s "$CURRENT_ROOT/RegCM/$1/daily/pr.nc" ]]; }
_both_done(){ _clwrf_done "$1" && _regcm_done "$1"; }

# ---- Da bat dau (co wdir) chua? --------------------------------------------
_has_wdir(){ [[ -d "$CLWRF_DIR/wdir/$1" || -d "$REGCM_DIR/wdir/$1" ]]; }

# ---- So lieu DA DU chua? (manual_download.sh -n, validate chuoi 6h) ---------
_data_ok(){ "$DL" -n "$1" 2>&1 | grep -q "DA DU"; }

# ---- He thong RANH? (khong con job/tien trinh tich phan cua nampq) ----------
_is_idle(){
    local q proc
    q=$(squeue -u nampq -h -n clWRF_run,RegCM_run -o '%i' 2>/dev/null | tr -d ' ')
    proc=$(ps -u nampq -o cmd= 2>/dev/null \
           | grep -Ei 'autorun|run_wps|run_wrf|real\.exe|wrf\.exe|regcm|grb2nc|wgrib2' \
           | grep -v grep || true)
    [[ -z "$q" && -z "$proc" ]]; }

# ---- Cho tich phan xong (khong con job clWRF_run/RegCM_run cua nampq) -------
_wait_jobs(){
    info "Cho tich phan xong (poll ${POLL_SECONDS}s)..."
    while :; do
        local ids; ids=$(squeue -u nampq -h -n clWRF_run,RegCM_run -o '%i' 2>/dev/null | tr -d ' ')
        [[ -z "$ids" ]] && { info "  khong con job tich phan cua nampq."; break; }
        info "  con job: $(echo $ids | tr '\n' ' ')"; sleep "$POLL_SECONDS"
    done; }

# ---- Chan doan MOT chu ky -> in ma trang thai (dung cho scan & fix) ---------
#  In: complete | need_pack | need_data | need_resume | need_fresh
_diagnose(){ local s=$1
    _gd3_complete "$s" && { echo complete;    return; }
    _both_done    "$s" && { echo need_pack;   return; }
    _data_ok      "$s" || { echo need_data;   return; }
    _has_wdir     "$s" && { echo need_resume; return; }
    echo need_fresh; }

# ---- Mo ta tieng Viet cho ma trang thai ------------------------------------
_desc(){ case "$1" in
    complete)    echo "DA HOAN TAT (co du GD3) -> bo qua";;
    need_pack)   echo "Tich phan xong, THIEU GD3 -> chi dong goi";;
    need_data)   echo "THIEU SO LIEU -> download truoc roi chay";;
    need_resume) echo "Dang chay do dang -> RESUME (giu WPS/Dataset)";;
    need_fresh)  echo "CHUA chay -> chay MOI";;
esac; }

# ========================= CHE DO scan =======================================
do_scan(){
    echo "===== QUET CHU KY HONG (cua so ${LOOKBACK_MONTHS}+1 thang, <= hom nay) ====="
    printf '%-12s  %-4s  %s\n' "CHU_KY" "MEM" "TRANG_THAI / VIEC_CAN_LAM"
    echo "----------------------------------------------------------------------"
    local s mem st n_broken=0
    while read -r s; do
        [[ -z "$s" ]] && continue
        mem=$(_mem_of_date "$s") || mem="--"
        st=$(_diagnose "$s")
        printf '%-12s  %-4s  %s\n' "$s" "$mem" "$(_desc "$st")"
        [[ "$st" != complete ]] && n_broken=$((n_broken+1))
    done < <(_candidates)
    echo "----------------------------------------------------------------------"
    if (( n_broken )); then
        _is_idle || echo "LUU Y: he thong DANG BAN (co job/tien trinh nampq) -> hoan submit den khi ranh."
        echo "Chay lai 1 ngay:  $0 fix YYYY MM DD     |  Tu dong ca loat:  $0 fixall"
    else
        echo "Khong co chu ky hong trong cua so. (Tang LOOKBACK_MONTHS de quet xa hon.)"
    fi; }

# ========================= Khoi phuc MOT chu ky ==============================
#  _fix_one S_DATE  -> thuc hien dung viec can lam. Tra 0 neu OK / da xong.
_fix_one(){
    local s=$1 Y=${1:0:4} M=${1:4:2} D=${1:6:2} mem st
    mem=$(_mem_of_date "$s") || { echo "  bo qua $s: khong phai ngay chu ky."; return 0; }
    st=$(_diagnose "$s")
    info "Chu ky $s (mem $mem): $(_desc "$st")"

    if [[ "$st" == complete ]]; then return 0; fi

    # --- B1: So lieu ---------------------------------------------------------
    if [[ "$st" == need_data ]]; then
        info "  -> download bo sung so lieu cho $s ..."
        if "$DL" "$s"; then
            info "  so lieu $s da du va lien tuc."
        else
            echo "  KHONG tai du duoc so lieu cho $s -> CHUA the chay. Bo qua." >&2
            return 1
        fi
        st=$(_diagnose "$s")   # cham lai: gio thuong la need_resume/need_fresh
    fi

    # --- B2: Tich phan (neu chua xong ca hai mo hinh) ------------------------
    if ! _both_done "$s"; then
        if ! _is_idle; then
            echo "  TU CHOI chay $s: he thong DANG BAN (job/tien trinh nampq). Thu lai sau." >&2
            return 1
        fi
        info "  -> nop tich phan (resume-an-toan): run_rcms.sh --execute --resume --model both $Y $M $D"
        "$RUN" --execute --resume --model both "$Y" "$M" "$D"
        _wait_jobs
        sleep 30   # cho GD2 (trong job) kip ket thuc ghi done/
        if ! _both_done "$s"; then
            echo "  Sau khi chay, $s VAN chua du GD2 (co the tich phan that bai) -> xem logs/{clwrf,regcm}_${s}.{out,err}." >&2
            return 1
        fi
    fi

    # --- B3: Dong goi GD3 ----------------------------------------------------
    if ! _gd3_complete "$s"; then
        info "  -> dong goi GD3: run_rcms.sh --execute --model both $Y $M $D"
        "$RUN" --execute --model both "$Y" "$M" "$D"
    fi
    if _gd3_complete "$s"; then
        info "  HOAN TAT $s: du GD3 (clWRF_$mem.nc, RegCM_$mem.nc)."; return 0
    else
        echo "  $s: da chay nhung GD3 CHUA du -> kiem tra $FORECAST_DIR va log." >&2; return 1
    fi; }

do_fix(){
    local Y=$1 M=$2 D=$3
    [[ -n "${Y:-}" && -n "${M:-}" && -n "${D:-}" ]] || die "cu phap: $0 fix YYYY MM DD"
    local s="${Y}${M}${D}00"
    _mem_of_date "$s" >/dev/null || die "ngay $D khong phai ngay chu ky (01/06/11/16/21/26)"
    _fix_one "$s"; }

# ========================= fixall ============================================
do_fixall(){
    local today s ok=0 fail=0 skip=0
    today="$(date +%Y%m%d)00"
    info "fixall: quet va khoi phuc cac ngay hong (bo qua chu ky = hom nay cho cron)."
    while read -r s; do
        [[ -z "$s" ]] && continue
        if [[ "$s" == "$today" ]]; then
            info "  bo qua $s (= hom nay, de cron lo)."; skip=$((skip+1)); continue
        fi
        [[ "$(_diagnose "$s")" == complete ]] && continue
        if _fix_one "$s"; then ok=$((ok+1)); else fail=$((fail+1)); fi
    done < <(_candidates)
    info "fixall xong: thanh_cong=$ok  that_bai/bo_qua=$fail  nhuong_cron=$skip"; }

# ========================= dispatch ==========================================
case "${1:-}" in
    scan)   do_scan ;;
    fix)    shift; do_fix "${1:-}" "${2:-}" "${3:-}" ;;
    fixall) do_fixall ;;
    *) die "cu phap: $0 <scan | fix YYYY MM DD | fixall>" ;;
esac