#!/bin/bash -l
# ============================================================================
#  resubmit_clwrf_2026091100.sh
#  Nop lai buoc TICH PHAN clWRF cho chu ky 2026091100, TRANH cap node loi
#  fabric (compute-1-2,3) khien job legacy (run_wrf.sh, job 1401243) chay
#  ~110x cham (~13.76 s/step vs baseline ~0.12 s/step thang 6).
#
#  Chay DUOI QUYEN nampq:
#      bash /home/tanpv/Progs/claude/Auto_RCMs/jobs/resubmit_clwrf_2026091100.sh
#
#  RegCM 2026091100 DA XONG + dong goi (RegCM_03.nc). Chi con thieu clWRF.
#  met_em / wrfinput / wrfbdy da co san trong wdir/2026091100 nen chi nop lai
#  tich phan (real.exe + wrf.exe). Chua co wrfrst -> chay lai tu dau (~5-6
#  ngay mo hinh da tinh, khong dang ke). Sau khi xong, GD3 tu dong dong goi
#  clWRF_03.nc.
# ============================================================================
set -euo pipefail

RCMS_HOME=/home/nampq/Auto_RCMs
export RCMS_HOME
export RCMS_CONF="$RCMS_HOME/config/rcms.conf"

S_DATE=2026091100
E_DATE=2027040100
EXCLUDE=compute-1-2,compute-1-3

# --- 1. Huy job clWRF cham neu dang chay (legacy run_wrf.sh, ten clWRF_run) --
old=$(squeue -u "$USER" -h -n clWRF_run -o '%i' 2>/dev/null | tr -d ' ')
if [[ -n "$old" ]]; then
    echo ">> Huy job clWRF cu: $old"
    scancel "$old"
    sleep 3
else
    echo ">> Khong thay job clWRF_run dang chay (co the da huy)."
fi

# --- 2. Nop lai tich phan, loai cap node loi -------------------------------
echo ">> Nop lai tich phan clWRF $S_DATE -> $E_DATE (exclude $EXCLUDE)"
sbatch \
    --job-name=clWRF_run \
    --nodes=2 \
    --ntasks-per-node=20 \
    --time=4-00:00:00 \
    --exclude="$EXCLUDE" \
    --output="$RCMS_HOME/logs/clwrf_${S_DATE}.out" \
    --error="$RCMS_HOME/logs/clwrf_${S_DATE}.err" \
    --export=ALL \
    "$RCMS_HOME/jobs/rcms_job.sh" _integrate_clwrf "$S_DATE" "$E_DATE"

echo
echo ">> Da nop. Kiem tra hang doi:"
squeue -u "$USER" -n clWRF_run,RegCM_run
echo
echo ">> Sau ~3-5 phut, kiem tra toc do step (tot: ~0.1-0.2 s/step):"
echo "   grep 'Timing for main' /work/users/nampq/working/Proj_T.Tan/Operational/Seasonal_FC/CLWRF/CFS/wdir/${S_DATE}/rsl.out.0000 | tail -5"