#!/bin/bash -l
# ============================================================================
#  resubmit_clwrf_2026090600.sh
#  Nop lai buoc TICH PHAN clWRF cho chu ky 2026090600, TRANH cap node loi
#  fabric (compute-1-2,3) da lam job cu chay ~85x cham.
#
#  Chay DUOI QUYEN nampq:
#      bash /home/tanpv/Progs/claude/Auto_RCMs/jobs/resubmit_clwrf_2026090600.sh
#
#  met_em / wrfinput / wrfbdy da co san trong wdir/2026090600 nen chi nop lai
#  tich phan (real.exe + wrf.exe). Chua co wrfrst -> chay lai tu dau (~12h
#  gio-mo-hinh da tinh, khong dang ke).
# ============================================================================
set -euo pipefail

RCMS_HOME=/home/nampq/Auto_RCMs
export RCMS_HOME
export RCMS_CONF="$RCMS_HOME/config/rcms.conf"

S_DATE=2026090600
E_DATE=2027040100
EXCLUDE=compute-1-2,compute-1-3

# --- 1. Huy job clWRF cham neu dang chay (khong dung toi RegCM) -------------
old=$(squeue -u "$USER" -h -n clWRF_run -o '%i' 2>/dev/null | tr -d ' ')
if [[ -n "$old" ]]; then
    echo ">> Huy job clWRF cu: $old"
    scancel "$old"
    sleep 3
else
    echo ">> Khong thay job clWRF_run dang chay (co the da huy)."
fi

# --- 2. Nop lai tich phan, loai cap node loi -------------------------------
echo ">> Nop lai tich phan clWRF $S_DATE -> $E_DATE (exclude $EXCLUDE)"
sbatch \
    --job-name=clWRF_run \
    --nodes=2 \
    --ntasks-per-node=20 \
    --time=4-00:00:00 \
    --exclude="$EXCLUDE" \
    --output="$RCMS_HOME/logs/clwrf_${S_DATE}.out" \
    --error="$RCMS_HOME/logs/clwrf_${S_DATE}.err" \
    --export=ALL \
    "$RCMS_HOME/jobs/rcms_job.sh" _integrate_clwrf "$S_DATE" "$E_DATE"

echo
echo ">> Da nop. Kiem tra hang doi:"
squeue -u "$USER" -n clWRF_run,RegCM_run
echo
echo ">> Sau ~3-5 phut, kiem tra toc do step (tot: ~0.1-0.2 s/step):"
echo "   grep 'Timing for main' /work/users/nampq/working/Proj_T.Tan/Operational/Seasonal_FC/CLWRF/CFS/wdir/${S_DATE}/rsl.out.0000 | tail -5"