#!/bin/bash
#SBATCH --job-name=EHD_v3_retrain
#SBATCH --partition=workq
#SBATCH --nodes=5
#SBATCH --exclusive
#SBATCH --time=04:00:00
#SBATCH --output=/home/tanpv/Progs/claude/QG_1/logs/v3_retrain_%j.out
#SBATCH --error=/home/tanpv/Progs/claude/QG_1/logs/v3_retrain_%j.err
set -euo pipefail
ROOT=/home/tanpv/Progs/claude/QG_1
PY=/home/tanpv/.conda/envs/ehd/bin/python
export LD_LIBRARY_PATH=/home/tanpv/.conda/envs/ehd/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
export NINO34_TXT=/home/tanpv/Progs/claude/HW/data/external/nino34.txt
export OMI_TXT=/home/tanpv/Progs/claude/HW/data/external/omi.txt
cd "$ROOT"
mkdir -p logs outputs/loyo_pred_v3sel

YEARLIST=$(seq 2003 2022)
NFOLD=$(echo $YEARLIST | wc -w)
export OMP_NUM_THREADS=5 MKL_NUM_THREADS=5 OPENBLAS_NUM_THREADS=5

echo "=== v3 selected-predictor retrain: $NFOLD folds | base vs selected HGB | 5 nodes x 4 x 5 cpu ==="
srun --nodes=5 --ntasks=$NFOLD --ntasks-per-node=4 --cpus-per-task=5 \
     --kill-on-bad-exit=1 bash -c \
     "YEARS=($YEARLIST); YR=\${YEARS[\$SLURM_PROCID]}; \
      echo \"[proc \$SLURM_PROCID @ \$(hostname)] fold \$YR start \$(date +%T)\"; \
      $PY scripts/20d_retrain_selected.py --folds \$YR; \
      echo \"[proc \$SLURM_PROCID] fold \$YR done \$(date +%T)\""

echo "=== folds returned; v3 pred files: $(ls outputs/loyo_pred_v3sel/pred_v3_*.npz 2>/dev/null | wc -l)/$NFOLD ==="
echo "=== stage 20e verification $(date +%T) ==="
$PY scripts/20e_verify_v3.py
echo "=== done $(date +%T) ==="