#!/bin/bash
#SBATCH --job-name=EHD_v3dl
#SBATCH --nodes=5
#SBATCH --exclusive
#SBATCH --time=12:00:00
#SBATCH --output=/home/tanpv/Progs/claude/QG_1/logs/v3dl_%j.out
#SBATCH --error=/home/tanpv/Progs/claude/QG_1/logs/v3dl_%j.err
set -euo pipefail
ROOT=/home/tanpv/Progs/claude/QG_1
PY=/home/tanpv/.conda/envs/ehd/bin/python
# torch needs the conda libstdc++ ahead of the system one (no CXXABI_1.3.9 in /lib64)
export LD_LIBRARY_PATH=/home/tanpv/.conda/envs/ehd/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
cd "$ROOT"
mkdir -p logs outputs/loyo_pred_v3dl

FAMILIES=${FAMILIES:-"cnn resunet transformer ann_torch"}
CAP=${CAP:-300000}                              # tabular (transformer/ann) train-row cap
YEARLIST=$(seq 2003 2022)
NFOLD=$(echo $YEARLIST | wc -w)

# torch is CPU-threaded; give each fold task 5 cores and pin the libs to them.
export OMP_NUM_THREADS=5 MKL_NUM_THREADS=5 OPENBLAS_NUM_THREADS=5 EHD_NJOBS=5
export TORCH_NUM_THREADS=5

echo "=== v3 DL LOYO: $NFOLD folds | families=[$FAMILIES] cap=$CAP | 5 nodes x 4 tasks x 5 cpus ==="
srun --nodes=5 --ntasks=$NFOLD --ntasks-per-node=4 --cpus-per-task=5 \
     --kill-on-bad-exit=1 bash -c \
     "YEARS=($YEARLIST); YR=\${YEARS[\$SLURM_PROCID]}; \
      echo \"[proc \$SLURM_PROCID @ \$(hostname)] fold \$YR start \$(date +%T)\"; \
      $PY scripts/22b_run_dl_v3.py --folds \$YR --families $FAMILIES --cap $CAP; \
      echo \"[proc \$SLURM_PROCID] fold \$YR done \$(date +%T)\""

echo "=== DL pred files: $(ls outputs/loyo_pred_v3dl/pred_v3dl_*.npz 2>/dev/null | wc -l)/$NFOLD ==="
# NOTE: the combined stack (stage 23 --dl-dir) is run AFTER both the v3zoo and v3dl jobs finish,
# so it sees a complete member set. Run it manually / from the driver once both are done:
#   $PY scripts/23_v3_diversity_stack.py --dl-dir outputs/loyo_pred_v3dl --dl-pattern pred_v3dl_*.npz
echo "=== v3 DL members done $(date +%T) ==="