#!/bin/bash
#SBATCH --job-name=EHD_AI_VERIFY
#SBATCH --nodes=6
#SBATCH --ntasks-per-node=20
#SBATCH --cpus-per-task=1
#SBATCH --time=2-00:00:00
#SBATCH --output=/home/tanpv/Progs/CLI/EHD/logs/ehd_ai_verify_%j.out
#SBATCH --error=/home/tanpv/Progs/CLI/EHD/logs/ehd_ai_verify_%j.err
set -euo pipefail
ROOT=/home/tanpv/Progs/CLI/EHD
PY=/home/hapt/miniconda3/bin/python3
cd "$ROOT"
mkdir -p logs outputs/AI_Results/parts
export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1
find outputs/AI_Results/parts -maxdepth 1 -type f -name 'part_*.npz' -delete
# All 120 ranks perform full-grid, untouched-test verification.
srun --nodes=6 --ntasks=120 --ntasks-per-node=20 --cpus-per-task=1 --kill-on-bad-exit=1 $PY ehd_ai.py worker
EXPECTED_TASKS=120 $PY ehd_ai.py finalize --tasks 120