#!/usr/bin/env bash
# Recover one explicitly deferred instantaneous month with global one-worker locks.
set -u -o pipefail

project_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
init_month=${1:-}
chunk_leads=${RECOVERY_CHUNK_LEADS:-112}
log_dir="$project_root/outputs/logs"
download_lock="$log_dir/ecmwf_baseline_download.lock"
recovery_lock="$log_dir/ecmwf_deferred_recovery.lock"
attempt_stamp="$log_dir/ecmwf_recovery_${init_month}.last_attempt"

if ! [[ "$init_month" =~ ^[0-9]{4}-(0[1-9]|1[0-2])$ ]] || \
   ! [[ "$chunk_leads" =~ ^[1-9][0-9]*$ ]]; then
  printf 'Usage: %s YYYY-MM (RECOVERY_CHUNK_LEADS must be positive)\n' "$0" >&2
  exit 2
fi

mkdir -p "$log_dir"
exec 9>"$recovery_lock"
if ! flock -n 9; then
  printf '%s RECOVERY_BUSY init=%s\n' "$(date -Is)" "$init_month"
  exit 75
fi

# Hold the normal downloader lock throughout recovery to enforce one CDS worker.
exec 8>"$download_lock"
if ! flock -n 8; then
  printf '%s DOWNLOADER_ACTIVE init=%s; recovery deferred\n' "$(date -Is)" "$init_month"
  exit 75
fi

touch "$attempt_stamp"
printf '%s RECOVERY_START init=%s chunk_leads=%s\n' \
  "$(date -Is)" "$init_month" "$chunk_leads"
PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$project_root/src" \
  python3 -m dr_gcm.recover_ecmwf_instantaneous \
    --init "$init_month" \
    --chunk-leads "$chunk_leads" \
    --reset-blocked-attempts \
    --log-level INFO
status=$?
printf '%s RECOVERY_END init=%s exit=%s\n' "$(date -Is)" "$init_month" "$status"
exit "$status"