#!/usr/bin/env bash
set -u -o pipefail

project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
init_month="${RECOVERY_INIT_MONTH:-1989-03}"
chunk_leads="${RECOVERY_CHUNK_LEADS:-112}"
poll_seconds="${RECOVERY_POLL_SECONDS:-300}"
once=false

if [[ "${1:-}" == "--once" ]]; then
  once=true
elif [[ -n "${1:-}" ]]; then
  echo "Usage: $0 [--once]" >&2
  exit 64
fi

log_dir="$project_root/outputs/logs"
supervisor_log="$log_dir/ecmwf_recovery_supervisor.log"
supervisor_pid_file="$log_dir/ecmwf_recovery_supervisor.pid"
supervisor_lock="$log_dir/ecmwf_recovery_supervisor.lock"
worker_log="$log_dir/ecmwf_recovery_${init_month}.log"
worker_pid_file="$log_dir/ecmwf_recovery_${init_month}.pid"
target="$project_root/data/ecmwf/raw_split/ecmwf_seas51_${init_month}_instantaneous.nc"
watchdog="$project_root/scripts/baseline_watchdog.sh"
watchdog_pid_file="$log_dir/baseline_watchdog.pid"
watchdog_lock="$log_dir/baseline_watchdog.lock"

mkdir -p "$log_dir"
touch "$supervisor_log"

log() {
  printf '%s %s\n' "$(date --iso-8601=seconds)" "$*" >> "$supervisor_log"
}

is_uint() {
  [[ "$1" =~ ^[0-9]+$ ]]
}

if ! is_uint "$poll_seconds" || (( poll_seconds < 10 )); then
  echo "RECOVERY_POLL_SECONDS must be an integer >= 10" >&2
  exit 64
fi
if ! is_uint "$chunk_leads" || (( chunk_leads < 1 )); then
  echo "RECOVERY_CHUNK_LEADS must be a positive integer" >&2
  exit 64
fi

exec 7>"$supervisor_lock"
if ! flock -n 7; then
  log "Another recovery supervisor owns $supervisor_lock; exiting."
  exit 75
fi

printf '%s\n' "$$" > "$supervisor_pid_file"
cleanup() {
  rm -f "$supervisor_pid_file"
}
terminate() {
  log "Recovery supervisor received a termination signal; exiting."
  exit 143
}
trap cleanup EXIT
trap terminate INT TERM

recovery_is_blocked() {
  local manifest="$project_root/data/ecmwf/recovery/${init_month}-instantaneous/recovery_manifest.json"
  [[ -f "$manifest" ]] || return 1
  python3 - "$manifest" <<'PY'
import json
import sys

with open(sys.argv[1], encoding="utf-8") as handle:
    manifest = json.load(handle)
raise SystemExit(0 if manifest.get("status") == "blocked_external" else 1)
PY
}

process_matches() {
  local pid="$1"
  local required="$2"
  local args
  is_uint "$pid" || return 1
  kill -0 "$pid" 2>/dev/null || return 1
  args="$(ps -p "$pid" -o args= 2>/dev/null)" || return 1
  [[ "$args" == *"$required"* ]]
}

worker_alive() {
  local pid
  [[ -s "$worker_pid_file" ]] || return 1
  read -r pid < "$worker_pid_file"
  process_matches "$pid" "dr_gcm.recover_ecmwf_instantaneous --init $init_month"
}

validate_target() {
  PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$project_root/src" \
    python3 - "$target" "$init_month" "$project_root" <<'PY'
import sys
from pathlib import Path

from dr_gcm.download_ecmwf_asia import load_domain
from dr_gcm.download_ecmwf_baseline import lead_hours_for_group, validate_group_file

target = Path(sys.argv[1])
init_month = sys.argv[2]
project_root = Path(sys.argv[3])
domain = load_domain(project_root / "config" / "ecmwf_domain.json")
spec = domain["download_groups"]["instantaneous"]
expected_leads = lead_hours_for_group(init_month, spec)
validate_group_file(
    target,
    init_month,
    spec,
    expected_leads,
)
PY
}

start_worker() {
  log "Recovery worker is absent; starting/resuming init=$init_month chunk_leads=$chunk_leads."
  (
    cd "$project_root" || exit 1
    nohup setsid env \
      PYTHONDONTWRITEBYTECODE=1 \
      PYTHONPATH="$project_root/src" \
      python3 -m dr_gcm.recover_ecmwf_instantaneous \
        --init "$init_month" \
        --chunk-leads "$chunk_leads" \
        --log-level INFO \
      >> "$worker_log" 2>&1 < /dev/null &
    printf '%s\n' "$!" > "$worker_pid_file"
  )
  sleep 2
  if worker_alive; then
    local pid
    read -r pid < "$worker_pid_file"
    log "Recovery worker started pid=$pid."
    return 0
  fi
  log "Recovery worker failed to remain active; it will be retried after the poll interval."
  return 1
}

watchdog_alive() {
  local pid
  [[ -s "$watchdog_pid_file" ]] || return 1
  read -r pid < "$watchdog_pid_file"
  process_matches "$pid" "baseline_watchdog.sh"
}

start_watchdog() {
  if watchdog_alive; then
    local pid
    read -r pid < "$watchdog_pid_file"
    log "Baseline watchdog is already active pid=$pid."
    return 0
  fi

  exec 8>"$watchdog_lock"
  if ! flock -n 8; then
    log "Baseline watchdog lock is already owned; treating handoff as active."
    return 0
  fi
  flock -u 8

  log "Starting baseline watchdog after successful strict validation."
  (
    cd "$project_root" || exit 1
    nohup setsid bash "$watchdog" >> "$log_dir/baseline_watchdog.log" 2>&1 < /dev/null &
    printf '%s\n' "$!" > "$watchdog_pid_file"
  )
  sleep 2
  if watchdog_alive; then
    local pid
    read -r pid < "$watchdog_pid_file"
    log "Baseline watchdog handoff complete pid=$pid."
    return 0
  fi
  log "Baseline watchdog failed to remain active."
  return 1
}

log "Recovery supervisor started pid=$$ init=$init_month poll_seconds=$poll_seconds."
while true; do
  if [[ -e "$target" ]]; then
    if validate_target >> "$supervisor_log" 2>&1; then
      log "Recovery target passed strict validation: $target"
      if start_watchdog; then
        log "Recovery supervision complete; normal baseline supervision resumed."
        exit 0
      fi
      log "Watchdog handoff failed; retrying after the poll interval."
    else
      log "FATAL: recovery target exists but failed strict validation; preserving it and stopping."
      exit 1
    fi
  elif worker_alive; then
    read -r worker_pid < "$worker_pid_file"
    log "Recovery remains active pid=$worker_pid."
  elif recovery_is_blocked; then
    log "Recovery is blocked by confirmed external data unavailability; stopping automatic retries."
    exit 69
  else
    start_worker || true
  fi

  if [[ "$once" == true ]]; then
    log "One-shot check complete."
    exit 0
  fi
  sleep "$poll_seconds"
done