#!/usr/bin/env bash
# Keep the approved one-worker baseline download progressing across disconnects.
# Completed raw pairs and explicitly deferred damaged-tape months are skipped;
# every other partial/missing month is resumed from the first gap at/after START.
set -u -o pipefail

project_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
start_month=${1:-1990-04}
end_month=${2:-2020-12}
poll_seconds=${ECMWF_MONITOR_POLL_SECONDS:-300}
recovery_cooldown_seconds=${ECMWF_RECOVERY_COOLDOWN_SECONDS:-86400}
once=0
if [[ ${3:-} == "--once" ]]; then
  once=1
fi

log_dir="$project_root/outputs/logs"
raw_dir="$project_root/data/ecmwf/raw_split"
monitor_log="$log_dir/ecmwf_baseline_monitor.log"
monitor_lock="$log_dir/ecmwf_baseline_monitor.lock"
monitor_pid="$log_dir/ecmwf_baseline_monitor.pid"
download_lock="$log_dir/ecmwf_baseline_download.lock"
deferred_gap_file="$log_dir/ecmwf_baseline_deferred_gaps.tsv"
known_gap_file="$project_root/config/ecmwf_known_damaged_tape_gaps.tsv"
recovery_lock="$log_dir/ecmwf_deferred_recovery.lock"
mkdir -p "$log_dir"

if ! [[ "$start_month" =~ ^[0-9]{4}-(0[1-9]|1[0-2])$ ]] || \
   ! [[ "$end_month" =~ ^[0-9]{4}-(0[1-9]|1[0-2])$ ]] || \
   [[ "$start_month" > "$end_month" ]] || \
   ! [[ "$poll_seconds" =~ ^[1-9][0-9]*$ ]] || \
   ! [[ "$recovery_cooldown_seconds" =~ ^[1-9][0-9]*$ ]]; then
  printf 'Invalid monitor range/poll/cooldown: start=%s end=%s poll=%s cooldown=%s\n' \
    "$start_month" "$end_month" "$poll_seconds" "$recovery_cooldown_seconds" >&2
  exit 2
fi

exec 9>"$monitor_lock"
if ! flock -n 9; then
  printf '%s Monitor already holds %s; exiting.\n' "$(date -Is)" "$monitor_lock" >>"$monitor_log"
  exit 75
fi
printf '%s\n' "$$" >"$monitor_pid"
trap 'rm -f "$monitor_pid"' EXIT

month_after() {
  date -d "${1}-01 +1 month" +%Y-%m
}

is_deferred() {
  local month=$1
  [[ -f "$deferred_gap_file" ]] && \
    awk -F '\t' -v month="$month" \
      '$2 == month && $3 == "damaged_tape" { found=1 } END { exit(found ? 0 : 1) }' \
      "$deferred_gap_file"
}

next_required_month() {
  local month="$start_month" instantaneous accumulated
  while [[ "$month" < "$end_month" || "$month" == "$end_month" ]]; do
    instantaneous="$raw_dir/ecmwf_seas51_${month}_instantaneous.nc"
    accumulated="$raw_dir/ecmwf_seas51_${month}_accumulated.nc"
    if [[ -s "$instantaneous" && -s "$accumulated" ]]; then
      month=$(month_after "$month")
      continue
    fi
    if is_deferred "$month"; then
      month=$(month_after "$month")
      continue
    fi
    printf '%s\n' "$month"
    return 0
  done
  return 1
}

download_is_active() {
  exec 8>"$download_lock"
  if flock -n 8; then
    flock -u 8
    exec 8>&-
    return 1
  fi
  exec 8>&-
  return 0
}

recovery_is_active() {
  exec 7>"$recovery_lock"
  if flock -n 7; then
    flock -u 7
    exec 7>&-
    return 1
  fi
  exec 7>&-
  return 0
}

all_deferred_gap_months() {
  {
    if [[ -f "$known_gap_file" ]]; then
      awk -F '\t' '$1 ~ /^[0-9]{4}-[0-9]{2}$/ && $2 == "damaged_tape" { print $1 }' \
        "$known_gap_file"
    fi
    if [[ -f "$deferred_gap_file" ]]; then
      awk -F '\t' '$2 ~ /^[0-9]{4}-[0-9]{2}$/ && $3 == "damaged_tape" { print $2 }' \
        "$deferred_gap_file"
    fi
  } | sort -u
}

gap_is_missing() {
  local month=$1
  [[ ! -s "$raw_dir/ecmwf_seas51_${month}_instantaneous.nc" || \
     ! -s "$raw_dir/ecmwf_seas51_${month}_accumulated.nc" ]]
}

missing_deferred_gap_exists() {
  local month
  while IFS= read -r month; do
    if gap_is_missing "$month"; then
      return 0
    fi
  done < <(all_deferred_gap_months)
  return 1
}

next_recovery_gap() {
  local month stamp now stamp_epoch
  now=$(date +%s)
  while IFS= read -r month; do
    gap_is_missing "$month" || continue
    stamp="$log_dir/ecmwf_recovery_${month}.last_attempt"
    if [[ -e "$stamp" ]]; then
      stamp_epoch=$(stat -c %Y "$stamp")
      if (( now - stamp_epoch < recovery_cooldown_seconds )); then
        continue
      fi
    fi
    printf '%s\n' "$month"
    return 0
  done < <(all_deferred_gap_months)
  return 1
}

start_recovery() {
  local month=$1
  printf '%s RECOVERY_LAUNCH init=%s cooldown=%ss\n' \
    "$(date -Is)" "$month" "$recovery_cooldown_seconds" >>"$monitor_log"
  nohup setsid bash "$project_root/scripts/recover_deferred_ecmwf_gap.sh" "$month" \
    </dev/null >>"$log_dir/ecmwf_recovery_${month}.log" 2>&1 &
}

printf '%s MONITOR_START range=%s..%s poll=%ss\n' \
  "$(date -Is)" "$start_month" "$end_month" "$poll_seconds" >>"$monitor_log"

while true; do
  if download_is_active; then
    printf '%s HEARTBEAT downloader=active\n' "$(date -Is)" >>"$monitor_log"
  else
    if next_month=$(next_required_month); then
      printf '%s RESTART next=%s end=%s workers=1\n' \
        "$(date -Is)" "$next_month" "$end_month" >>"$monitor_log"
      bash "$project_root/scripts/submit_ecmwf_baseline_detached.sh" \
        "$next_month" "$end_month" 1 >>"$monitor_log" 2>&1
    else
      if recovery_is_active; then
        printf '%s HEARTBEAT recovery=active\n' "$(date -Is)" >>"$monitor_log"
      elif recovery_month=$(next_recovery_gap); then
        start_recovery "$recovery_month"
      elif missing_deferred_gap_exists; then
        printf '%s HEARTBEAT recovery=cooldown\n' "$(date -Is)" >>"$monitor_log"
      else
        printf '%s MONITOR_COMPLETE range=%s..%s gaps=recovered\n' \
          "$(date -Is)" "$start_month" "$end_month" >>"$monitor_log"
        exit 0
      fi
    fi
  fi
  if (( once == 1 )); then
    exit 0
  fi
  sleep "$poll_seconds"
done