#!/usr/bin/env bash
# Wait for the baseline/recovery monitor, then keep the soil-moisture worker alive.
set -u -o pipefail

project_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
poll_seconds=${ECMWF_SM_MONITOR_POLL_SECONDS:-300}
pilot_cooldown_seconds=${ECMWF_SM_PILOT_COOLDOWN_SECONDS:-86400}
log_dir="$project_root/outputs/logs"
monitor_log="$log_dir/ecmwf_soil_moisture_monitor.log"
monitor_lock="$log_dir/ecmwf_soil_moisture_monitor.lock"
monitor_pid="$log_dir/ecmwf_soil_moisture_monitor.pid"
baseline_monitor_pid="$log_dir/ecmwf_baseline_monitor.pid"
worker_pid="$log_dir/ecmwf_soil_moisture_download.pid"
pilot_failure_file="$log_dir/ecmwf_soil_moisture_pilot.last_failure"
mkdir -p "$log_dir"

exec 9>"$monitor_lock"
if ! flock -n 9; then
  printf '%s Soil-moisture monitor already active.\n' "$(date -Is)" >>"$monitor_log"
  exit 75
fi
printf '%s\n' "$$" >"$monitor_pid"
trap 'rm -f "$monitor_pid"' EXIT

pid_is_alive() {
  local path=$1 pid
  [[ -f "$path" ]] || return 1
  pid=$(cat "$path")
  [[ "$pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$pid" 2>/dev/null
}

printf '%s MONITOR_START range=1981-01..2020-12 poll=%ss\n' \
  "$(date -Is)" "$poll_seconds" >>"$monitor_log"
while true; do
  if pid_is_alive "$baseline_monitor_pid"; then
    printf '%s WAIT baseline_monitor=active\n' "$(date -Is)" >>"$monitor_log"
  elif pid_is_alive "$worker_pid"; then
    printf '%s HEARTBEAT soil_worker=active\n' "$(date -Is)" >>"$monitor_log"
  elif [[ -s "$project_root/data/ecmwf/soil_moisture/ecmwf_seas51_2020-12_soil_moisture.nc" ]]; then
    printf '%s MONITOR_COMPLETE range=1981-01..2020-12\n' "$(date -Is)" >>"$monitor_log"
    exit 0
  elif [[ -e "$pilot_failure_file" ]] && \
       (( $(date +%s) - $(stat -c %Y "$pilot_failure_file") < pilot_cooldown_seconds )); then
    printf '%s WAIT pilot_failure=cooldown cooldown=%ss\n' \
      "$(date -Is)" "$pilot_cooldown_seconds" >>"$monitor_log"
  else
    printf '%s LAUNCH soil_worker=detached\n' "$(date -Is)" >>"$monitor_log"
    nohup setsid bash "$project_root/scripts/download_ecmwf_soil_moisture.sh" \
      1981-01 2020-12 </dev/null >>"$monitor_log" 2>&1 &
  fi
  sleep "$poll_seconds"
done