#!/bin/bash
# check_mk local check: has the overnight run succeeded lately, for the host and each database?
# Installed by muli-neomuli-jobs; neomuli-jobs records every run in login_db.
#
# One service per target. CRIT when the latest run failed, or nothing has succeeded in 26 hours;
# WARN when the latest run was skipped (a database held for a restore) or no run is recorded yet.
# The overnight run was dead on ten hosts for up to fourteen months in 2025-26 with nothing to show
# it; this is what shows it.

limit_hours=26
psql_login() { psql -h localhost -p 59999 -U muli -d login_db -AtF '|' -c "$1" 2>/dev/null; }

if ! psql_login "SELECT 1" >/dev/null; then
  echo "2 \"Muli overnight\" - Cannot read the overnight run's records from login_db"
  exit 0
fi

targets="host $(cut -c1-6 /usr/local/mulidata/site/contract_list 2>/dev/null)"
for target in $targets; do
  # The latest run, and the newest success: age in hours, status, business date, message.
  row=$(psql_login "
    SELECT (SELECT round(extract(epoch FROM now() - max(finished_utc)) / 3600)
              FROM job_runs WHERE job = 'overnight' AND target = '$target' AND status = 'succeeded'),
           status, business_date, coalesce(message, '')
    FROM job_runs WHERE job = 'overnight' AND target = '$target'
    ORDER BY business_date DESC, id DESC LIMIT 1")
  name="\"Muli overnight $target\""
  if [ -z "$row" ]; then echo "1 $name - No overnight run recorded yet"; continue; fi
  IFS='|' read -r age status date message <<< "$row"
  metric="hours_since_success=${age:-0};;$limit_hours"
  case "$status" in
    failed)  echo "2 $name $metric Run for $date: ${message:-failed}" ;;
    skipped) echo "1 $name $metric Skipped for $date: $message" ;;
    missed)  echo "2 $name $metric Missed $date: $message" ;;
    running) echo "0 $name $metric Running for $date" ;;
    *)
      if [ -z "$age" ] || [ "$age" -ge "$limit_hours" ]; then
        echo "2 $name $metric No successful run in ${age:-?} hours"
      else
        echo "0 $name $metric Succeeded for $date, ${age} hours ago"
      fi ;;
  esac
done
exit 0
