← back to Nas Setup

scripts/daemon-health.sh

193 lines

#!/bin/bash
# daemon-health.sh — HONEST read-only health of the on-prem backup MIRROR (belt-and-suspenders:
# the -root SYSTEM daemon + the gui/ USER agent, both redundant writers to /Volumes/Henry).
#
# Why (TK-10547): pull.log + latest.json reflect whoever ran the script LAST (incl. a MANUAL
# user-context rescue run), so a root daemon exiting 1 every night (TCC can't write /Volumes/
# Henry) hides behind a rosy manual PARTIAL. Worse, dw-backup-canary only checks the KAMATERA
# SOURCE dump, never the Henry MIRROR; and pull-dw-dump.sh's own FAIL email is dead in the root
# daemon (HOME=/var/root has no george-send.sh) so failures went CNCP-only, no email.
#
# This probe reads each root daemon's launchd exit code (NON-sudo) + the newest Henry dump's
# freshness and emits a PASS/WARN/FAIL verdict to stdout AND a JSON heartbeat. Run WITH --alert
# (user context, where george-send.sh exists) to post a CNCP card + email Steve on FAIL.
#
# READ-ONLY on the backup system: no sudo, no launchd mutation, no backup run, no writes to the
# pipeline's latest.json. Writes only its own data/daemon-health-latest.json. $0 local.
set -uo pipefail

ALERT=0; [ "${1:-}" = "--alert" ] && ALERT=1
STALE_WARN_H="${STALE_WARN_H:-30}"   # a nightly 03:45 job's Henry dump older than this = missed run
# Strictness AUTO-DETECTS from the loaded root LaunchDaemon (Steve 8/15, TK-10547 — root-belt finish):
# while the daemon still calls /opt/homebrew/bin/bash the root belt is accepted-down pending FDA, so
# ONE belt down = PASS (data safe via the user belt). The moment Option B's installer swaps the loaded
# plist to the stable signed launcher /usr/local/bin/nas-backup-sh, BOTH belts are expected up and we
# flip to STRICT: one belt down = WARN (degraded redundancy, loud). BOTH down / Henry stale = FAIL
# always. This means the single `sudo install-optionB-launcher.sh` finishes everything — no manual
# flag flip, and no premature WARN before the install. Env REQUIRE_BOTH_BELTS=1|0 overrides.
_rp=/Library/LaunchDaemons/com.steve.nas-dwdump-mirror-root.plist
if /usr/libexec/PlistBuddy -c 'Print :ProgramArguments:0' "$_rp" 2>/dev/null | grep -q '/usr/local/bin/nas-backup-sh'; then _autoboth=1; else _autoboth=0; fi
REQUIRE_BOTH_BELTS="${REQUIRE_BOTH_BELTS:-$_autoboth}"
HERE="$(cd "$(dirname "$0")" && pwd)"; DATA="$HERE/../data"; mkdir -p "$DATA"
OUT="$DATA/daemon-health-latest.json"
now=$(date +%s)
MYUID=$(id -u)   # (kept for reference; the canonical scheduled job is the -root SYSTEM daemon per Option B)
worst="PASS"   # PASS < WARN < FAIL
rank(){ case "$1" in FAIL) echo 2;; WARN) echo 1;; *) echo 0;; esac; }
rows=()

# check <db-name> <henry-dir> <dump-prefix> <base-label>
check(){
  local name="$1" dir="$2" prefix="$3" base="$4" logf="${5:-}"
  local rootp userp root_ec user_ec newest age_h fresh verdict reason
  # BELT-AND-SUSPENDERS (Steve ruled 8/14 "Both", TK-10547): the Henry mirror is written by TWO
  # redundant schedulers — the -root SYSTEM LaunchDaemon (Option B: durable, survives logout, needs
  # FDA on /usr/local/bin/nas-backup-sh) AND the gui/ USER LaunchAgent via /opt/homebrew/bin/bash
  # (Option C: runs while logged in, proven writing). So check BOTH; the verdict is OUTCOME-based on
  # Henry freshness (fresh via EITHER writer = data safe) but we still WARN if one belt is down so a
  # broken writer is never hidden — the ticket's whole point.
  rootp=$(launchctl print "system/${base}-root" 2>/dev/null)
  userp=$(launchctl print "gui/$MYUID/${base}" 2>/dev/null)
  # launchd shows "last exit code = (never exited)" for a loaded-but-never-run job (runs=0). That
  # raw text is NOT valid JSON, so feeding it to `jq --argjson` throws and the heartbeat latest.json
  # never lands — blinding fleet-health-rollup (the TK-10546 anti-pattern). Capture the raw text for
  # the human-readable writers string, but derive a JSON-safe number (or null) for the heartbeat, and
  # a status word so the reason can say "never ran" instead of the misleading "broken" (TK-10547).
  local root_raw user_raw root_json user_json root_word user_word
  root_raw=$(printf '%s\n' "$rootp" | awk -F'= ' '/last exit code/{print $2; exit}')
  user_raw=$(printf '%s\n' "$userp" | awk -F'= ' '/last exit code/{print $2; exit}')
  ec_json(){ case "$1" in ''|*[!0-9]*) echo null;; *) echo "$1";; esac; }
  ec_word(){ # $1=loaded-print $2=raw-exit -> unloaded|never-ran|exit:N
    [ -z "$1" ] && { echo unloaded; return; }
    case "$2" in ''|'(never'*|*[!0-9]*) echo never-ran;; 0) echo exit:0;; *) echo "exit:$2";; esac; }
  root_json=$(ec_json "$root_raw"); user_json=$(ec_json "$user_raw")
  root_word=$(ec_word "$rootp" "$root_raw"); user_word=$(ec_word "$userp" "$user_raw")
  local root_ok=0 user_ok=0
  [ -n "$rootp" ] && [ "$root_json" = "0" ] && root_ok=1
  [ -n "$userp" ] && [ "$user_json" = "0" ] && user_ok=1
  # launchd's runs/exit-code RESETS to 0 on every reload/reboot and shows "(never exited)" for a
  # loaded-but-not-yet-refired job, so it cannot prove the USER belt landed a scheduled clean run
  # (TK-10547 follow-up: after the Aug pm2-fracture reboots it false-FAILed while the belt wrote a
  # VERIFIED Henry copy nightly). Ground truth = the belt's OWN success log: a recent
  # "[Henry] PASS: <prefix>_..." line means the most recent run landed a verified copy. Recency
  # (log mtime within the freshness window) preserves the anti-stale guard — a dead belt stops
  # appending PASS and its dump goes stale -> still FAIL. This is STRICTER than the exit code: it
  # validates the actual outcome, not just that launchd thinks a process exited 0.
  local user_landed=0
  if [ -n "$logf" ] && [ -f "$logf" ]; then
    local lmtime lage_h
    lmtime=$(stat -f %m "$logf" 2>/dev/null || echo 0)
    lage_h=$(( (now - lmtime) / 3600 ))
    if [ "$lage_h" -le "$STALE_WARN_H" ] && tail -n 25 "$logf" | grep -q "\[Henry\] PASS: ${prefix}_"; then
      user_landed=1; user_ok=1
    fi
  fi
  local up=$(( root_ok + user_ok ))

  newest=$(ls -t "$dir/${prefix}"_*.dump 2>/dev/null | head -1)
  if [ -n "$newest" ]; then
    local mtime; mtime=$(stat -f %m "$newest" 2>/dev/null || echo 0)
    age_h=$(( (now - mtime) / 3600 ))
    [ "$age_h" -le "$STALE_WARN_H" ] && fresh="fresh" || fresh="stale"
  else age_h=-1; fresh="missing"; fi

  local writers="root(sys)=${root_word} user(gui)=${user_word}"
  [ "$user_landed" = "1" ] && writers="$writers user-log=verified-PASS@${lage_h}h"
  # Henry-freshness is authoritative. Fresh via at least one healthy writer = data safe; both writers
  # healthy = PASS; exactly one down = WARN (degraded redundancy, fix the down belt); both down or
  # Henry missing/stale = FAIL (no working writer → the silent-death setup).
  if [ "$fresh" = "missing" ]; then verdict="FAIL"; reason="no Henry dump for $prefix — neither writer landed [$writers]"
  elif [ "$fresh" = "stale" ]; then verdict="FAIL"; reason="Henry dump ${age_h}h old (> ${STALE_WARN_H}h) — neither writer is landing [$writers]"
  elif [ "$up" -eq 0 ]; then verdict="FAIL"; reason="Henry fresh (${age_h}h) but NO scheduled writer has landed a clean run — the mirror is only being kept fresh by manual rescue; the automated belts are not proven [$writers]"
  elif [ "$up" -eq 1 ] && [ "$REQUIRE_BOTH_BELTS" = "1" ]; then verdict="WARN"; reason="Henry fresh (${age_h}h) via one writer — redundancy DEGRADED, other belt down [$writers]"
  elif [ "$up" -eq 1 ]; then verdict="PASS"; reason="Henry fresh (${age_h}h) via a proven writer (verified PASS in its own log); root belt accepted-down pending FDA (TK-10547) [$writers]"
  else verdict="PASS"; reason="Henry fresh (${age_h}h), both writers healthy [$writers]"; fi
  [ "$(rank "$verdict")" -gt "$(rank "$worst")" ] && worst="$verdict"

  echo "  $name -> $verdict ($reason)"
  echo "     henry: $fresh ${newest:+$(basename "$newest")} (${age_h}h) | writers: $writers"
  rows+=("$(jq -n --arg l "$name" --arg v "$verdict" --arg r "$reason" \
     --argjson rec "${root_json:-null}" --argjson uec "${user_json:-null}" --argjson age "${age_h:-null}" \
     '{db:$l,verdict:$v,reason:$r,root_last_exit:$rec,user_last_exit:$uec,henry_dump_age_h:$age}')")
}

echo "== on-prem backup daemon health ($(date -Iseconds)) =="
check dw_unified  /Volumes/Henry/dw-backups/dw_unified  dw_unified  com.steve.nas-dwdump-mirror          "$HERE/../data/launchd.out.log"
check realestate  /Volumes/Henry/dw-backups/realestate  realestate  com.steve.nas-realestate-dump-mirror  "$HERE/../data/launchd-realestate.out.log"

# --- repo/config bundle belt (added 2026-09-10, TK-11233 follow-up) ---------
# backup-repos-to-henry.sh writes a PASS/WARN/FAIL verdict but NOTHING read it, so a
# silently-dead bundler emitted zero alerts - the exact shape of the 12-day pg_dump death.
# It is now the ONLY off-machine copy of ~/.claude (452 skill definitions) and its 73
# nested skill repos, so its freshness matters as much as the DB mirrors.
REPO_JSON="$HERE/../data/repo-backup-latest.json"
REPO_STALE_H="${REPO_STALE_H:-36}"   # daily job at 04:30; 36h = one missed run + slack
rv=FAIL; rr="repo-backup-latest.json missing - bundler has never run or its data dir moved"
if [ -f "$REPO_JSON" ]; then
  rts=$(jq -r '.ts // empty' "$REPO_JSON" 2>/dev/null)
  rvd=$(jq -r '.verdict // "UNKNOWN"' "$REPO_JSON" 2>/dev/null)
  rok=$(jq -r '.repos_ok // 0' "$REPO_JSON" 2>/dev/null)
  rfail=$(jq -r '.repos_fail // 0' "$REPO_JSON" 2>/dev/null)
  if [ -n "$rts" ]; then
    # -u is REQUIRED: the ts is UTC (trailing Z) and macOS `date -j -f` otherwise parses it
    # as LOCAL time, yielding a future epoch and a NEGATIVE age that silently passes the
    # staleness test. Clamp negatives to 0 and treat a large negative as clock skew.
    rts_epoch=$(date -j -u -f "%Y-%m-%dT%H:%M:%SZ" "$rts" +%s 2>/dev/null || echo 0)
    rage=$(( ( $(date +%s) - rts_epoch ) / 3600 ))
    [ "$rage" -lt 0 ] && rage=0
    if [ "$rage" -gt "$REPO_STALE_H" ]; then rv=FAIL; rr="last bundle run ${rage}h ago (> ${REPO_STALE_H}h) - the only off-machine copy of ~/.claude is going stale"
    elif [ "$rvd" != "PASS" ]; then rv=WARN; rr="bundler reported $rvd (${rfail} repos failed) ${rage}h ago"
    else
      # Freshness is necessary but not sufficient: assert ~/.claude is actually IN the set.
      if [ -f /Volumes/Henry/mac2-archive/repo-backups/dotclaude.bundle ]; then
        rv=PASS; rr="bundled ${rok} repos ${rage}h ago incl. dotclaude + $(ls /Volumes/Henry/mac2-archive/repo-backups/dotclaude-skill-*.bundle 2>/dev/null | wc -l | tr -d ' ') nested skill repos"
      else
        rv=FAIL; rr="bundler PASSed ${rage}h ago but dotclaude.bundle is ABSENT - ~/.claude is not actually being backed up"
      fi
    fi
  fi
fi
[ "$(rank "$rv")" -gt "$(rank "$worst")" ] && worst="$rv"
echo "  repo-bundles -> $rv ($rr)"
rows+=("$(jq -n --arg l "repo-bundles" --arg v "$rv" --arg r "$rr" \
   '{db:$l,verdict:$v,reason:$r,root_last_exit:null,user_last_exit:null,henry_dump_age_h:null}')")

echo "== overall: $worst =="

# JSON heartbeat (PASS/WARN/FAIL vocab so fleet-health-rollup + meta-watchdog read it right)
printf '%s\n' "${rows[@]}" | jq -s --arg v "$worst" \
  '{checked_at:(now|todate),verdict:$v,daemons:.}' > "$OUT"
# Also publish where the FLEET monitors read it (~/.claude/skills/*/data/latest.json) so the
# morning fleet-health-rollup panel + dw-canary-meta-watchdog surface the Henry-mirror health.
ROLLUP_DIR="$HOME/.claude/skills/backup-daemon-health/data"
[ -d "$ROLLUP_DIR" ] && cp "$OUT" "$ROLLUP_DIR/latest.json" 2>/dev/null || true

# --alert with DE-NAG: only alert on a WORSENING transition (e.g. PASS->FAIL) or a daily re-nag
# while still non-PASS — so an hourly canary can't spam ~24 FAIL emails/day. Matches the fleet
# canary convention. Runs in USER context, where george-send.sh + ~/.claude.json creds EXIST
# (the root daemon's own email path is dead — HOME=/var/root). Default (no --alert) = never sends.
STATE="$DATA/daemon-health-alert-state"; RENAG_H="${RENAG_H:-20}"
if [ "$ALERT" = "1" ]; then
  last_v="PASS"; last_t=0
  [ -f "$STATE" ] && { last_v=$(cut -d' ' -f1 "$STATE" 2>/dev/null || echo PASS); last_t=$(cut -d' ' -f2 "$STATE" 2>/dev/null || echo 0); }
  fire=0
  if [ "$worst" != "PASS" ]; then
    [ "$(rank "$worst")" -gt "$(rank "$last_v")" ] && fire=1                       # worsened
    [ $(( (now - last_t) / 3600 )) -ge "$RENAG_H" ] && fire=1                      # daily re-nag while broken
  fi
  if [ "$fire" = "1" ]; then
    CNCP="${CNCP_URL:-http://localhost:3333}"
    note="[BACKUP DAEMON HEALTH $(date +%F)] Henry mirror $worst — $(printf '%s\n' "${rows[@]}" | jq -r 'select(.verdict!="PASS")|"\(.db): \(.reason)"' | paste -sd'; ' -). Belt-and-suspenders: WARN = one of the two writers is down (data still safe via the other); FAIL = both down or Henry stale. To restore the root belt, grant FDA to /usr/local/bin/nas-backup-sh (TK-10547)."
    curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
      -d "$(jq -n --arg u "onprem://henry-mirror-daemon" --arg note "$note" '{url:$u,note:$note}')" >/dev/null 2>&1 || true
    if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
      . "$HOME/.claude/skills/_shared/george-send.sh"
      body="<div style=\"font-family:-apple-system,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ Henry backup daemon $worst</h3><div>$note</div></div>"
      george_send steve-office "${BACKUP_HEALTH_TO:-steve@designerwallcoverings.com}" "⚠ Henry backup daemon $worst — $(date +%F)" "$body" >/dev/null 2>&1 || true
    fi
  fi
  echo "$worst $now" > "$STATE"   # remember state so we don't re-nag until RENAG_H passes / it worsens
fi

[ "$worst" = "FAIL" ] && exit 1 || exit 0