daemon-health: fix false-PASS regression — restore mechanism-loud verdict (broken daemon=FAIL even when Henry manually-fresh) + watch the -root SYSTEM daemons not the retired gui/ user agents (Steve ruled 8/14, TK-10547)

272bd47d2ee2aef6e176b285d57181f18c03d34a · 2026-08-14 09:54:23 -0700 · Steve Abrams
Files touched

M scripts/daemon-health.sh
Diff

commit 272bd47d2ee2aef6e176b285d57181f18c03d34a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Fri Aug 14 09:54:23 2026 -0700

    daemon-health: fix false-PASS regression — restore mechanism-loud verdict (broken daemon=FAIL even when Henry manually-fresh) + watch the -root SYSTEM daemons not the retired gui/ user agents (Steve ruled 8/14, TK-10547)
---
 scripts/daemon-health.sh | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/scripts/daemon-health.sh b/scripts/daemon-health.sh
index c4bffe1..ace052b 100755
--- a/scripts/daemon-health.sh
+++ b/scripts/daemon-health.sh
@@ -20,7 +20,7 @@ STALE_WARN_H="${STALE_WARN_H:-30}"   # a nightly 03:45 job's Henry dump older th
 HERE="$(cd "$(dirname "$0")" && pwd)"; DATA="$HERE/../data"; mkdir -p "$DATA"
 OUT="$DATA/daemon-health-latest.json"
 now=$(date +%s)
-MYUID=$(id -u)   # the mirror now runs as a USER LaunchAgent (gui/$MYUID) via /opt/homebrew/bin/bash
+MYUID=$(id -u)   # (kept for reference; the canonical scheduled job is the -root SYSTEM daemon per Option B)
 worst="PASS"   # PASS < WARN < FAIL
 rank(){ case "$1" in FAIL) echo 2;; WARN) echo 1;; *) echo 0;; esac; }
 rows=()
@@ -29,10 +29,10 @@ rows=()
 check(){
   local label="$1" dir="$2" prefix="$3"
   local print exit_code runs state verdict reason newest age_h fresh
-  # The mirror now runs as a USER LaunchAgent (gui/$MYUID) via /opt/homebrew/bin/bash — the non-SIP
-  # launcher that CAN write the /Volumes/Henry external volume (the old root daemon's signed launcher
-  # was TCC-denied on the removable volume, failing every night). So look it up in gui/, not system/.
-  print=$(launchctl print "gui/$MYUID/$label" 2>/dev/null)
+  # Option B (Steve-approved, TK-10547): the canonical scheduled job is the -root SYSTEM LaunchDaemon
+  # (runs nightly 03:45 as root). Option C (a user LaunchAgent) was REJECTED — it hits the same SIP-bash
+  # TCC wall and only runs while logged in. So look up the actually-scheduled job in system/, not gui/.
+  print=$(launchctl print "system/$label" 2>/dev/null)
   exit_code=$(printf '%s\n' "$print" | awk -F'= ' '/last exit code/{print $2; exit}')
   runs=$(printf '%s\n' "$print" | awk -F'= ' '/^\truns/{print $2; exit}')
   state=$(printf '%s\n' "$print" | awk -F'= ' '/^\tstate/{print $2; exit}')
@@ -44,15 +44,16 @@ check(){
     [ "$age_h" -le "$STALE_WARN_H" ] && fresh="fresh" || fresh="stale"
   else age_h=-1; fresh="missing"; fi
 
-  # Henry-freshness is AUTHORITATIVE — it's the actual off-machine backup we care about; the agent's
-  # exit code is secondary context. Missing/stale Henry = FAIL (the silent-death signal we exist to
-  # catch). A FRESH Henry dump means data is safe even if the agent hiccupped: agent-unloaded or
-  # exit!=0 with a fresh dump = WARN (worth a look, not a data emergency). Fresh + exit 0 = PASS.
-  if [ "$fresh" = "missing" ]; then verdict="FAIL"; reason="no Henry dump for $prefix — mirror never landed"
-  elif [ "$fresh" = "stale" ]; then verdict="FAIL"; reason="Henry dump ${age_h}h old (> ${STALE_WARN_H}h) — nightly mirror not landing"
-  elif [ -z "$print" ]; then verdict="WARN"; reason="agent not loaded but Henry dump fresh (${age_h}h) — data safe, scheduler gone"
-  elif [ "${exit_code:-1}" != "0" ]; then verdict="WARN"; reason="agent last_exit=${exit_code:-?} but Henry dump fresh (${age_h}h) — data safe, check writer"
-  else verdict="PASS"; reason="agent exit 0, Henry dump ${age_h}h old"; fi
+  # MECHANISM-LOUD verdict (Steve ruled 8/14, TK-10547): a broken SCHEDULED daemon = FAIL even when a
+  # manual rescue left Henry fresh — the entire point of this ticket is that monitoring must NOT hide a
+  # broken daemon behind a rosy manual PARTIAL. Henry-freshness enriches the REASON (data-safe vs
+  # data-lost), it does NOT soften the verdict. Only a clean daemon (exit 0) + fresh Henry = PASS.
+  if [ -z "$print" ]; then verdict="FAIL"; reason="launchd state unreadable for $label — scheduler gone"
+  elif [ "${exit_code:-1}" != "0" ] && [ "$fresh" = "fresh" ]; then verdict="FAIL"; reason="daemon last_exit=${exit_code:-?} but Henry dump fresh (${age_h}h) — mechanism broken, data safe (launcher swap pending)"
+  elif [ "${exit_code:-1}" != "0" ]; then verdict="FAIL"; reason="daemon last_exit=${exit_code:-?} — mechanism broken AND Henry ${fresh} (${age_h}h)"
+  elif [ "$fresh" = "missing" ]; then verdict="FAIL"; reason="no Henry dump for $prefix — mirror never landed"
+  elif [ "$fresh" = "stale" ]; then verdict="WARN"; reason="Henry dump ${age_h}h old (> ${STALE_WARN_H}h) — daemon exit 0 but mirror not landing"
+  else verdict="PASS"; reason="daemon exit 0, Henry dump ${age_h}h old"; fi
   [ "$(rank "$verdict")" -gt "$(rank "$worst")" ] && worst="$verdict"
 
   echo "  $label -> $verdict ($reason)"
@@ -63,8 +64,8 @@ check(){
 }
 
 echo "== on-prem backup daemon health ($(date -Iseconds)) =="
-check com.steve.nas-dwdump-mirror          /Volumes/Henry/dw-backups/dw_unified  dw_unified
-check com.steve.nas-realestate-dump-mirror /Volumes/Henry/dw-backups/realestate  realestate
+check com.steve.nas-dwdump-mirror-root          /Volumes/Henry/dw-backups/dw_unified  dw_unified
+check com.steve.nas-realestate-dump-mirror-root /Volumes/Henry/dw-backups/realestate  realestate
 echo "== overall: $worst =="
 
 # JSON heartbeat (PASS/WARN/FAIL vocab so fleet-health-rollup + meta-watchdog read it right)
← d3111b7 Henry mirror: switch to USER LaunchAgents via homebrew bash · back to Nas Setup · daemon-health: belt-and-suspenders model (Steve ruled 8/14 ' 4a447c8 →