← back to Nas Setup
daemon-health: emit PASS/WARN/FAIL JSON heartbeat + guarded --alert (user-context CNCP+George), so fleet-rollup reads it right and a broken Henry daemon self-alerts (TK-10547)
e1954e8ae5c3445c380a7e5515652c7cb30aab04 · 2026-08-13 20:18:13 -0700 · Steve
Files touched
M scripts/daemon-health.sh
Diff
commit e1954e8ae5c3445c380a7e5515652c7cb30aab04
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Aug 13 20:18:13 2026 -0700
daemon-health: emit PASS/WARN/FAIL JSON heartbeat + guarded --alert (user-context CNCP+George), so fleet-rollup reads it right and a broken Henry daemon self-alerts (TK-10547)
---
scripts/daemon-health.sh | 84 ++++++++++++++++++++++++++++++++++--------------
1 file changed, 59 insertions(+), 25 deletions(-)
diff --git a/scripts/daemon-health.sh b/scripts/daemon-health.sh
index 2314872..5d6a11d 100755
--- a/scripts/daemon-health.sh
+++ b/scripts/daemon-health.sh
@@ -1,48 +1,82 @@
#!/bin/bash
# daemon-health.sh — HONEST read-only health of the on-prem backup ROOT DAEMONS.
#
-# Why this exists (TK-10547, yoloforever cycle 2): pull.log + latest.json reflect whoever
-# ran the script LAST — including a MANUAL user-context run — so a root daemon that has
-# been exiting 1 every night (TCC can't write /Volumes/Henry) hides behind a rosy manual
-# PARTIAL. This probe reads the DAEMON's own launchd state (readable WITHOUT sudo) plus the
-# freshness of the newest Henry dump, and prints a truthful per-daemon verdict.
+# Why (TK-10547): pull.log + latest.json reflect whoever ran the script LAST (incl. a MANUAL
+# user-context rescue run), so a root daemon exiting 1 every night (TCC can't write /Volumes/
+# Henry) hides behind a rosy manual PARTIAL. Worse, dw-backup-canary only checks the KAMATERA
+# SOURCE dump, never the Henry MIRROR; and pull-dw-dump.sh's own FAIL email is dead in the root
+# daemon (HOME=/var/root has no george-send.sh) so failures went CNCP-only, no email.
#
-# READ-ONLY: no sudo, no writes to latest.json, no launchd mutation, no backup run. $0 local.
+# This probe reads each root daemon's launchd exit code (NON-sudo) + the newest Henry dump's
+# freshness and emits a PASS/WARN/FAIL verdict to stdout AND a JSON heartbeat. Run WITH --alert
+# (user context, where george-send.sh exists) to post a CNCP card + email Steve on FAIL.
+#
+# READ-ONLY on the backup system: no sudo, no launchd mutation, no backup run, no writes to the
+# pipeline's latest.json. Writes only its own data/daemon-health-latest.json. $0 local.
set -uo pipefail
-STALE_HOURS="${STALE_HOURS:-30}" # a nightly 03:45 job older than this = missed a run
+ALERT=0; [ "${1:-}" = "--alert" ] && ALERT=1
+STALE_WARN_H="${STALE_WARN_H:-30}" # a nightly 03:45 job's Henry dump older than this = missed run
+HERE="$(cd "$(dirname "$0")" && pwd)"; DATA="$HERE/../data"; mkdir -p "$DATA"
+OUT="$DATA/daemon-health-latest.json"
now=$(date +%s)
-rc=0
+worst="PASS" # PASS < WARN < FAIL
+rank(){ case "$1" in FAIL) echo 2;; WARN) echo 1;; *) echo 0;; esac; }
+rows=()
-# daemon <label> <henry-dir> <dump-prefix>
+# check <label> <henry-dir> <dump-prefix>
check(){
local label="$1" dir="$2" prefix="$3"
- local print exit_code runs state
+ local print exit_code runs state verdict reason newest age_h fresh
print=$(launchctl print "system/$label" 2>/dev/null)
- if [ -z "$print" ]; then echo " $label: launchd state UNREADABLE"; rc=1; return; fi
exit_code=$(printf '%s\n' "$print" | awk -F'= ' '/last exit code/{print $2; exit}')
runs=$(printf '%s\n' "$print" | awk -F'= ' '/^\truns/{print $2; exit}')
state=$(printf '%s\n' "$print" | awk -F'= ' '/^\tstate/{print $2; exit}')
- # freshness of the newest Henry dump for this DB
- local newest age_h="?" fresh="NO DUMP"
newest=$(ls -t "$dir/${prefix}"_*.dump 2>/dev/null | head -1)
if [ -n "$newest" ]; then
local mtime; mtime=$(stat -f %m "$newest" 2>/dev/null || echo 0)
age_h=$(( (now - mtime) / 3600 ))
- if [ "$age_h" -le "$STALE_HOURS" ]; then fresh="FRESH (${age_h}h)"; else fresh="STALE (${age_h}h)"; fi
- fi
+ [ "$age_h" -le "$STALE_WARN_H" ] && fresh="fresh" || fresh="stale"
+ else age_h=-1; fresh="missing"; fi
- local verdict="OK"
- { [ "${exit_code:-1}" != "0" ] || [ "$fresh" = "NO DUMP" ] || [[ "$fresh" == STALE* ]]; } && { verdict="DEGRADED"; rc=1; }
- echo " $label"
- echo " daemon: state=${state:-?} runs=${runs:-?} last_exit=${exit_code:-?} (last_exit!=0 = the nightly write is failing)"
- echo " henry : $fresh ${newest:+($(basename "$newest"))}"
- echo " >>> $verdict"
+ # verdict: daemon exit!=0 = FAIL (the nightly write is failing); dump missing = FAIL;
+ # dump present but stale (exit somehow 0 yet old) = WARN; else PASS.
+ if [ -z "$print" ]; then verdict="FAIL"; reason="launchd state unreadable"
+ elif [ "${exit_code:-1}" != "0" ]; then verdict="FAIL"; reason="daemon last_exit=${exit_code:-?} (nightly write failing)"
+ elif [ "$fresh" = "missing" ]; then verdict="FAIL"; reason="no Henry dump for $prefix"
+ elif [ "$fresh" = "stale" ]; then verdict="WARN"; reason="Henry dump ${age_h}h old (> ${STALE_WARN_H}h)"
+ else verdict="PASS"; reason="daemon exit 0, Henry dump ${age_h}h old"; fi
+ [ "$(rank "$verdict")" -gt "$(rank "$worst")" ] && worst="$verdict"
+
+ echo " $label -> $verdict ($reason)"
+ echo " daemon: state=${state:-?} runs=${runs:-?} last_exit=${exit_code:-?} | henry: $fresh ${newest:+$(basename "$newest")} (${age_h}h)"
+ rows+=("$(jq -n --arg l "$label" --arg v "$verdict" --arg r "$reason" \
+ --argjson ec "${exit_code:-null}" --argjson runs "${runs:-null}" --argjson age "${age_h:-null}" \
+ '{daemon:$l,verdict:$v,reason:$r,last_exit:$ec,runs:$runs,henry_dump_age_h:$age}')")
}
echo "== on-prem backup daemon health ($(date -Iseconds)) =="
-check com.steve.nas-dwdump-mirror-root /Volumes/Henry/dw-backups/dw_unified dw_unified
-check com.steve.nas-realestate-dump-mirror-root /Volumes/Henry/dw-backups/realestate realestate
-echo "== overall: $([ $rc -eq 0 ] && echo HEALTHY || echo DEGRADED — a daemon is exit!=0 or its Henry dump is stale) =="
-exit $rc
+check com.steve.nas-dwdump-mirror-root /Volumes/Henry/dw-backups/dw_unified dw_unified
+check com.steve.nas-realestate-dump-mirror-root /Volumes/Henry/dw-backups/realestate realestate
+echo "== overall: $worst =="
+
+# JSON heartbeat (PASS/WARN/FAIL vocab so fleet-health-rollup + meta-watchdog read it right)
+printf '%s\n' "${rows[@]}" | jq -s --arg v "$worst" \
+ '{checked_at:(now|todate),verdict:$v,daemons:.}' > "$OUT"
+
+# --alert: on non-PASS, post CNCP card + email Steve. Runs in USER context, where george-send.sh
+# EXISTS (the root daemon's own email path is dead — HOME=/var/root). Default (no flag) = no send.
+if [ "$ALERT" = "1" ] && [ "$worst" != "PASS" ]; then
+ CNCP="${CNCP_URL:-http://localhost:3333}"
+ note="[BACKUP DAEMON HEALTH $(date +%F)] Henry mirror $worst — $(printf '%s\n' "${rows[@]}" | jq -r 'select(.verdict!="PASS")|"\(.daemon): \(.reason)"' | paste -sd'; ' -). The nightly root daemon can't write /Volumes/Henry (TCC/FDA). Fix: grant FDA to the daemon's launcher (TK-10547)."
+ curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
+ -d "$(jq -n --arg u "onprem://henry-mirror-daemon" --arg note "$note" '{url:$u,note:$note}')" >/dev/null 2>&1 || true
+ if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
+ . "$HOME/.claude/skills/_shared/george-send.sh"
+ body="<div style=\"font-family:-apple-system,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ Henry backup daemon $worst</h3><div>$note</div></div>"
+ george_send steve-office "${BACKUP_HEALTH_TO:-steve@designerwallcoverings.com}" "⚠ Henry backup daemon $worst — $(date +%F)" "$body" >/dev/null 2>&1 || true
+ fi
+fi
+
+[ "$worst" = "FAIL" ] && exit 1 || exit 0
← b7fbe23 Add read-only daemon-health probe: surfaces root-daemon last
·
back to Nas Setup
·
pull-dw-dump: write per-DB heartbeat latest-<db>.json so dw_ b1e42e2 →