watchdog cycle1: honest heal exit codes (no false-positive heal on cross-user nodes) + TB-vs-LAN fabric awareness

5f15512d231c514d6250ed1c07586981f0136cfa · 2026-08-17 22:16:51 -0700 · steve
Files touched

M data/latest.json
M watchdog.sh
Diff

commit 5f15512d231c514d6250ed1c07586981f0136cfa
Author: steve <steve@designerwallcoverings.com>
Date:   Mon Aug 17 22:16:51 2026 -0700

    watchdog cycle1: honest heal exit codes (no false-positive heal on cross-user nodes) + TB-vs-LAN fabric awareness
---
 data/latest.json |  2 +-
 watchdog.sh      | 25 ++++++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/data/latest.json b/data/latest.json
index ef315b4..65dc94b 100644
--- a/data/latest.json
+++ b/data/latest.json
@@ -1 +1 @@
-{"generated_at":"2026-08-18T05:03:21Z","verdict":"PASS","status":"PASS","serving":3,"expected":3,"fabric":"LAN","self_up":1,"down":"","healed":"","headline":"PASS: exo cluster 3/3 serving | fabric: LAN"}
\ No newline at end of file
+{"generated_at":"2026-08-18T05:16:02Z","verdict":"PASS","status":"PASS","serving":3,"expected":3,"fabric":"LAN","self_up":1,"down":"","healed":"","headline":"PASS: exo cluster 3/3 serving | fabric: LAN"}
\ No newline at end of file
diff --git a/watchdog.sh b/watchdog.sh
index b966f33..eb3c179 100755
--- a/watchdog.sh
+++ b/watchdog.sh
@@ -29,10 +29,25 @@ now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
 exo_up() { curl -s --max-time 4 "http://$1:$PORT/state" >/dev/null 2>&1; }
 pings()  { ping -c1 -t2 "$1" >/dev/null 2>&1; }
 
-# Remote self-contained heal: only acts if exo is actually down on that host.
+# Fabric detection (LOCAL queries — reliable even if this shell's net egress is sandboxed):
+# TB  = this box has a 10.0.0.x IP AND the route to a TB peer resolves to a non-en0 iface.
+# LAN = still riding regular Ethernet (the pre-upgrade state).
+detect_fabric() {
+  ifconfig 2>/dev/null | grep -q "inet 10\.0\.0\." || { echo LAN; return; }
+  local ifc; ifc=$(route get 10.0.0.1 2>/dev/null | awk '/interface:/{print $2}')
+  case "$ifc" in en0|"") echo LAN;; *) echo TB;; esac
+}
+
+# Remote self-contained heal — HONEST exit codes (only claims success if it really acted):
+#   0 = exo already up locally on the peer   7 = kicked its exo-keepalive launchd label
+#   8 = started exo directly (keepalive absent, but ~/exo+uv exist for THIS ssh user)
+#   9 = CANNOT heal: no keepalive AND this ssh user has no ~/exo (exo likely runs under a
+#       different account) -> needs harden-peer.sh under the exo-owning user; do NOT fake it.
 REMOTE_HEAL='curl -s --max-time 3 http://127.0.0.1:52415/state >/dev/null 2>&1 && exit 0; \
   launchctl kickstart -k gui/$(id -u)/com.steve.exo-keepalive 2>/dev/null && exit 7; \
-  cd ~/exo 2>/dev/null && nohup ~/.local/bin/uv run --extra mlx exo >/tmp/exo-watchdog.out 2>&1 & exit 8'
+  if [ -x "$HOME/.local/bin/uv" ] && [ -d "$HOME/exo" ]; then \
+    cd "$HOME/exo" && nohup "$HOME/.local/bin/uv" run --extra mlx exo >/tmp/exo-watchdog.out 2>&1 & exit 8; \
+  fi; exit 9'
 
 serving=0; down=(); healed=(); unreachable=(); notes=()
 
@@ -52,6 +67,8 @@ for row in "${NODES[@]}"; do
     rc=$(ssh -o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new "$tgt" "$REMOTE_HEAL" >/dev/null 2>&1; echo $?)
     if [ "$rc" = "7" ] || [ "$rc" = "8" ] || [ "$rc" = "0" ]; then
       healed+=("$name(ssh-heal rc=$rc)"); notes+=("$name exo down -> ssh heal rc=$rc")
+    elif [ "$rc" = "9" ]; then
+      down+=("$name"); notes+=("$name exo down + reachable, but NO keepalive and this ssh-user has no ~/exo (exo runs under another account) — run harden-peer.sh as the exo-owning user; NOT faking a heal")
     else
       down+=("$name"); notes+=("$name UP on LAN but exo down AND ssh heal failed (rc=$rc, no key/authz) — CANNOT self-heal, needs Steve")
     fi
@@ -75,9 +92,10 @@ else
   verdict="WARN"     # serving < expected but self is up (peer(s) degraded)
 fi
 
+fabric=$(detect_fabric)
 downstr=$(IFS=,; echo "${down[*]:-}")
 healstr=$(IFS=,; echo "${healed[*]:-}")
-head="exo cluster $serving/$EXPECTED serving"
+head="exo cluster $serving/$EXPECTED serving | fabric: $fabric"
 [ -n "$downstr" ] && head="$head | down: $downstr"
 [ -n "$healstr" ] && head="$head | healed: $healstr"
 
@@ -88,6 +106,7 @@ head="exo cluster $serving/$EXPECTED serving"
   printf '"verdict":"%s",' "$verdict"
   printf '"status":"%s",' "$verdict"
   printf '"serving":%d,"expected":%d,' "$serving" "$EXPECTED"
+  printf '"fabric":"%s",' "$fabric"
   printf '"self_up":%s,' "$self_up"
   printf '"down":"%s","healed":"%s",' "$downstr" "$healstr"
   printf '"headline":"%s: %s"' "$verdict" "$head"
← bf037fb auto-data-snapshot: 2026-08-17T22:03:36 (1 data files) — dat · back to Exo Cluster Watchdog · watchdog cycle1 (Cody-gated): healed=verified-after-reprobe 6c3c5b9 →