watchdog cycle1 (Cody-gated): healed=verified-after-reprobe (no lying heartbeat) + fabric detection hardened vs VPN/Colima 10.0.0.x false-positive

6c3c5b962b29841360e3ca5e3512b5b63efc79c2 · 2026-08-17 22:22:54 -0700 · steve
Files touched

M data/latest.json
M watchdog.sh
Diff

commit 6c3c5b962b29841360e3ca5e3512b5b63efc79c2
Author: steve <steve@designerwallcoverings.com>
Date:   Mon Aug 17 22:22:54 2026 -0700

    watchdog cycle1 (Cody-gated): healed=verified-after-reprobe (no lying heartbeat) + fabric detection hardened vs VPN/Colima 10.0.0.x false-positive
---
 data/latest.json |  2 +-
 watchdog.sh      | 37 ++++++++++++++++++++++++++++---------
 2 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/data/latest.json b/data/latest.json
index 65dc94b..149db2b 100644
--- a/data/latest.json
+++ b/data/latest.json
@@ -1 +1 @@
-{"generated_at":"2026-08-18T05:16:02Z","verdict":"PASS","status":"PASS","serving":3,"expected":3,"fabric":"LAN","self_up":1,"down":"","healed":"","headline":"PASS: exo cluster 3/3 serving | fabric: LAN"}
\ No newline at end of file
+{"generated_at":"2026-08-18T05:22:22Z","verdict":"PASS","status":"PASS","serving":3,"expected":3,"fabric":"LAN","self_up":1,"down":"","healed":"","headline":"PASS: exo cluster 3/3 serving | fabric: LAN"}
\ No newline at end of file
diff --git a/watchdog.sh b/watchdog.sh
index eb3c179..290902c 100755
--- a/watchdog.sh
+++ b/watchdog.sh
@@ -29,13 +29,19 @@ now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
 exo_up() { curl -s --max-time 4 "http://$1:$PORT/state" >/dev/null 2>&1; }
 pings()  { ping -c1 -t2 "$1" >/dev/null 2>&1; }
 
-# Fabric detection (LOCAL queries — reliable even if this shell's net egress is sandboxed):
-# TB  = this box has a 10.0.0.x IP AND the route to a TB peer resolves to a non-en0 iface.
-# LAN = still riding regular Ethernet (the pre-upgrade state).
+# Fabric detection (LOCAL queries — reliable even if this shell's net egress is sandboxed).
+# TB only if a 10.0.0.x address lives on the Thunderbolt Bridge (bridge0) or a Thunderbolt
+# hardware en* port — NOT a VPN utun / Colima / Docker bridge that also uses 10.0.0.x.
 detect_fabric() {
-  ifconfig 2>/dev/null | grep -q "inet 10\.0\.0\." || { echo LAN; return; }
-  local ifc; ifc=$(route get 10.0.0.1 2>/dev/null | awk '/interface:/{print $2}')
-  case "$ifc" in en0|"") echo LAN;; *) echo TB;; esac
+  local ifc
+  ifc=$(ifconfig 2>/dev/null | awk '/^[a-z0-9]+:/{i=$1; sub(":","",i)} /inet 10\.0\.0\./{print i; exit}')
+  [ -z "$ifc" ] && { echo LAN; return; }
+  case "$ifc" in
+    bridge0) echo TB ;;                                   # macOS Thunderbolt Bridge
+    en*) networksetup -listallhardwareports 2>/dev/null \
+           | grep -B1 "Device: $ifc\$" | grep -qi thunderbolt && echo TB || echo LAN ;;
+    *) echo LAN ;;                                        # utun/vmenet/bridge100/etc = not TB
+  esac
 }
 
 # Remote self-contained heal — HONEST exit codes (only claims success if it really acted):
@@ -49,7 +55,7 @@ REMOTE_HEAL='curl -s --max-time 3 http://127.0.0.1:52415/state >/dev/null 2>&1 &
     cd "$HOME/exo" && nohup "$HOME/.local/bin/uv" run --extra mlx exo >/tmp/exo-watchdog.out 2>&1 & exit 8; \
   fi; exit 9'
 
-serving=0; down=(); healed=(); unreachable=(); notes=()
+serving=0; down=(); healed=(); heal_candidates=(); unreachable=(); notes=()
 
 for row in "${NODES[@]}"; do
   IFS='|' read -r name host tgt <<<"$row"
@@ -59,14 +65,14 @@ for row in "${NODES[@]}"; do
   # exo is DOWN on this node -> try to heal
   if [ "$tgt" = "LOCAL" ]; then
     launchctl kickstart -k "gui/$(id -u)/com.steve.exo-keepalive" 2>/dev/null
-    healed+=("$name(local-kick)"); notes+=("$name exo was down -> local exo-keepalive kicked")
+    heal_candidates+=("$name|$host|local-kick"); notes+=("$name exo was down -> local exo-keepalive kicked (pending re-probe)")
   elif [ "$tgt" = "NOAUTH" ]; then
     down+=("$name"); notes+=("$name exo down — ssh heal DISABLED (key not authorized yet); probe+alert only, NOT touching this box")
   elif pings "$host"; then
     # host up; try ssh heal (BatchMode so it fails fast if key not authorized)
     rc=$(ssh -o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new "$tgt" "$REMOTE_HEAL" >/dev/null 2>&1; echo $?)
     if [ "$rc" = "7" ] || [ "$rc" = "8" ] || [ "$rc" = "0" ]; then
-      healed+=("$name(ssh-heal rc=$rc)"); notes+=("$name exo down -> ssh heal rc=$rc")
+      heal_candidates+=("$name|$host|ssh-rc$rc"); notes+=("$name exo down -> ssh heal rc=$rc (pending re-probe)")
     elif [ "$rc" = "9" ]; then
       down+=("$name"); notes+=("$name exo down + reachable, but NO keepalive and this ssh-user has no ~/exo (exo runs under another account) — run harden-peer.sh as the exo-owning user; NOT faking a heal")
     else
@@ -82,6 +88,19 @@ sleep 6
 serving=0
 for row in "${NODES[@]}"; do IFS='|' read -r name host tgt <<<"$row"; exo_up "$host" && serving=$((serving+1)); done
 
+# Resolve heal candidates by a FRESH probe: 'healed' means VERIFIED up, not merely attempted.
+# A candidate that still isn't serving becomes 'down' — so a WARN can never ship with a
+# populated 'healed' for a node that's actually still down (Cody gate, cycle 1).
+for cand in "${heal_candidates[@]:-}"; do
+  [ -z "$cand" ] && continue
+  IFS='|' read -r cname chost cvia <<<"$cand"
+  if exo_up "$chost"; then
+    healed+=("$cname($cvia)")
+  else
+    down+=("$cname"); notes+=("$cname heal attempted ($cvia) but still DOWN on re-probe — NOT counting as healed")
+  fi
+done
+
 # Verdict (fleet vocab)
 self_up=$(exo_up 127.0.0.1 && echo 1 || echo 0)
 if [ "$serving" -ge "$EXPECTED" ]; then
← 5f15512 watchdog cycle1: honest heal exit codes (no false-positive h · back to Exo Cluster Watchdog · auto-data-snapshot: 2026-08-17T22:33:50 (1 data files) — dat 0df1193 →