exo-cluster-watchdog: auto-restart crashed exo across the ring, alert on unrecoverable node

aeb1f7e9447bb86a2cbfe840531e840fc0816328 · 2026-08-17 11:16:36 -0700 · steve
Files touched

A .gitignore
A data/latest.json
A watchdog.sh
Diff

commit aeb1f7e9447bb86a2cbfe840531e840fc0816328
Author: steve <steve@designerwallcoverings.com>
Date:   Mon Aug 17 11:16:36 2026 -0700

    exo-cluster-watchdog: auto-restart crashed exo across the ring, alert on unrecoverable node
---
 .gitignore       |   3 ++
 data/latest.json |   1 +
 watchdog.sh      | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 108 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2239462
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+data/*.log
+data/.prev_verdict
+.DS_Store
diff --git a/data/latest.json b/data/latest.json
new file mode 100644
index 0000000..dac01fb
--- /dev/null
+++ b/data/latest.json
@@ -0,0 +1 @@
+{"generated_at":"2026-08-17T18:16:04Z","verdict":"WARN","status":"WARN","serving":2,"expected":3,"self_up":1,"down":"mac2","healed":"","headline":"WARN: exo cluster 2/3 serving | down: mac2"}
\ No newline at end of file
diff --git a/watchdog.sh b/watchdog.sh
new file mode 100755
index 0000000..c9e1b8b
--- /dev/null
+++ b/watchdog.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+# exo-cluster-watchdog — "NEVER allow exo to just be off."
+# Runs every 2 min (launchd com.steve.exo-cluster-watchdog).
+# For each node: probe exo :52415. If DOWN and reachable, RESTART it.
+# Local node -> kickstart its own exo-keepalive. Peers -> ssh in and heal.
+# Anything unreachable/unrecoverable -> loud alert + FAIL/WARN heartbeat.
+#
+# Heartbeat verdict uses the fleet-health vocabulary (PASS/WARN/FAIL) so
+# fleet-health-rollup reads it correctly (per TK-10546 rule).
+set -uo pipefail
+
+DIR="$HOME/Projects/exo-cluster-watchdog"
+DATA="$DIR/data"
+LATEST="$DATA/latest.json"
+ALERTS="$DATA/alerts.log"
+PORT=52415
+mkdir -p "$DATA"
+
+# Node table:  name | host | ssh_target(or LOCAL)
+NODES=(
+  "macstudio3|127.0.0.1|LOCAL"
+  "mac1|192.168.1.133|steve@192.168.1.133"
+  "mac2|192.168.1.54|steve@192.168.1.54"
+)
+EXPECTED=${#NODES[@]}
+
+now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
+exo_up() { curl -s --max-time 4 "http://$1:$PORT/state" >/dev/null 2>&1; }
+pings()  { ping -c1 -t2 "$1" >/dev/null 2>&1; }
+
+# Remote self-contained heal: only acts if exo is actually down on that host.
+REMOTE_HEAL='curl -s --max-time 3 http://127.0.0.1:52415/state >/dev/null 2>&1 && exit 0; \
+  launchctl kickstart -k gui/$(id -u)/com.steve.exo-keepalive 2>/dev/null && exit 7; \
+  cd ~/exo 2>/dev/null && nohup ~/.local/bin/uv run --extra mlx exo >/tmp/exo-watchdog.out 2>&1 & exit 8'
+
+serving=0; down=(); healed=(); unreachable=(); notes=()
+
+for row in "${NODES[@]}"; do
+  IFS='|' read -r name host tgt <<<"$row"
+  if exo_up "$host"; then
+    serving=$((serving+1)); continue
+  fi
+  # exo is DOWN on this node -> try to heal
+  if [ "$tgt" = "LOCAL" ]; then
+    launchctl kickstart -k "gui/$(id -u)/com.steve.exo-keepalive" 2>/dev/null
+    healed+=("$name(local-kick)"); notes+=("$name exo was down -> local exo-keepalive kicked")
+  elif pings "$host"; then
+    # host up; try ssh heal (BatchMode so it fails fast if key not authorized)
+    rc=$(ssh -o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new "$tgt" "$REMOTE_HEAL" >/dev/null 2>&1; echo $?)
+    if [ "$rc" = "7" ] || [ "$rc" = "8" ] || [ "$rc" = "0" ]; then
+      healed+=("$name(ssh-heal rc=$rc)"); notes+=("$name exo down -> ssh heal rc=$rc")
+    else
+      down+=("$name"); notes+=("$name UP on LAN but exo down AND ssh heal failed (rc=$rc, no key/authz) — CANNOT self-heal, needs Steve")
+    fi
+  else
+    unreachable+=("$name"); down+=("$name"); notes+=("$name UNREACHABLE (machine off/asleep) — cannot power on remotely")
+  fi
+done
+
+# Re-probe after heal attempts (give exo a moment to bind) to recount serving
+sleep 6
+serving=0
+for row in "${NODES[@]}"; do IFS='|' read -r name host tgt <<<"$row"; exo_up "$host" && serving=$((serving+1)); done
+
+# Verdict (fleet vocab)
+self_up=$(exo_up 127.0.0.1 && echo 1 || echo 0)
+if [ "$serving" -ge "$EXPECTED" ]; then
+  verdict="PASS"
+elif [ "$self_up" = "0" ]; then
+  verdict="FAIL"     # this node's exo is down and didn't recover
+else
+  verdict="WARN"     # serving < expected but self is up (peer(s) degraded)
+fi
+
+downstr=$(IFS=,; echo "${down[*]:-}")
+healstr=$(IFS=,; echo "${healed[*]:-}")
+head="exo cluster $serving/$EXPECTED serving"
+[ -n "$downstr" ] && head="$head | down: $downstr"
+[ -n "$healstr" ] && head="$head | healed: $healstr"
+
+# Write heartbeat
+{
+  printf '{'
+  printf '"generated_at":"%s",' "$(now)"
+  printf '"verdict":"%s",' "$verdict"
+  printf '"status":"%s",' "$verdict"
+  printf '"serving":%d,"expected":%d,' "$serving" "$EXPECTED"
+  printf '"self_up":%s,' "$self_up"
+  printf '"down":"%s","healed":"%s",' "$downstr" "$healstr"
+  printf '"headline":"%s: %s"' "$verdict" "$head"
+  printf '}'
+} > "$LATEST"
+
+# Alert on any non-PASS (append + macOS notification). De-nag: only if state changed.
+PREVF="$DATA/.prev_verdict"
+prev=$(cat "$PREVF" 2>/dev/null || echo "")
+echo "$verdict" > "$PREVF"
+if [ "$verdict" != "PASS" ]; then
+  echo "$(now) $verdict :: $head :: ${notes[*]}" >> "$ALERTS"
+  if [ "$verdict" != "$prev" ]; then
+    osascript -e "display notification \"$head\" with title \"exo watchdog: $verdict\"" >/dev/null 2>&1 || true
+  fi
+fi
+echo "$verdict :: $head"
(oldest) · back to Exo Cluster Watchdog · add harden-peer.sh: per-node exo-keepalive + pmset autoresta 5081ca1 →