watchdog: model-instance keepalive — re-POST wanted-instance.json when ring has 0 instances; instances=0/unmeasured never PASS; --test seam + WD_TEST_STATE (TK-11903)

089ac8c552c7a45343748fe5a618b8b170eb4d3c · 2026-09-18 13:20:25 -0700 · Steve Abrams

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YT3be6iadPzjKJDiaTasEE

Files touched

A data/wanted-instance.json
M watchdog.sh

Diff

commit 089ac8c552c7a45343748fe5a618b8b170eb4d3c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Fri Sep 18 13:20:25 2026 -0700

    watchdog: model-instance keepalive — re-POST wanted-instance.json when ring has 0 instances; instances=0/unmeasured never PASS; --test seam + WD_TEST_STATE (TK-11903)
    
    Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
    Claude-Session: https://claude.ai/code/session_01YT3be6iadPzjKJDiaTasEE
---
 data/wanted-instance.json |  1 +
 watchdog.sh               | 63 ++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/data/wanted-instance.json b/data/wanted-instance.json
new file mode 100644
index 0000000..c9db193
--- /dev/null
+++ b/data/wanted-instance.json
@@ -0,0 +1 @@
+{"MlxRingInstance":{"instanceId":"a7abbac7-5332-4cb0-b37f-97f9d790374a","shardAssignments":{"modelId":"mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit","runnerToShard":{"53d26e99-7ac0-451c-b09a-b3eabc13d20d":{"PipelineShardMetadata":{"modelCard":{"modelId":"mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit","storageSize":{"inBytes":17180913664},"nLayers":48,"hiddenSize":2048,"supportsTensor":true,"numKeyValueHeads":4,"tasks":["TextGeneration"],"components":null,"family":"","quantization":"","baseModel":"","capabilities":[],"backends":["MlxMetal","MlxCpu","MlxCuda","Vllm"],"reasoningDialect":"none","contextLength":262144,"usesCfg":false,"trustRemoteCode":false,"isCustom":true,"vision":null,"samplingDefaults":{"temperature":null,"topP":null,"topK":null,"minP":null,"repetitionPenalty":null,"presencePenalty":null,"frequencyPenalty":null,"thinking":null,"nonThinking":null}},"deviceRank":0,"worldSize":1,"immediateException":false,"shouldTimeout":null,"startLayer":0,"endLayer":48,"nLayers":48}}},"nodeToRunner":{"247012f1ea60ac692a1c2f0d96b31370":"53d26e99-7ac0-451c-b09a-b3eabc13d20d"}},"hostsByNode":{"247012f1ea60ac692a1c2f0d96b31370":[{"ip":"0.0.0.0","port":63939}]},"ephemeralPort":63939}}
diff --git a/watchdog.sh b/watchdog.sh
index b0f9e4f..fe032fe 100755
--- a/watchdog.sh
+++ b/watchdog.sh
@@ -101,10 +101,66 @@ for cand in "${heal_candidates[@]:-}"; do
   fi
 done
 
+# ---- Model-instance keepalive (TK-11903, 2026-09-18) ----
+# exo "up" is not the same as "a model is resident". The ring evicts instances under memory
+# pressure (two 32GB nodes sit at 1-6GB free); openclaw night jobs then fail every call.
+# If the local ring reports 0 instances and data/wanted-instance.json exists, re-POST it.
+# Rate-limited (one attempt per 5 min: a fresh load takes ~130s to first token) and NEVER
+# counted as healed on the same pass — verdict stays WARN until a later pass sees it resident.
+# Test seam: `watchdog.sh --test` reads state from $WD_TEST_STATE and only PRINTS the POST.
+TEST_MODE=0; [ "${1:-}" = "--test" ] && TEST_MODE=1
+WANTED="$DATA/wanted-instance.json"; RELAUNCH_STAMP="$DATA/.instance-relaunch-ts"
+instances=-1
+if [ "$TEST_MODE" = 1 ]; then
+  state_json=$(cat "${WD_TEST_STATE:-/dev/null}" 2>/dev/null)
+else
+  state_json=$(curl -s --max-time 4 "http://127.0.0.1:$PORT/state" 2>/dev/null)
+fi
+if [ -n "$state_json" ]; then
+  instances=$(printf '%s' "$state_json" | python3 -c 'import json,sys
+try: print(len(json.load(sys.stdin).get("instances") or {}))
+except Exception: print(-1)' 2>/dev/null || echo -1)
+fi
+instance_note=""
+if [ "$instances" = "0" ] && [ -s "$WANTED" ]; then
+  last=$(cat "$RELAUNCH_STAMP" 2>/dev/null || echo 0); nowe=$(date +%s)
+  if [ $((nowe - last)) -ge 300 ]; then
+    echo "$nowe" > "$RELAUNCH_STAMP"
+    wanted_model=$(python3 -c 'import json,sys
+def walk(o):
+    if isinstance(o,dict):
+        if "modelId" in o: return o["modelId"]
+        for v in o.values():
+            r=walk(v)
+            if r: return r
+    if isinstance(o,list):
+        for v in o:
+            r=walk(v)
+            if r: return r
+print(walk(json.load(open(sys.argv[1]))) or "?")' "$WANTED" 2>/dev/null)
+    if [ "$TEST_MODE" = 1 ]; then
+      instance_note="0 instances on ring -> WOULD re-POST $wanted_model (test mode, not sent)"
+    else
+      body=$(python3 -c 'import json,sys; print(json.dumps({"instance": json.load(open(sys.argv[1]))}))' "$WANTED")
+      rc=$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$PORT/instance" -H 'Content-Type: application/json' -d "$body")
+      instance_note="0 instances on ring -> re-POSTed $wanted_model (http $rc); pending re-probe, NOT counted as healed"
+    fi
+  else
+    instance_note="0 instances on ring; relaunch attempted $((nowe - last))s ago, waiting for load (no re-POST)"
+  fi
+elif [ "$instances" = "0" ]; then
+  instance_note="0 instances on ring and no $WANTED to relaunch — needs a human"
+elif [ "$instances" = "-1" ]; then
+  instance_note="instance count NOT MEASURED (state unreadable)"
+fi
+[ -n "$instance_note" ] && notes+=("$instance_note")
+
 # Verdict (fleet vocab)
 self_up=$(exo_up 127.0.0.1 && echo 1 || echo 0)
-if [ "$serving" -ge "$EXPECTED" ]; then
+if [ "$serving" -ge "$EXPECTED" ] && [ "$instances" -ge 1 ]; then
   verdict="PASS"
+elif [ "$serving" -ge "$EXPECTED" ]; then
+  verdict="WARN"     # nodes serving but NO model instance resident (or not measured) -> never PASS
 elif [ "$self_up" = "0" ]; then
   verdict="FAIL"     # this node's exo is down and didn't recover
 else
@@ -114,7 +170,7 @@ fi
 fabric=$(detect_fabric)
 downstr=$(IFS=,; echo "${down[*]:-}")
 healstr=$(IFS=,; echo "${healed[*]:-}")
-head="exo cluster $serving/$EXPECTED serving | fabric: $fabric"
+head="exo cluster $serving/$EXPECTED serving | instances: $instances | fabric: $fabric"
 [ -n "$downstr" ] && head="$head | down: $downstr"
 [ -n "$healstr" ] && head="$head | healed: $healstr"
 
@@ -133,7 +189,7 @@ for nz in "${notes[@]:-}"; do [ -n "$nz" ] && printf '%s\n' "$nz" >> "$NOTES_TMP
 
 # Write heartbeat (python3 = correct JSON escaping). verdict/status stay PASS/WARN/FAIL
 # (fleet-health-rollup vocabulary); adds notes[] + last_incident for flap observability.
-VERDICT="$verdict" SERVING="$serving" EXPECTED="$EXPECTED" FABRIC="$fabric" \
+INSTANCES="$instances" VERDICT="$verdict" SERVING="$serving" EXPECTED="$EXPECTED" FABRIC="$fabric" \
 SELF_UP="$self_up" DOWNSTR="$downstr" HEALSTR="$healstr" HEAD="$verdict: $head" \
 GEN="$(now)" LAST_INCIDENT="${last_incident:-}" NOTES_FILE="$NOTES_TMP" \
 python3 - "$LATEST" <<'PY'
@@ -148,6 +204,7 @@ d = {
     "verdict": os.environ['VERDICT'], "status": os.environ['VERDICT'],
     "serving": int(os.environ['SERVING']), "expected": int(os.environ['EXPECTED']),
     "fabric": os.environ['FABRIC'], "self_up": int(os.environ['SELF_UP']),
+    "instances": int(os.environ.get('INSTANCES', '-1')),
     "down": os.environ['DOWNSTR'], "healed": os.environ['HEALSTR'],
     "notes": notes, "last_incident": os.environ.get('LAST_INCIDENT', ''),
     "headline": os.environ['HEAD'],

← da41f33 auto-data-snapshot: 2026-09-18T13:12:29 (1 data files) — dat · back to Exo Cluster Watchdog · auto-data-snapshot: 2026-09-18T13:45:34 (1 data files) — dat 8b72651 →