add read-only health-scan.mjs (full per-process health table) + document run-health tier

6414a53a7388d40c4599ac0a53200956c9577bb4 · 2026-06-25 15:47:59 -0700 · Steve Abrams
Files touched

M README.md
A health-scan.mjs
Diff

commit 6414a53a7388d40c4599ac0a53200956c9577bb4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jun 25 15:47:59 2026 -0700

    add read-only health-scan.mjs (full per-process health table) + document run-health tier
---
 README.md       | 18 +++++++++++++
 health-scan.mjs | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)

diff --git a/README.md b/README.md
index ea161d3..9d42482 100644
--- a/README.md
+++ b/README.md
@@ -23,11 +23,29 @@ keeps fresh. Never edit the other repo's files or its plist.
 - **Q4=B restricted restart** — DEFAULT-DENY allowlist (`safe-restart-allowlist.json`);
   customer-facing / mid-write / Kamatera services are GATED (memo, never auto-bounced)
 
+## Run-health tier (the claude-mail crash-loop fix, 2026-06-25)
+
+`com.steve.claude-mail` crash-looped for HOURS unnoticed: rejected IMAP credential
+→ launchd `loaded` between fires but exit-1 every fire. Every liveness/online check
+called it healthy. Now closed:
+
+- **Generator** (`_shared/fleet-registry`) records `last_exit` (decoded `launchctl
+  list <label>` LastExitStatus) + `log_health` (FATAL log-tail scan) per launchd job.
+- **`fleet-act.mjs`** adds an **UNHEALTHY** tier: active + expected_up + looks-live
+  (loaded/online) BUT crash-looping → **gated memo, NEVER auto-restarted** (a rejected
+  credential / EADDRINUSE / missing module won't fix on a bounce — a kick just hides
+  the loop). Benign SIGTERM-on-a-running-daemon false positives are filtered.
+- **Embedded self-heal complement** for critical jobs: `~/Projects/claude-mail/health.js`
+  pages Steve via CNCP :3333 + macOS osascript + George :9850 after 3 consecutive fails.
+
 ## Run
 
 ```sh
 node fleet-act.mjs            # dry-run — prints findings, touches nothing
 node fleet-act.mjs --apply    # safe restarts + writes gated memos (idempotent)
+node health-scan.mjs          # READ-ONLY full health table of every process
+node health-scan.mjs --bad    # only non-HEALTHY rows
+node health-scan.mjs --json   # machine-readable
 ```
 
 Memos land in `~/.claude/yolo-queue/pending-approval/fleet-*.md` — Steve approves.
diff --git a/health-scan.mjs b/health-scan.mjs
new file mode 100644
index 0000000..e84d9ce
--- /dev/null
+++ b/health-scan.mjs
@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+// health-scan.mjs — READ-ONLY health report for EVERY process in the fleet manifest.
+// Reports per-process: name, kind, expected_up, live_status, run-health (last_exit /
+// log_health), and a verdict. Takes NO heal action — pure visibility. This is the
+// "debug every process" view Steve asked for: the same manifest the steward consumes,
+// rendered as a health table that surfaces the loaded-but-crash-looping class
+// (the com.steve.claude-mail failure mode) that liveness/online checks miss.
+//
+// Usage:  node health-scan.mjs            # all mac2 + kamatera, grouped by verdict
+//         node health-scan.mjs --bad      # only non-HEALTHY rows
+//         node health-scan.mjs --json      # machine-readable
+// Never restarts/stops/deletes anything. Exit 0 always.
+
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const MANIFEST = process.env.FLEET_MANIFEST || join(homedir(), "Projects/_shared/fleet-registry/manifest.json");
+const ONLY_BAD = process.argv.includes("--bad");
+const AS_JSON = process.argv.includes("--json");
+
+const manifest = JSON.parse(readFileSync(MANIFEST, "utf8"));
+const REAL = new Set(["mac2", "kamatera", "mac1"]);
+const entries = (manifest.entries || []).filter(e => e && REAL.has(e.machine) && e.kind && !String(e.id || "").startsWith("_"));
+
+const HEALTHY_SCHED = new Set(["loaded", "running", "online"]);
+const isScheduled = e => !!e.schedule || e.kind === "launchd" || e.kind === "plist-only";
+const looksLive = e => isScheduled(e) ? HEALTHY_SCHED.has(e.live_status) : e.live_status === "online";
+
+// classify each entry into a verdict (mirrors fleet-act.mjs but report-only)
+function verdict(e) {
+  if (e.lifecycle_status === "retired") {
+    return e.live_status === "online"
+      ? { v: "ZOMBIE", why: "retired but still online" }
+      : { v: "RETIRED", why: "retired, not running (steady state)" };
+  }
+  if (e.lifecycle_status === "deprecated") return { v: "DEPRECATED", why: "winding down" };
+  if (e.expected_up !== true) return { v: "OFF-OK", why: "not expected up, not running" };
+
+  // expected_up === true && active:
+  if (!looksLive(e)) return { v: "DARK", why: `expected up but live_status=${e.live_status}` };
+
+  // looks-live — check run-health (launchd crash-loop class)
+  if (e.kind === "launchd" || e.kind === "plist-only") {
+    const exit = e.last_exit, raw = e.last_exit_raw;
+    const running = e.live_status === "running", loaded = e.live_status === "loaded";
+    const signalled = raw !== null && raw !== undefined && raw < 0;
+    if (loaded) {
+      if (exit !== null && exit !== undefined && exit !== 0)
+        return { v: "UNHEALTHY", why: `loaded but last_exit=${exit} (crash-loop)` };
+      if ((exit === null || exit === undefined) && e.log_health === "fatal")
+        return { v: "UNHEALTHY", why: "loaded, FATAL log tail (no exit to clear it)" };
+    } else if (running && !signalled && exit > 0 && e.log_health === "fatal") {
+      return { v: "UNHEALTHY", why: `running but flapping (last_exit=${exit} + FATAL log)` };
+    }
+  }
+  return { v: "HEALTHY", why: e.health_url ? `live + health_url ${e.health_url}` : "live" };
+}
+
+const rows = entries.map(e => ({ ...verdict(e), id: e.id, name: e.name, machine: e.machine, kind: e.kind,
+  expected_up: e.expected_up, live_status: e.live_status, last_exit: e.last_exit, log_health: e.log_health, owner: e.owner }));
+
+const ORDER = ["UNHEALTHY", "ZOMBIE", "DARK", "DEPRECATED", "HEALTHY", "OFF-OK", "RETIRED"];
+const ICON = { UNHEALTHY: "🟠", ZOMBIE: "🧟", DARK: "🔴", DEPRECATED: "🟡", HEALTHY: "🟢", "OFF-OK": "⚪", RETIRED: "⚫" };
+const counts = rows.reduce((a, r) => (a[r.v] = (a[r.v] || 0) + 1, a), {});
+
+if (AS_JSON) {
+  console.log(JSON.stringify({ generated_at: manifest.generated_at, total: rows.length, counts, rows }, null, 2));
+} else {
+  console.log(`fleet health-scan (READ-ONLY) — ${rows.length} processes · manifest @ ${manifest.generated_at}`);
+  console.log(ORDER.map(v => `${ICON[v]} ${v}:${counts[v] || 0}`).join("  "));
+  console.log("");
+  for (const v of ORDER) {
+    if (ONLY_BAD && ["HEALTHY", "OFF-OK", "RETIRED"].includes(v)) continue;
+    const grp = rows.filter(r => r.v === v).sort((a, b) => a.id.localeCompare(b.id));
+    if (!grp.length) continue;
+    console.log(`${ICON[v]} ${v} (${grp.length})`);
+    const show = (v === "HEALTHY" || v === "OFF-OK" || v === "RETIRED") ? grp.slice(0, 6) : grp;
+    for (const r of show) console.log(`   ${r.name.padEnd(38)} ${r.machine}/${r.kind.padEnd(11)} ${r.why}`);
+    if (show.length < grp.length) console.log(`   … and ${grp.length - show.length} more`);
+    console.log("");
+  }
+  console.log("cost: $0 (local) — read-only, no heal action taken");
+}
← e92fb43 reconciler: UNHEALTHY tier — catch loaded-but-crash-looping · back to Fleet Lifecycle Steward · (newest)