← back to Fleet Lifecycle Steward
reconciler: UNHEALTHY tier — catch loaded-but-crash-looping launchd jobs (claude-mail failure mode), GATED never auto-restart
e92fb430d803e1fd0e7a2a80d8fb7e33208e4f0e · 2026-06-25 15:45:52 -0700 · Steve Abrams
Files touched
Diff
commit e92fb430d803e1fd0e7a2a80d8fb7e33208e4f0e
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 15:45:52 2026 -0700
reconciler: UNHEALTHY tier — catch loaded-but-crash-looping launchd jobs (claude-mail failure mode), GATED never auto-restart
---
fleet-act.mjs | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 67 insertions(+), 1 deletion(-)
diff --git a/fleet-act.mjs b/fleet-act.mjs
index b272362..55d518a 100644
--- a/fleet-act.mjs
+++ b/fleet-act.mjs
@@ -40,6 +40,45 @@ function snapshotHealthy(e) {
return isScheduled(e) ? HEALTHY_SCHED.has(e.live_status) : e.live_status === "online";
}
+// --- run-health (the claude-mail crash-loop class) -------------------------
+// A launchd job can be live_status=loaded (passes snapshotHealthy above) yet
+// exit-≠0 on EVERY fire — loaded-but-broken. The generator now records last_exit
+// (decoded exit code; 0 = clean) and log_health ('ok'|'fatal'|null). A job is
+// UNHEALTHY when it looks live but its last run failed. We weight last_exit over
+// log_health: a clean current exit + a stale FATAL log tail == recovered (silent).
+function runUnhealthy(e) {
+ if (!snapshotHealthy(e)) return null; // already DARK/handled elsewhere
+ if (e.kind !== "launchd" && e.kind !== "plist-only") return null; // exit-code signal is launchd-only
+ const exit = e.last_exit;
+ const raw = e.last_exit_raw;
+ const running = e.live_status === "running"; // has a live PID right now
+ const loaded = e.live_status === "loaded"; // between fires: last exit IS its health
+ // A SIGNAL-terminated prior exit (raw < 0, e.g. SIGTERM -15 from a reload) on a
+ // job that is RUNNING NOW is a benign false positive — the daemon is up. Skip it.
+ const signalled = raw !== null && raw !== undefined && raw < 0;
+ if (running && signalled) return null;
+
+ if (loaded) {
+ // Scheduled job sitting between fires — its last_exit IS its run-health.
+ if (exit !== null && exit !== undefined && exit !== 0) {
+ return { why: `last_exit=${exit} (loaded between fires but failing every run)`, signal: "exit" };
+ }
+ if ((exit === null || exit === undefined) && e.log_health === "fatal") {
+ return { why: "log tail shows FATAL/auth/crash pattern (no exit code to clear it)", signal: "log" };
+ }
+ return null;
+ }
+ if (running) {
+ // Daemon up now, but its last recorded exit was a real non-zero CODE (not a
+ // signal) AND the log tail confirms a fatal pattern → it's flapping/restarting.
+ if (exit !== null && exit !== undefined && exit > 0 && e.log_health === "fatal") {
+ return { why: `running but flapping — last_exit=${exit} + FATAL log tail`, signal: "flap" };
+ }
+ return null;
+ }
+ return null;
+}
+
// optional real-liveness poll for daemons that CLAIM online but expose a health_url
async function trulyAlive(e) {
if (!snapshotHealthy(e)) return false;
@@ -83,7 +122,7 @@ function draftMemo(kind, e, body) {
return { file, written: APPLY };
}
-const out = { dark_safe_restarted: [], dark_gated: [], zombies: [], stale: [], errors: [] };
+const out = { dark_safe_restarted: [], dark_gated: [], unhealthy: [], zombies: [], stale: [], errors: [] };
for (const e of entries) {
// ZOMBIE: retired but still online
@@ -92,6 +131,18 @@ for (const e of entries) {
out.zombies.push({ id: e.id, memo: m.file || "(pending)", skipped: m.skipped });
continue;
}
+ // UNHEALTHY: expected_up + looks-live (loaded) but crash-looping (last_exit≠0 / FATAL log).
+ // This is the claude-mail blind spot — it passed snapshotHealthy so DARK never fired.
+ // NEVER auto-restarted: a loaded-but-failing job (rejected credential, EADDRINUSE,
+ // missing module) won't fix on a bounce — bouncing just hides the crash-loop. GATED only.
+ if (e.expected_up === true && e.lifecycle_status === "active") {
+ const ru = runUnhealthy(e);
+ if (ru) {
+ const m = draftMemo("unhealthy", e, unhealthyMemo(e, ru));
+ out.unhealthy.push({ id: e.id, owner: e.owner, why: ru.why, memo: m.file || "(pending)", skipped: m.skipped });
+ continue; // don't ALSO dark-restart it — it's loaded, just broken
+ }
+ }
// STALE: deprecated past retire_after, OR expected_up never-seen in STALE_DAYS (and not schedule-healthy)
const pastRetire = e.lifecycle_status === "deprecated" && e.retire_after && daysSince(e.retire_after) > 0;
const ghosted = e.expected_up === true && !snapshotHealthy(e) && daysSince(e.last_seen) > STALE_DAYS;
@@ -131,6 +182,18 @@ function darkMemo(e) {
`**To restart (after confirming it's idle / no mid-write):**\n\`\`\`sh\n${restartCmd(e) || "# (no standard restart for this kind — investigate)"}\n\`\`\`\n` +
`**To abort:** delete this memo.\n\n_Drafted by fleet-reconcile.mjs — Fleet Lifecycle Officer._\n`;
}
+function unhealthyMemo(e, ru) {
+ const log = `~/Library/Logs/${e.name}.log`; // best-effort hint; real path is in the plist
+ return `# UNHEALTHY — \`${e.id}\` is loaded but crash-looping (GATED, NOT auto-restarted)\n\n${head(e)}` +
+ `- **Run-health:** ${ru.why}\n\n` +
+ `**Why this matters:** the job is \`live_status=loaded\` so every liveness/online check calls it healthy — but it FAILS on every fire. This is the exact failure mode that let \`com.steve.claude-mail\` crash-loop for hours invisibly (rejected IMAP credential, exit-1 every 10s, nothing noticed).\n\n` +
+ `**Why NOT auto-restarted:** a loaded-but-failing job won't fix on a bounce — a rejected credential / EADDRINUSE / missing module survives a restart. Bouncing it just HIDES the crash-loop. This needs a real fix, not a kick.\n\n` +
+ `**To diagnose:**\n\`\`\`sh\nlaunchctl list ${e.name} | grep LastExitStatus # confirm non-zero\ntail -50 ${log} 2>/dev/null # real log path is in ~/Library/LaunchAgents/${e.name}.plist\n\`\`\`\n` +
+ `**If it's a credential/config issue:** fix the secret/config, then \`launchctl kickstart -k gui/$(id -u)/${e.name}\`.\n` +
+ `**If the job is obsolete:** set \`lifecycle_status: retired\` for \`${e.id}\` in overlay.json and bootout the plist.\n` +
+ `**Consider** an embedded self-heal layer (see \`~/Projects/claude-mail/health.js\`) for any critical job so it pages over mailbox-independent channels after N consecutive fails.\n\n` +
+ `**To abort:** delete this memo.\n\n_Drafted by fleet-act.mjs — Fleet Lifecycle Officer (run-health tier)._\n`;
+}
function zombieMemo(e) {
return `# ZOMBIE ALERT — \`${e.id}\` is RETIRED but still online (decommission target)\n\n${head(e)}\n` +
`**Meaning:** Steve decided to retire this, but it's still running — the decommission didn't finish, or something resurrected it.\n\n` +
@@ -151,6 +214,7 @@ const summary = {
ts: new Date().toISOString(), mode: APPLY ? "apply" : "dry-run",
entries: entries.length,
dark_safe: out.dark_safe_restarted.length, dark_gated: out.dark_gated.length,
+ unhealthy: out.unhealthy.length,
zombies: out.zombies.length, stale: out.stale.length, errors: out.errors.length,
};
try { appendFileSync(RUNLOG, JSON.stringify(summary) + "\n"); } catch {}
@@ -159,6 +223,8 @@ console.log(` 🟢 safe auto-restart: ${out.dark_safe_restarted.length}`);
out.dark_safe_restarted.forEach(d => console.log(` ${d.id} ${d.cmd}`));
console.log(` 🔴 DARK gated (drafted/pending): ${out.dark_gated.length}`);
out.dark_gated.slice(0, 10).forEach(d => console.log(` ${d.id} owner=${d.owner || "?"} ${d.skipped ? "(memo already pending)" : ""}`));
+console.log(` 🟠 UNHEALTHY crash-loop (loaded-but-failing, GATED): ${out.unhealthy.length}`);
+out.unhealthy.slice(0, 15).forEach(d => console.log(` ${d.id} ${d.why} ${d.skipped ? "(memo already pending)" : ""}`));
console.log(` 🧟 ZOMBIE: ${out.zombies.length}`);
out.zombies.slice(0, 10).forEach(d => console.log(` ${d.id} ${d.skipped ? "(pending)" : ""}`));
console.log(` 🟡 retirement candidates: ${out.stale.length}`);
← 0341b79 Fleet Lifecycle action tier — schedule-aware safe restart +
·
back to Fleet Lifecycle Steward
·
add read-only health-scan.mjs (full per-process health table 6414a53 →