`fleet-act.mjs`

234 lines
#!/usr/bin/env node
// fleet-reconcile.mjs — the cheap mechanical tier of the Fleet Lifecycle Officer.
// DTD-decided 2026-06-24: scheduled launchd SCRIPT (not a daemon) that diffs the
// service registry manifest, auto-restarts ONLY the safe allowlisted set (Q4=B),
// and drafts GATED memos for everything risky/stale. Escalates judgment to the
// `fleet-lifecycle-steward` Claude subagent. Consumes the registry, never forks it.
//
// Modes:  (default) dry-run — print only, touch nothing.
//         --apply        — perform safe restarts + write gated memo drafts (idempotent).
// Exit 0 always (a scheduled job that hard-fails is itself a silent death).

import { readFileSync, writeFileSync, existsSync, mkdirSync, appendFileSync, readdirSync } from "node:fs";
import { execSync } from "node:child_process";
import { homedir } from "node:os";
import { join } from "node:path";

const HOME = homedir();
const MANIFEST = process.env.FLEET_MANIFEST || join(HOME, "Projects/_shared/fleet-registry/manifest.json");
const ALLOWLIST = join(HOME, "Projects/fleet-lifecycle-steward/safe-restart-allowlist.json");
const PENDING = join(HOME, ".claude/yolo-queue/pending-approval");
const RUNLOG = join(HOME, "Projects/fleet-lifecycle-steward/runs.jsonl");
const APPLY = process.argv.includes("--apply");
const STALE_DAYS = 14;            // Q3 = A (14d). Bump to 30 here if the queue gets noisy.

// --- load ---
const manifest = JSON.parse(readFileSync(MANIFEST, "utf8"));
const all = Array.isArray(manifest.entries) ? manifest.entries : [];
// real services only — drop the _README sentinel + anything without a real machine/kind
const REAL_MACHINES = new Set(["mac2", "kamatera", "mac1"]);
const entries = all.filter(e => e && REAL_MACHINES.has(e.machine) && e.kind && !String(e.id || "").startsWith("_"));

const allow = existsSync(ALLOWLIST) ? JSON.parse(readFileSync(ALLOWLIST, "utf8")) : { allow: [], deny_patterns: [] };
const allowSet = new Set(allow.allow || []);
const denyRe = (allow.deny_patterns || []).map(p => new RegExp(p, "i"));

// --- liveness (schedule-aware) ---
const isScheduled = e => !!e.schedule || e.kind === "launchd" || e.kind === "plist-only";
const HEALTHY_SCHED = new Set(["loaded", "running", "online"]);
function snapshotHealthy(e) {
  return isScheduled(e) ? HEALTHY_SCHED.has(e.live_status) : e.live_status === "online";
}

// --- run-health (the claude-mail crash-loop class) -------------------------
// A launchd job can be live_status=loaded (passes snapshotHealthy above) yet
// exit-≠0 on EVERY fire — loaded-but-broken. The generator now records last_exit
// (decoded exit code; 0 = clean) and log_health ('ok'|'fatal'|null). A job is
// UNHEALTHY when it looks live but its last run failed. We weight last_exit over
// log_health: a clean current exit + a stale FATAL log tail == recovered (silent).
function runUnhealthy(e) {
  if (!snapshotHealthy(e)) return null;          // already DARK/handled elsewhere
  if (e.kind !== "launchd" && e.kind !== "plist-only") return null; // exit-code signal is launchd-only
  const exit = e.last_exit;
  const raw = e.last_exit_raw;
  const running = e.live_status === "running";   // has a live PID right now
  const loaded = e.live_status === "loaded";     // between fires: last exit IS its health
  // A SIGNAL-terminated prior exit (raw < 0, e.g. SIGTERM -15 from a reload) on a
  // job that is RUNNING NOW is a benign false positive — the daemon is up. Skip it.
  const signalled = raw !== null && raw !== undefined && raw < 0;
  if (running && signalled) return null;

  if (loaded) {
    // Scheduled job sitting between fires — its last_exit IS its run-health.
    if (exit !== null && exit !== undefined && exit !== 0) {
      return { why: `last_exit=${exit} (loaded between fires but failing every run)`, signal: "exit" };
    }
    if ((exit === null || exit === undefined) && e.log_health === "fatal") {
      return { why: "log tail shows FATAL/auth/crash pattern (no exit code to clear it)", signal: "log" };
    }
    return null;
  }
  if (running) {
    // Daemon up now, but its last recorded exit was a real non-zero CODE (not a
    // signal) AND the log tail confirms a fatal pattern → it's flapping/restarting.
    if (exit !== null && exit !== undefined && exit > 0 && e.log_health === "fatal") {
      return { why: `running but flapping — last_exit=${exit} + FATAL log tail`, signal: "flap" };
    }
    return null;
  }
  return null;
}

// optional real-liveness poll for daemons that CLAIM online but expose a health_url
async function trulyAlive(e) {
  if (!snapshotHealthy(e)) return false;
  if (!e.health_url) return true;                 // worker/job: snapshot is all we have
  try {
    const ac = new AbortController();
    const t = setTimeout(() => ac.abort(), 3000);
    const r = await fetch(e.health_url, { signal: ac.signal, redirect: "manual" });
    clearTimeout(t);
    return r.status >= 200 && r.status < 400;      // 200 with a dead worker behind it = NOT alive
  } catch { return false; }
}

// --- classification ---
const daysSince = iso => iso ? (Date.now() - new Date(iso).getTime()) / 86400000 : Infinity;
function restartSafe(e) {
  if (e.machine !== "mac2") return false;          // kamatera needs ssh → gate in v1
  if (denyRe.some(re => re.test(e.id) || re.test(e.name || ""))) return false; // customer-facing / mid-write
  return allowSet.has(e.id);                        // DEFAULT-DENY: only explicitly-trusted ids auto-restart
}
function restartCmd(e) {
  if (e.kind === "pm2") return `pm2 restart ${JSON.stringify(e.name)}`;
  if (e.kind === "launchd" || e.kind === "plist-only")
    return `launchctl kickstart -k gui/$(id -u)/${e.name}`;
  return null;
}

// --- gated memo drafting (idempotent by stable filename) ---
function memoExists(prefix) {
  if (!existsSync(PENDING)) return false;
  return readdirSync(PENDING).some(f => f.startsWith(prefix));
}
function draftMemo(kind, e, body) {
  if (!existsSync(PENDING)) mkdirSync(PENDING, { recursive: true });
  const safeId = String(e.id).replace(/[^a-zA-Z0-9._-]/g, "_");
  const prefix = `fleet-${kind}-${safeId}`;
  if (memoExists(prefix)) return { skipped: true };   // already pending — don't spam
  const day = new Date().toISOString().slice(0, 10);
  const file = join(PENDING, `${prefix}-${day}.md`);
  if (APPLY) writeFileSync(file, body);
  return { file, written: APPLY };
}

const out = { dark_safe_restarted: [], dark_gated: [], unhealthy: [], zombies: [], stale: [], errors: [] };

for (const e of entries) {
  // ZOMBIE: retired but still online
  if (e.lifecycle_status === "retired" && e.live_status === "online") {
    const m = draftMemo("zombie", e, zombieMemo(e));
    out.zombies.push({ id: e.id, memo: m.file || "(pending)", skipped: m.skipped });
    continue;
  }
  // UNHEALTHY: expected_up + looks-live (loaded) but crash-looping (last_exit≠0 / FATAL log).
  // This is the claude-mail blind spot — it passed snapshotHealthy so DARK never fired.
  // NEVER auto-restarted: a loaded-but-failing job (rejected credential, EADDRINUSE,
  // missing module) won't fix on a bounce — bouncing just hides the crash-loop. GATED only.
  if (e.expected_up === true && e.lifecycle_status === "active") {
    const ru = runUnhealthy(e);
    if (ru) {
      const m = draftMemo("unhealthy", e, unhealthyMemo(e, ru));
      out.unhealthy.push({ id: e.id, owner: e.owner, why: ru.why, memo: m.file || "(pending)", skipped: m.skipped });
      continue; // don't ALSO dark-restart it — it's loaded, just broken
    }
  }
  // STALE: deprecated past retire_after, OR expected_up never-seen in STALE_DAYS (and not schedule-healthy)
  const pastRetire = e.lifecycle_status === "deprecated" && e.retire_after && daysSince(e.retire_after) > 0;
  const ghosted = e.expected_up === true && !snapshotHealthy(e) && daysSince(e.last_seen) > STALE_DAYS;
  if (pastRetire || ghosted) {
    const m = draftMemo("retire", e, retireMemo(e, { pastRetire, ghosted }));
    out.stale.push({ id: e.id, why: pastRetire ? "past retire_after" : `unseen ${STALE_DAYS}d+`, memo: m.file || "(pending)", skipped: m.skipped });
    // ghosted is ALSO a dark alert — fall through to DARK handling below
    if (!ghosted) continue;
  }
  // DARK: expected_up but not (truly) alive
  if (e.expected_up === true) {
    const alive = await trulyAlive(e);
    if (alive) continue;
    if (restartSafe(e)) {
      const cmd = restartCmd(e);
      if (!cmd) { out.dark_gated.push({ id: e.id, reason: "no restart cmd for kind" }); continue; }
      if (APPLY) {
        try { execSync(cmd, { stdio: "pipe", timeout: 20000 }); out.dark_safe_restarted.push({ id: e.id, cmd }); }
        catch (err) { out.errors.push({ id: e.id, cmd, err: String(err.message || err).slice(0, 200) }); }
      } else out.dark_safe_restarted.push({ id: e.id, cmd: `${cmd}  (dry-run)` });
    } else {
      const m = draftMemo("dark", e, darkMemo(e));
      out.dark_gated.push({ id: e.id, owner: e.owner, memo: m.file || "(pending)", skipped: m.skipped });
    }
  }
}

// --- memo bodies ---
function head(e) {
  return `- **Service:** \`${e.id}\` (${e.name}) on ${e.machine}, kind=${e.kind}\n` +
         `- **Owner:** ${e.owner || "?"}\n- **live_status:** ${e.live_status}  ·  **lifecycle:** ${e.lifecycle_status}  ·  **last_seen:** ${e.last_seen || "?"}\n` +
         (e.notes ? `- **Registry notes:** ${e.notes}\n` : "");
}
function darkMemo(e) {
  return `# DARK ALERT — \`${e.id}\` is expected_up but down (GATED restart)\n\n${head(e)}\n` +
    `**Why gated:** this service is customer-facing or mid-write (or on Kamatera), so the officer will NOT auto-bounce it — a half-applied restart can be worse than the gap (Q4=B).\n\n` +
    `**To restart (after confirming it's idle / no mid-write):**\n\`\`\`sh\n${restartCmd(e) || "# (no standard restart for this kind — investigate)"}\n\`\`\`\n` +
    `**To abort:** delete this memo.\n\n_Drafted by fleet-reconcile.mjs — Fleet Lifecycle Officer._\n`;
}
function unhealthyMemo(e, ru) {
  const log = `~/Library/Logs/${e.name}.log`; // best-effort hint; real path is in the plist
  return `# UNHEALTHY — \`${e.id}\` is loaded but crash-looping (GATED, NOT auto-restarted)\n\n${head(e)}` +
    `- **Run-health:** ${ru.why}\n\n` +
    `**Why this matters:** the job is \`live_status=loaded\` so every liveness/online check calls it healthy — but it FAILS on every fire. This is the exact failure mode that let \`com.steve.claude-mail\` crash-loop for hours invisibly (rejected IMAP credential, exit-1 every 10s, nothing noticed).\n\n` +
    `**Why NOT auto-restarted:** a loaded-but-failing job won't fix on a bounce — a rejected credential / EADDRINUSE / missing module survives a restart. Bouncing it just HIDES the crash-loop. This needs a real fix, not a kick.\n\n` +
    `**To diagnose:**\n\`\`\`sh\nlaunchctl list ${e.name} | grep LastExitStatus   # confirm non-zero\ntail -50 ${log} 2>/dev/null                       # real log path is in ~/Library/LaunchAgents/${e.name}.plist\n\`\`\`\n` +
    `**If it's a credential/config issue:** fix the secret/config, then \`launchctl kickstart -k gui/$(id -u)/${e.name}\`.\n` +
    `**If the job is obsolete:** set \`lifecycle_status: retired\` for \`${e.id}\` in overlay.json and bootout the plist.\n` +
    `**Consider** an embedded self-heal layer (see \`~/Projects/claude-mail/health.js\`) for any critical job so it pages over mailbox-independent channels after N consecutive fails.\n\n` +
    `**To abort:** delete this memo.\n\n_Drafted by fleet-act.mjs — Fleet Lifecycle Officer (run-health tier)._\n`;
}
function zombieMemo(e) {
  return `# ZOMBIE ALERT — \`${e.id}\` is RETIRED but still online (decommission target)\n\n${head(e)}\n` +
    `**Meaning:** Steve decided to retire this, but it's still running — the decommission didn't finish, or something resurrected it.\n\n` +
    `**To finish decommission (GATED — customer-facing teardown needs your sign-off):**\n\`\`\`sh\n` +
    (e.kind === "pm2" ? `pm2 delete ${JSON.stringify(e.name)} && pm2 save` : `launchctl bootout gui/$(id -u)/${e.name}`) +
    `\n\`\`\`\n**To keep it (reactivate):** set lifecycle_status back to active in overlay.json.\n**To abort:** delete this memo.\n\n_Drafted by fleet-reconcile.mjs._\n`;
}
function retireMemo(e, { pastRetire, ghosted }) {
  return `# RETIREMENT CANDIDATE — \`${e.id}\` is old & irrelevant\n\n${head(e)}\n` +
    `**Why flagged:** ${pastRetire ? `deprecated and past its retire_after (${e.retire_after})` : `expected_up but unseen for >${STALE_DAYS} days`}.\n\n` +
    `**To retire (record in the ledger):** set \`lifecycle_status: retired\` + \`retire_after\` for \`${e.id}\` in \`~/Projects/_shared/fleet-registry/overlay.json\`.\n` +
    `**Reversibility:** archive-before-delete — agent files → ~/.claude/agents/_archived/; \`pm2 resurrect\` / re-load plist to undo.\n` +
    `**To keep it:** add it to safe-restart-allowlist or set expected_up appropriately.\n**To abort:** delete this memo.\n\n_Drafted by fleet-reconcile.mjs — NEVER retires autonomously (Steve approves)._\n`;
}

// --- report + run-log ---
const summary = {
  ts: new Date().toISOString(), mode: APPLY ? "apply" : "dry-run",
  entries: entries.length,
  dark_safe: out.dark_safe_restarted.length, dark_gated: out.dark_gated.length,
  unhealthy: out.unhealthy.length,
  zombies: out.zombies.length, stale: out.stale.length, errors: out.errors.length,
};
try { appendFileSync(RUNLOG, JSON.stringify(summary) + "\n"); } catch {}
console.log(`fleet-reconcile [${summary.mode}] — ${entries.length} real services`);
console.log(`  🟢 safe auto-restart: ${out.dark_safe_restarted.length}`);
out.dark_safe_restarted.forEach(d => console.log(`     ${d.id}  ${d.cmd}`));
console.log(`  🔴 DARK gated (drafted/pending): ${out.dark_gated.length}`);
out.dark_gated.slice(0, 10).forEach(d => console.log(`     ${d.id}  owner=${d.owner || "?"}  ${d.skipped ? "(memo already pending)" : ""}`));
console.log(`  🟠 UNHEALTHY crash-loop (loaded-but-failing, GATED): ${out.unhealthy.length}`);
out.unhealthy.slice(0, 15).forEach(d => console.log(`     ${d.id}  ${d.why}  ${d.skipped ? "(memo already pending)" : ""}`));
console.log(`  🧟 ZOMBIE: ${out.zombies.length}`);
out.zombies.slice(0, 10).forEach(d => console.log(`     ${d.id}  ${d.skipped ? "(pending)" : ""}`));
console.log(`  🟡 retirement candidates: ${out.stale.length}`);
out.stale.slice(0, 10).forEach(d => console.log(`     ${d.id}  (${d.why})  ${d.skipped ? "(pending)" : ""}`));
if (out.errors.length) { console.log(`  ⚠️ restart errors: ${out.errors.length}`); out.errors.forEach(e => console.log(`     ${e.id}: ${e.err}`)); }
console.log(`  cost: $0 (local)`);