`health-scan.mjs`

85 lines
#!/usr/bin/env node
// health-scan.mjs — READ-ONLY health report for EVERY process in the fleet manifest.
// Reports per-process: name, kind, expected_up, live_status, run-health (last_exit /
// log_health), and a verdict. Takes NO heal action — pure visibility. This is the
// "debug every process" view Steve asked for: the same manifest the steward consumes,
// rendered as a health table that surfaces the loaded-but-crash-looping class
// (the com.steve.claude-mail failure mode) that liveness/online checks miss.
//
// Usage:  node health-scan.mjs            # all mac2 + kamatera, grouped by verdict
//         node health-scan.mjs --bad      # only non-HEALTHY rows
//         node health-scan.mjs --json      # machine-readable
// Never restarts/stops/deletes anything. Exit 0 always.

import { readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";

const MANIFEST = process.env.FLEET_MANIFEST || join(homedir(), "Projects/_shared/fleet-registry/manifest.json");
const ONLY_BAD = process.argv.includes("--bad");
const AS_JSON = process.argv.includes("--json");

const manifest = JSON.parse(readFileSync(MANIFEST, "utf8"));
const REAL = new Set(["mac2", "kamatera", "mac1"]);
const entries = (manifest.entries || []).filter(e => e && REAL.has(e.machine) && e.kind && !String(e.id || "").startsWith("_"));

const HEALTHY_SCHED = new Set(["loaded", "running", "online"]);
const isScheduled = e => !!e.schedule || e.kind === "launchd" || e.kind === "plist-only";
const looksLive = e => isScheduled(e) ? HEALTHY_SCHED.has(e.live_status) : e.live_status === "online";

// classify each entry into a verdict (mirrors fleet-act.mjs but report-only)
function verdict(e) {
  if (e.lifecycle_status === "retired") {
    return e.live_status === "online"
      ? { v: "ZOMBIE", why: "retired but still online" }
      : { v: "RETIRED", why: "retired, not running (steady state)" };
  }
  if (e.lifecycle_status === "deprecated") return { v: "DEPRECATED", why: "winding down" };
  if (e.expected_up !== true) return { v: "OFF-OK", why: "not expected up, not running" };

  // expected_up === true && active:
  if (!looksLive(e)) return { v: "DARK", why: `expected up but live_status=${e.live_status}` };

  // looks-live — check run-health (launchd crash-loop class)
  if (e.kind === "launchd" || e.kind === "plist-only") {
    const exit = e.last_exit, raw = e.last_exit_raw;
    const running = e.live_status === "running", loaded = e.live_status === "loaded";
    const signalled = raw !== null && raw !== undefined && raw < 0;
    if (loaded) {
      if (exit !== null && exit !== undefined && exit !== 0)
        return { v: "UNHEALTHY", why: `loaded but last_exit=${exit} (crash-loop)` };
      if ((exit === null || exit === undefined) && e.log_health === "fatal")
        return { v: "UNHEALTHY", why: "loaded, FATAL log tail (no exit to clear it)" };
    } else if (running && !signalled && exit > 0 && e.log_health === "fatal") {
      return { v: "UNHEALTHY", why: `running but flapping (last_exit=${exit} + FATAL log)` };
    }
  }
  return { v: "HEALTHY", why: e.health_url ? `live + health_url ${e.health_url}` : "live" };
}

const rows = entries.map(e => ({ ...verdict(e), id: e.id, name: e.name, machine: e.machine, kind: e.kind,
  expected_up: e.expected_up, live_status: e.live_status, last_exit: e.last_exit, log_health: e.log_health, owner: e.owner }));

const ORDER = ["UNHEALTHY", "ZOMBIE", "DARK", "DEPRECATED", "HEALTHY", "OFF-OK", "RETIRED"];
const ICON = { UNHEALTHY: "🟠", ZOMBIE: "🧟", DARK: "🔴", DEPRECATED: "🟡", HEALTHY: "🟢", "OFF-OK": "⚪", RETIRED: "⚫" };
const counts = rows.reduce((a, r) => (a[r.v] = (a[r.v] || 0) + 1, a), {});

if (AS_JSON) {
  console.log(JSON.stringify({ generated_at: manifest.generated_at, total: rows.length, counts, rows }, null, 2));
} else {
  console.log(`fleet health-scan (READ-ONLY) — ${rows.length} processes · manifest @ ${manifest.generated_at}`);
  console.log(ORDER.map(v => `${ICON[v]} ${v}:${counts[v] || 0}`).join("  "));
  console.log("");
  for (const v of ORDER) {
    if (ONLY_BAD && ["HEALTHY", "OFF-OK", "RETIRED"].includes(v)) continue;
    const grp = rows.filter(r => r.v === v).sort((a, b) => a.id.localeCompare(b.id));
    if (!grp.length) continue;
    console.log(`${ICON[v]} ${v} (${grp.length})`);
    const show = (v === "HEALTHY" || v === "OFF-OK" || v === "RETIRED") ? grp.slice(0, 6) : grp;
    for (const r of show) console.log(`   ${r.name.padEnd(38)} ${r.machine}/${r.kind.padEnd(11)} ${r.why}`);
    if (show.length < grp.length) console.log(`   … and ${grp.length - show.length} more`);
    console.log("");
  }
  console.log("cost: $0 (local) — read-only, no heal action taken");
}