`canary.mjs`

189 lines
#!/usr/bin/env node
// dw-launchd-canary — one tier above the meta-watchdog.
//
// The meta-watchdog proves the DW *canaries* ran. This proves the broader
// launchd fleet is HEALTHY: it catches the two blind spots that hid the
// 12-day pg_dump death and that the Officer Council flagged 2026-06-17 —
//   (1) a job that exits nonzero every run, and
//   (2) a plist that should be scheduled but isn't even LOADED (drift on a
//       reboot/fleet restart, failed bootstrap, or label mismatch).
//
// HONEST-by-design (mirrors scripts/launchd-audit/audit.sh's hard-won rules):
//   * Auto-discovers ~/Library/LaunchAgents/com.steve.*.plist. Jobs renamed
//     .DISABLED / .done / .RETIRED, or moved to disabled-*/ , do NOT match the
//     glob — so deliberate-offs never false-alarm.
//   * health_check_jobs (manifest) exit 1 = "reporting a problem", NOT broken.
//   * Debounced: a job must be bad on >= consecutive_fail_threshold runs before
//     it FAILs — one transient (a 5am restart mid-run) won't page anyone.
// READ-ONLY: never loads/unloads/edits a job. On a confirmed regression: CNCP
// card + George email. Writes data/latest.json so the meta-watchdog covers it.
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { execSync } from 'node:child_process';

const HOME = os.homedir();
const SKILL = path.join(HOME, 'Projects/dw-launchd-canary');
const DATA = path.join(SKILL, 'data');
const LA = path.join(HOME, 'Library/LaunchAgents');
fs.mkdirSync(DATA, { recursive: true });
const now = Date.now();
const ENV = process.env;
const LOG = path.join(DATA, 'run.log');
const HIST = path.join(DATA, 'history.jsonl');
const log = (m) => { const l = `[${new Date(now).toISOString()}] ${m}`; console.log(l); try { fs.appendFileSync(LOG, l + '\n'); } catch {} };

const manifest = JSON.parse(fs.readFileSync(path.join(SKILL, 'manifest.json'), 'utf8'));
const healthCheck = new Set(manifest.health_check_jobs || []);
const expectedUnloaded = new Set(manifest.expected_unloaded || []);
const notes = manifest.notes || {};
const THRESH = manifest.consecutive_fail_threshold || 2;

// --- discover the jobs that SHOULD be scheduled (active .plist files only) ---
const plists = fs.readdirSync(LA).filter(f => /^com\.steve\..+\.plist$/.test(f));

function launchctlInfo(label) {
  // returns { loaded, running, exit } — exit is the macOS-raw LastExitStatus or null.
  // running = a live PID is present; a KeepAlive daemon that's running now is NOT
  // failing even if LastExitStatus is a stale nonzero from a prior respawn cycle.
  let line = '';
  try { line = execSync(`launchctl list ${label} 2>/dev/null`, { encoding: 'utf8' }); } catch { line = ''; }
  if (!line.trim()) {
    // fall back to the column form (some labels only show in the table)
    try {
      const tbl = execSync(`launchctl list 2>/dev/null | grep -F ${label}`, { encoding: 'utf8' }).trim();
      if (!tbl) return { loaded: false, running: false, exit: null };
      const cols = tbl.split(/\s+/); // PID  STATUS  LABEL
      const ex = cols[1] === '-' ? null : parseInt(cols[1], 10);
      return { loaded: true, running: cols[0] !== '-' && /^\d+$/.test(cols[0]), exit: Number.isNaN(ex) ? null : ex };
    } catch { return { loaded: false, running: false, exit: null }; }
  }
  const m = line.match(/"LastExitStatus"\s*=\s*(\d+)/);
  const p = line.match(/"PID"\s*=\s*(\d+)/);
  return { loaded: true, running: !!p, exit: m ? parseInt(m[1], 10) : null };
}

const results = [];
for (const f of plists) {
  const label = f.replace(/\.plist$/, '');
  const short = label.replace(/^com\.steve\./, '');
  // is it Disabled=true in the plist itself? (deliberate pause, keeps the .plist name)
  let disabled = false;
  try { disabled = /<key>Disabled<\/key>\s*<true\/>/.test(fs.readFileSync(path.join(LA, f), 'utf8')); } catch {}
  const { loaded, running, exit } = launchctlInfo(label);

  let verdict = 'OK', reason = 'loaded, exit 0';
  if (disabled) { verdict = 'PAUSED'; reason = 'Disabled=true in plist (deliberate)'; }
  else if (expectedUnloaded.has(short)) { verdict = 'EXPECTED_OFF'; reason = 'intentionally unloaded (manifest expected_unloaded)'; }
  else if (!loaded) { verdict = 'DROPPED'; reason = 'plist on disk but NOT in launchctl — silently unloaded (drift/reboot/failed bootstrap)'; }
  else if (running) { verdict = 'OK'; reason = 'running now (live PID) — a stale LastExitStatus is prior-cycle residue, not a failure'; }
  else if (exit === null) { verdict = 'OK'; reason = 'loaded, no exit status yet (not run since load)'; }
  else if (exit === 0) { verdict = 'OK'; reason = 'loaded, exit 0'; }
  else if (exit % 256 !== 0) {
    // raw status whose low byte is nonzero = terminated by a signal (9=KILL,15=TERM).
    // For KeepAlive daemons that's restart noise, not a job failure — surface, don't page.
    verdict = 'KILLED'; reason = `terminated by signal ${exit & 0x7f} (daemon restart / kill — not a job failure)`;
  } else {
    const code = exit / 256; // clean exit code: 256→1, 512→2 …
    if (healthCheck.has(short) && code === 1) { verdict = 'OK'; reason = 'exit 1 = health-check reporting a finding (intended)'; }
    else { verdict = 'FAILING'; reason = `nonzero exit (raw ${exit}, code ${code})`; }
  }
  results.push({ short, verdict, reason, note: notes[short] || null });
}

// --- debounce against history: a regression must persist THRESH runs ---
let hist = [];
try { hist = fs.readFileSync(HIST, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); } catch {}
const recent = hist.slice(-(THRESH - 1)); // prior runs to combine with this one
const bad = new Set(['DROPPED', 'FAILING']);
function confirmed(short, thisVerdict) {
  if (!bad.has(thisVerdict)) return false;
  // bad now AND bad in each of the prior (THRESH-1) runs
  if (recent.length < THRESH - 1) return false; // not enough history yet → hold
  return recent.every(run => {
    const r = (run.results || []).find(x => x.short === short);
    return r && bad.has(r.verdict);
  });
}

// One-time baseline: the set of jobs already bad when this canary was installed.
// They're acknowledged (the existing bootstrap backlog → fed to the #5 plist
// batcher), so we surface them but DON'T page about them every hour. Only a job
// that goes bad AFTER the baseline is a true regression worth an alert.
const BASE = path.join(DATA, 'baseline.json');
let baseline;
if (fs.existsSync(BASE)) { try { baseline = new Set(JSON.parse(fs.readFileSync(BASE, 'utf8')).known || []); } catch { baseline = new Set(); } }
else {
  baseline = new Set(results.filter(r => bad.has(r.verdict)).map(r => r.short));
  fs.writeFileSync(BASE, JSON.stringify({ seeded_at: new Date(now).toISOString(), known: [...baseline] }, null, 2));
  log(`seeded baseline with ${baseline.size} pre-existing bad job(s): ${[...baseline].join(', ') || '(none)'}`);
}

const confirmedBad = results.filter(r => bad.has(r.verdict) && confirmed(r.short, r.verdict));
const regressions = confirmedBad.filter(r => !baseline.has(r.short)); // NEW breakage → page
const acknowledged = confirmedBad.filter(r => baseline.has(r.short));  // pre-existing backlog → surface only
const pending = results.filter(r => bad.has(r.verdict) && !confirmed(r.short, r.verdict)); // bad but not yet THRESH runs

const summary = {
  ts: new Date(now).toISOString(),
  total: results.length,
  ok: results.filter(r => r.verdict === 'OK').length,
  paused: results.filter(r => r.verdict === 'PAUSED').length,
  expected_off: results.filter(r => r.verdict === 'EXPECTED_OFF').length,
  killed: results.filter(r => r.verdict === 'KILLED').length,
  dropped: results.filter(r => r.verdict === 'DROPPED').length,
  failing: results.filter(r => r.verdict === 'FAILING').length,
  confirmed_regressions: regressions.map(r => r.short),
  acknowledged_backlog: acknowledged.map(r => r.short),
  pending_confirmation: pending.map(r => r.short),
  results
};
fs.writeFileSync(path.join(DATA, 'latest.json'), JSON.stringify(summary, null, 2));
fs.appendFileSync(HIST, JSON.stringify({ ts: summary.ts, results: results.map(r => ({ short: r.short, verdict: r.verdict })) }) + '\n');
// keep history bounded
try { const lines = fs.readFileSync(HIST, 'utf8').trim().split('\n'); if (lines.length > 500) fs.writeFileSync(HIST, lines.slice(-500).join('\n') + '\n'); } catch {}

log(`launchd-canary: ${summary.total} jobs · ${summary.ok} OK · ${summary.paused} paused · ${summary.expected_off} expected-off · ${summary.killed} killed · ${summary.dropped} dropped · ${summary.failing} failing`);
for (const r of regressions) log(`  REGRESSION ${r.short} — ${r.reason}`);
for (const r of acknowledged) log(`  acknowledged-backlog ${r.short} — ${r.reason}`);
for (const r of pending) log(`  pending ${r.short} — ${r.reason}`);

// self-heartbeat so the meta-watchdog can prove THIS canary ran
try { fs.mkdirSync(path.join(HOME, '.claude/heartbeats'), { recursive: true }); fs.writeFileSync(path.join(HOME, '.claude/heartbeats/dw-launchd-canary.stamp'), String(now)); } catch {}

// --- alerting (George flipped to HTTPS-only 2026-06-17 — use the MagicDNS host) ---
const GEORGE = ENV.GEORGE_URL || 'https://kamatera.tail79cb8e.ts.net:9850';
async function postCNCP(note) {
  const base = ENV.CNCP_URL || 'http://localhost:3333';
  try { const ac = new AbortController(); const t = setTimeout(() => ac.abort(), 8000);
    const r = await fetch(`${base}/api/parking-lot`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: 'file://' + SKILL, note }), signal: ac.signal });
    clearTimeout(t); return r.ok; } catch { return false; }
}
async function emailGeorge(subject, html) {
  let auth = ENV.GEORGE_AUTH;
  if (!auth) { try { auth = (fs.readFileSync(path.join(HOME, 'Projects/secrets-manager/.env'), 'utf8').match(/^GEORGE_AUTH=(.*)$/m) || [])[1]; } catch {} }
  if (!auth) { log('email skipped (no GEORGE_AUTH)'); return false; }
  try { const ac = new AbortController(); const t = setTimeout(() => ac.abort(), 15000);
    const r = await fetch(`${GEORGE}/api/send`, { method: 'POST', headers: { 'Authorization': auth, 'Content-Type': 'application/json; charset=utf-8' },
      body: JSON.stringify({ to: ENV.CANARY_TO || 'steve@designerwallcoverings.com', subject, body: html }), signal: ac.signal });
    clearTimeout(t); const j = await r.json().catch(() => ({})); return !!(j.success || j.messageId); } catch { return false; }
}

if (regressions.length) {
  const note = `[LAUNCHD CANARY ${summary.ts.slice(0, 10)}] ${regressions.length} confirmed launchd regression(s) (bad ${THRESH}+ runs): `
    + regressions.map(r => `${r.short} — ${r.verdict} (${r.reason})`).join('; ')
    + `. A job that exits nonzero every run or fell out of launchctl emits zero alerts on its own — that's the silent-death class.`;
  await postCNCP(note);
  const html = `<div style="font-family:-apple-system,sans-serif;color:#222"><h3 style="color:#b23b3b">⚠ launchd canary — ${regressions.length} confirmed regression(s)</h3>`
    + regressions.map(r => `<div style="margin:6px 0"><b>${r.short}</b> — <span style="color:#b23b3b">${r.verdict}</span> ${r.reason}${r.note ? ` <span style="color:#888">(${r.note})</span>` : ''}</div>`).join('')
    + (pending.length ? `<hr><div style="color:#a07000">Watching (not yet ${THRESH} runs): ${pending.map(p => p.short + ' — ' + p.verdict).join('; ')}</div>` : '')
    + `<hr><div style="color:#888;font-size:12px">${summary.ok} OK · ${summary.paused} paused (deliberate) · ${summary.dropped} dropped · ${summary.failing} failing</div></div>`;
  const emailed = await emailGeorge(`⚠ launchd canary — ${regressions.length} regression — ${summary.ts.slice(0, 10)}`, html);
  log(emailed ? 'regression alert emailed' : 'regression alert email NOT sent');
  process.exitCode = 1;
} else {
  log('✓ no NEW launchd regressions'
    + (acknowledged.length ? ` (${acknowledged.length} pre-existing backlog, not paged: ${acknowledged.map(a => a.short).join(', ')})` : '')
    + (pending.length ? ` (${pending.length} watching: ${pending.map(p => p.short).join(', ')})` : ''));
}