`scripts/canary-meta-watchdog/canary-meta-watchdog.js`

135 lines
#!/usr/bin/env node
/**
 * Canary Meta-Watchdog — proves the canaries themselves actually RAN.
 *
 * The fleet's worst failures are silent monitoring deaths: the dw_unified
 * pg_dump wrote 0 bytes for 12 days unnoticed, the Kamatera health-watchdog
 * went down, and ≥6 launchd jobs are failing — each a case of "the thing that
 * should have warned us is the thing that broke." This meta-watchdog reads
 * every com.steve.* LaunchAgent, derives its expected cadence, and FAILs
 * (report-only) if a job is unloaded, last-exited non-zero, or hasn't produced
 * fresh log output within its expected interval (× a grace factor).
 *
 * READ-ONLY: parses plists + stats log files + reads `launchctl print`. Writes
 * only a local report JSON. No prod writes, no network. Cost: $0 (local).
 *
 *   node canary-meta-watchdog.js            # human table + write report JSON
 *   node canary-meta-watchdog.js --json     # machine JSON to stdout
 */
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const HOME = process.env.HOME;
const LA_DIR = path.join(HOME, 'Library/LaunchAgents');
const UID = parseInt(execSync('id -u').toString().trim(), 10);
const GRACE = 1.5; // allow 1.5× the expected interval before flagging stale
const OUT = path.join(HOME, '.claude/yolo-queue', `canary-meta-watchdog-${new Date().toISOString().slice(0,10)}.json`);
const JSON_ONLY = process.argv.includes('--json');

const now = Date.now();
const sec = (ms) => ms / 1000;

function plistToObj(file) {
  try { return JSON.parse(execSync(`plutil -convert json -o - ${JSON.stringify(file)}`, { stdio: ['ignore','pipe','ignore'] }).toString()); }
  catch (e) { return null; }
}

// Expected period in seconds from StartInterval or StartCalendarInterval.
function expectedPeriodSec(p) {
  if (typeof p.StartInterval === 'number') return p.StartInterval;
  const sci = p.StartCalendarInterval;
  if (sci) {
    const entries = Array.isArray(sci) ? sci : [sci];
    // crude cadence inference: Minute-only => hourly; Hour set => daily; Weekday set => weekly; Day set => monthly
    const e = entries[0] || {};
    if ('Weekday' in e) return 7 * 86400;
    if ('Day' in e) return 30 * 86400;
    if ('Hour' in e) return 86400;            // runs at a fixed hour daily
    if ('Minute' in e) return 3600;           // runs each hour at a minute
    return 86400;
  }
  return null; // RunAtLoad-only / on-demand — no cadence to check
}

// Last-run proxy = newest mtime among the job's stdout/stderr log paths.
function lastRunMs(p) {
  const paths = [p.StandardOutPath, p.StandardErrorPath].filter(Boolean);
  let newest = 0;
  for (const f of paths) {
    try { const st = fs.statSync(f); newest = Math.max(newest, st.mtimeMs); } catch (_) {}
  }
  return newest || null;
}

// launchctl state: loaded? last exit code? pid?
function launchctlState(label) {
  try {
    const out = execSync(`launchctl print gui/${UID}/${label} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString();
    const pid = (out.match(/pid = (\d+)/) || [])[1] || null;
    const lastExit = (out.match(/last exit code = (\d+)/) || [])[1];
    return { loaded: true, pid, lastExit: lastExit !== undefined ? parseInt(lastExit, 10) : null };
  } catch (e) {
    // fall back to `launchctl list` membership
    try {
      const line = execSync(`launchctl list | grep ${JSON.stringify(label)} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString().trim();
      if (!line) return { loaded: false, pid: null, lastExit: null };
      const cols = line.split(/\s+/);
      return { loaded: true, pid: cols[0] === '-' ? null : cols[0], lastExit: cols[1] === '-' ? null : parseInt(cols[1], 10) };
    } catch (_) { return { loaded: false, pid: null, lastExit: null }; }
  }
}

const files = fs.readdirSync(LA_DIR).filter(f => /^com\.steve\..*\.plist$/.test(f));
const results = [];

for (const f of files) {
  const full = path.join(LA_DIR, f);
  const p = plistToObj(full);
  if (!p || !p.Label) { results.push({ label: f.replace(/\.plist$/, ''), verdict: 'WARN', why: 'unparseable plist' }); continue; }
  const label = p.Label;
  const period = expectedPeriodSec(p);
  const last = lastRunMs(p);
  const st = launchctlState(label);
  const ageSec = last ? sec(now - last) : null;

  let verdict = 'PASS', why = [];
  if (!st.loaded) { verdict = 'FAIL'; why.push('NOT LOADED in launchd'); }
  if (st.lastExit && st.lastExit !== 0) { verdict = 'FAIL'; why.push(`last exit = ${st.lastExit}`); }
  if (period && ageSec !== null) {
    if (ageSec > period * GRACE) { verdict = 'FAIL'; why.push(`STALE: last log ${(ageSec/3600).toFixed(1)}h ago vs ${(period/3600).toFixed(1)}h cadence`); }
  } else if (period && last === null) {
    if (verdict === 'PASS') verdict = 'WARN'; why.push('no log output found (never ran or logs elsewhere)');
  } else if (!period) {
    if (verdict === 'PASS') verdict = 'INFO'; why.push('on-demand/RunAtLoad — no cadence to check');
  }
  results.push({
    label, verdict,
    cadence_h: period ? +(period/3600).toFixed(1) : null,
    last_run_age_h: ageSec !== null ? +(ageSec/3600).toFixed(1) : null,
    loaded: st.loaded, last_exit: st.lastExit, pid: st.pid,
    why: why.join('; ') || 'ok',
  });
}

const rank = { FAIL: 0, WARN: 1, INFO: 2, PASS: 3 };
results.sort((a, b) => (rank[a.verdict] - rank[b.verdict]) || a.label.localeCompare(b.label));
const summary = results.reduce((m, r) => (m[r.verdict] = (m[r.verdict]||0)+1, m), {});
const report = { generated_at: new Date().toISOString(), launch_agents_dir: LA_DIR, grace_factor: GRACE, total: results.length, summary, results };

fs.writeFileSync(OUT, JSON.stringify(report, null, 2));

if (JSON_ONLY) { console.log(JSON.stringify(report, null, 2)); process.exit(0); }

console.log(`\nCanary Meta-Watchdog — ${results.length} com.steve.* jobs  (${new Date().toISOString().slice(0,16)})`);
console.log(`Summary: ${Object.entries(summary).map(([k,v])=>`${k}=${v}`).join('  ')}`);
console.log('─'.repeat(96));
for (const r of results) {
  const tag = { FAIL:'🔴', WARN:'🟠', INFO:'⚪', PASS:'🟢' }[r.verdict];
  console.log(`${tag} ${r.verdict.padEnd(4)} ${r.label.padEnd(42)} ${r.why}`);
}
console.log('─'.repeat(96));
console.log(`Report: ${OUT}`);
const fails = results.filter(r => r.verdict === 'FAIL');
if (fails.length) console.log(`\n⚠️  ${fails.length} canary FAIL(s) — these monitors are not provably running.`);