`scripts/canary-meta-watchdog/canary-meta-watchdog.js`

194 lines
#!/usr/bin/env node
/**
 * Canary Meta-Watchdog — proves the canaries themselves actually RAN.
 *
 * The fleet's worst failures are silent monitoring deaths: the dw_unified
 * pg_dump wrote 0 bytes for 12 days unnoticed, the Kamatera health-watchdog
 * went down, and ≥6 launchd jobs are failing — each a case of "the thing that
 * should have warned us is the thing that broke." This meta-watchdog reads
 * every com.steve.* LaunchAgent, derives its expected cadence, and FAILs
 * (report-only) if a job is unloaded, last-exited non-zero, or hasn't produced
 * fresh log output within its expected interval (× a grace factor).
 *
 * READ-ONLY: parses plists + stats log files + reads `launchctl print`. Writes
 * only a local report JSON. No prod writes, no network. Cost: $0 (local).
 *
 *   node canary-meta-watchdog.js            # human table + write report JSON
 *   node canary-meta-watchdog.js --json     # machine JSON to stdout
 */
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const HOME = process.env.HOME;
const LA_DIR = path.join(HOME, 'Library/LaunchAgents');
const UID = parseInt(execSync('id -u').toString().trim(), 10);
const GRACE = 1.5; // allow 1.5× the expected interval before flagging stale
const OUT = path.join(HOME, '.claude/yolo-queue', `canary-meta-watchdog-${new Date().toISOString().slice(0,10)}.json`);
const JSON_ONLY = process.argv.includes('--json');

const now = Date.now();
const sec = (ms) => ms / 1000;

function plistToObj(file) {
  try { return JSON.parse(execSync(`plutil -convert json -o - ${JSON.stringify(file)}`, { stdio: ['ignore','pipe','ignore'] }).toString()); }
  catch (e) { return null; }
}

// Expected period in seconds from StartInterval or StartCalendarInterval.
function expectedPeriodSec(p) {
  if (typeof p.StartInterval === 'number') return p.StartInterval;
  const sci = p.StartCalendarInterval;
  if (sci) {
    const entries = Array.isArray(sci) ? sci : [sci];
    // crude cadence inference: Minute-only => hourly; Hour set => daily; Weekday set => weekly; Day set => monthly
    const e = entries[0] || {};
    if ('Weekday' in e) return 7 * 86400;
    if ('Day' in e) return 30 * 86400;
    if ('Hour' in e) return 86400;            // runs at a fixed hour daily
    if ('Minute' in e) return 3600;           // runs each hour at a minute
    return 86400;
  }
  return null; // RunAtLoad-only / on-demand — no cadence to check
}

// Some jobs `exec >> "$LOG"` to a DIFFERENT file than launchd's StandardOutPath, so the
// captured .out/.err freeze at deploy time and falsely read as "stale/dead" (cost hours on
// com.steve.dw-cadence-hourly 2026-06-23 — the real heartbeat is .log, not .out). Map such
// jobs to their REAL log here so liveness is judged on the file the script actually writes.
const REAL_LOG_OVERRIDES = {
  'com.steve.dw-cadence-hourly': '/tmp/dw-cadence-hourly.log',
};
// Non-zero exit codes that are HEALTHY-BUT-CAPPED, not outages. The cadence exits 3 when it
// hits Shopify's rolling 1,000-variant/24h cap (DAILY_VARIANT_LIMIT) — it ran fine and will
// resume next slot after the cap rolls over, so it must NOT read as a dead job.
const HEALTHY_EXIT_CODES = {
  'com.steve.dw-cadence-hourly': [3],
};
// Last-run proxy = newest mtime among the job's real log + stdout/stderr paths. Also probes a
// sibling ".log" of any ".out" path (the common `exec >> *.log` pattern) so other such jobs
// self-heal without an explicit override.
function lastRunMs(p) {
  const paths = [REAL_LOG_OVERRIDES[p.Label], p.StandardOutPath, p.StandardErrorPath].filter(Boolean);
  for (const f of [p.StandardOutPath, p.StandardErrorPath]) {
    if (f && f.endsWith('.out')) paths.push(f.replace(/\.out$/, '.log'));
    if (f && f.endsWith('.err')) paths.push(f.replace(/\.err$/, '.log'));
  }
  let newest = 0;
  for (const f of paths) {
    try { const st = fs.statSync(f); newest = Math.max(newest, st.mtimeMs); } catch (_) {}
  }
  return newest || null;
}

// launchctl state: loaded? last exit code? pid?
function launchctlState(label) {
  try {
    const out = execSync(`launchctl print gui/${UID}/${label} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString();
    const pid = (out.match(/pid = (\d+)/) || [])[1] || null;
    const lastExit = (out.match(/last exit code = (\d+)/) || [])[1];
    return { loaded: true, pid, lastExit: lastExit !== undefined ? parseInt(lastExit, 10) : null };
  } catch (e) {
    // fall back to `launchctl list` membership
    try {
      const line = execSync(`launchctl list | grep ${JSON.stringify(label)} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString().trim();
      if (!line) return { loaded: false, pid: null, lastExit: null };
      const cols = line.split(/\s+/);
      return { loaded: true, pid: cols[0] === '-' ? null : cols[0], lastExit: cols[1] === '-' ? null : parseInt(cols[1], 10) };
    } catch (_) { return { loaded: false, pid: null, lastExit: null }; }
  }
}

const files = fs.readdirSync(LA_DIR).filter(f => /^com\.steve\..*\.plist$/.test(f));
const results = [];

for (const f of files) {
  const full = path.join(LA_DIR, f);
  const p = plistToObj(full);
  if (!p || !p.Label) { results.push({ label: f.replace(/\.plist$/, ''), verdict: 'WARN', why: 'unparseable plist' }); continue; }
  const label = p.Label;
  const period = expectedPeriodSec(p);
  const last = lastRunMs(p);
  const st = launchctlState(label);
  const ageSec = last ? sec(now - last) : null;

  let verdict = 'PASS', why = [];
  if (!st.loaded) { verdict = 'FAIL'; why.push('NOT LOADED in launchd'); }
  if (st.lastExit && st.lastExit !== 0) {
    if ((HEALTHY_EXIT_CODES[label] || []).includes(st.lastExit)) {
      why.push(`exit ${st.lastExit} = healthy-but-capped (not an outage)`);
    } else { verdict = 'FAIL'; why.push(`last exit = ${st.lastExit}`); }
  }
  if (period && ageSec !== null) {
    if (ageSec > period * GRACE) { verdict = 'FAIL'; why.push(`STALE: last log ${(ageSec/3600).toFixed(1)}h ago vs ${(period/3600).toFixed(1)}h cadence`); }
  } else if (period && last === null) {
    if (verdict === 'PASS') verdict = 'WARN'; why.push('no log output found (never ran or logs elsewhere)');
  } else if (!period) {
    if (verdict === 'PASS') verdict = 'INFO'; why.push('on-demand/RunAtLoad — no cadence to check');
  }
  results.push({
    label, verdict,
    cadence_h: period ? +(period/3600).toFixed(1) : null,
    last_run_age_h: ageSec !== null ? +(ageSec/3600).toFixed(1) : null,
    loaded: st.loaded, last_exit: st.lastExit, pid: st.pid,
    why: why.join('; ') || 'ok',
  });
}

// --- Status-file probes: jobs that publish a data/latest.json with self-reported
// health. The plist loop above only proves a job RAN; this reads what it FOUND, so a
// job that ran fine (exit 0, fresh log) but DETECTED a problem (e.g. George token
// dead) still FAILs. Registry: status-files.json (optional — never breaks the watchdog).
try {
  const regPath = path.join(__dirname, 'status-files.json');
  if (fs.existsSync(regPath)) {
    const reg = JSON.parse(fs.readFileSync(regPath, 'utf8'));
    for (const entry of (reg.status_files || [])) {
      const file = (entry.file || '').replace(/^~/, HOME);
      let verdict = 'PASS', why = [], data = null, ageH = null;
      try { data = JSON.parse(fs.readFileSync(file, 'utf8')); }
      catch (e) { verdict = 'FAIL'; why.push(`no/unreadable latest.json (${e.code || e.message})`); }
      if (data) {
        if (data.ts) ageH = +(((now - Date.parse(data.ts)) / 3600000)).toFixed(1);
        if (entry.max_age_h != null && ageH !== null && ageH > entry.max_age_h) {
          verdict = 'FAIL'; why.push(`STALE status: ${ageH}h old vs ${entry.max_age_h}h max`);
        }
        const ov = String(data.overall || '').toLowerCase();
        if (ov && ov !== 'ok') {
          verdict = 'FAIL';
          const dead = Array.isArray(data.deadTokens) && data.deadTokens.length ? ` (dead: ${data.deadTokens.join(',')})` : '';
          why.push(`reported overall=${data.overall}${dead}`);
        } else if (ov === 'ok' && verdict === 'PASS') {
          why.push(`overall=ok${ageH !== null ? `, ${ageH}h fresh` : ''}`);
        }
      }
      results.push({
        label: (entry.label || 'status') + ' (status)', verdict,
        cadence_h: null, last_run_age_h: ageH, loaded: null, last_exit: null, pid: null,
        why: why.join('; ') || 'ok', source: 'status-file',
      });
    }
  }
} catch (e) { /* registry is optional — a parse error must never break the plist watchdog */ }

const rank = { FAIL: 0, WARN: 1, INFO: 2, PASS: 3 };
results.sort((a, b) => (rank[a.verdict] - rank[b.verdict]) || a.label.localeCompare(b.label));
const summary = results.reduce((m, r) => (m[r.verdict] = (m[r.verdict]||0)+1, m), {});
const report = { generated_at: new Date().toISOString(), launch_agents_dir: LA_DIR, grace_factor: GRACE, total: results.length, summary, results };

fs.writeFileSync(OUT, JSON.stringify(report, null, 2));

if (JSON_ONLY) { console.log(JSON.stringify(report, null, 2)); process.exit(0); }

console.log(`\nCanary Meta-Watchdog — ${results.length} com.steve.* jobs  (${new Date().toISOString().slice(0,16)})`);
console.log(`Summary: ${Object.entries(summary).map(([k,v])=>`${k}=${v}`).join('  ')}`);
console.log('─'.repeat(96));
for (const r of results) {
  const tag = { FAIL:'🔴', WARN:'🟠', INFO:'⚪', PASS:'🟢' }[r.verdict];
  console.log(`${tag} ${r.verdict.padEnd(4)} ${r.label.padEnd(42)} ${r.why}`);
}
console.log('─'.repeat(96));
console.log(`Report: ${OUT}`);
const fails = results.filter(r => r.verdict === 'FAIL');
if (fails.length) console.log(`\n⚠️  ${fails.length} canary FAIL(s) — these monitors are not provably running.`);