`watchdog.mjs`

99 lines
#!/usr/bin/env node
// Canary Meta-Watchdog — proves the DW canaries actually RAN recently.
// DTD-C hybrid (2026-06-15 unanimous): detection precedence per canary =
//   explicit stamp  >  result artifact mtime (authoritative completion)
//   >  launchd /tmp log mtime (DEGRADED, stricter)  >  never-ran (FAIL).
// READ-ONLY w.r.t. the canaries: only stats their files. On FAIL: CNCP + George.
// Self-coverage: writes its own stamp + posts a CNCP "watchdog alive" beat each run.
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';

const SKILL = path.join(os.homedir(), 'Projects/dw-canary-watchdog');
const DATA = path.join(SKILL, 'data');
fs.mkdirSync(DATA, { recursive: true });
const LOG = path.join(DATA, 'run.log');
const now = Date.now();
const ENV = process.env;
const exp = (p) => p ? p.replace(/^~(?=\/)/, os.homedir()) : p;
const log = (m) => { const l = `[${new Date(now).toISOString()}] ${m}`; console.log(l); try { fs.appendFileSync(LOG, l + '\n'); } catch {} };

const manifest = JSON.parse(fs.readFileSync(path.join(SKILL, 'manifest.json'), 'utf8'));

function freshestSignal(c) {
  // returns {source, mtime, confidence, emptyRecent} or null
  const tryStat = (p) => { try { const s = fs.statSync(exp(p)); return s; } catch { return null; } };
  // 1. explicit stamp — authoritative
  if (c.stamp) { const s = tryStat(c.stamp); if (s) return { source: 'stamp', mtime: s.mtimeMs, confidence: 'authoritative', size: s.size }; }
  // 2. result artifact — authoritative completion proof
  if (c.artifact) { const s = tryStat(c.artifact); if (s) return { source: 'artifact', mtime: s.mtimeMs, confidence: 'authoritative', size: s.size }; }
  // 3. launchd log — degraded (proves wrapper started, not that work finished)
  if (c.launchd_log) { const s = tryStat(c.launchd_log); if (s) return { source: 'launchd_log', mtime: s.mtimeMs, confidence: 'degraded', size: s.size }; }
  return null;
}

const results = [];
for (const c of manifest.canaries) {
  const sig = freshestSignal(c);
  if (!sig) { results.push({ label: c.label, verdict: 'FAIL', reason: 'no run trace found (stamp/artifact/log all absent) — never ran or wiped', confidence: 'none', age_h: null, cadence: c.cadence }); continue; }
  const ageS = (now - sig.mtime) / 1000;
  // DEGRADED → stricter thresholds (×0.8) per Codex refinement
  const k = sig.confidence === 'degraded' ? 0.8 : 1.0;
  const warnAfter = c.warn_after * k, failAfter = c.fail_after * k;
  // "ran but wrote nothing" false-OK guard: recent BUT empty launchd log
  const emptySuspect = sig.source === 'launchd_log' && sig.size === 0 && ageS < failAfter;
  let verdict = 'OK', reason = `fresh via ${sig.source}`;
  if (emptySuspect) { verdict = 'WARN'; reason = `recent launchd log is 0-byte — ran but produced no output (the silent-death class)`; }
  if (ageS >= failAfter) { verdict = 'FAIL'; reason = `stale ${(ageS / 3600).toFixed(1)}h (fail≥${(failAfter / 3600).toFixed(1)}h) via ${sig.source}`; }
  else if (ageS >= warnAfter) { verdict = verdict === 'WARN' ? 'WARN' : 'WARN'; reason = `aging ${(ageS / 3600).toFixed(1)}h (warn≥${(warnAfter / 3600).toFixed(1)}h) via ${sig.source}`; }
  results.push({ label: c.label, verdict, reason, confidence: sig.confidence, source: sig.source, age_h: +(ageS / 3600).toFixed(2), cadence: c.cadence });
}

const fail = results.filter(r => r.verdict === 'FAIL');
const warn = results.filter(r => r.verdict === 'WARN');
const degraded = results.filter(r => r.confidence === 'degraded').map(r => r.label);
const summary = { ts: new Date(now).toISOString(), total: results.length, ok: results.filter(r => r.verdict === 'OK').length, warn: warn.length, fail: fail.length, degraded_monitoring: degraded, results };
fs.writeFileSync(path.join(DATA, 'latest.json'), JSON.stringify(summary, null, 2));
log(`watchdog: ${summary.total} canaries · ${summary.ok} OK · ${warn.length} WARN · ${fail.length} FAIL${degraded.length ? ` · degraded-monitoring: ${degraded.join(',')}` : ''}`);
for (const r of [...fail, ...warn]) log(`  ${r.verdict} ${r.label} — ${r.reason}`);

// self-heartbeat (so the watchdog isn't an unmonitored SPOF)
try { fs.writeFileSync(exp('~/.claude/heartbeats/dw-canary-watchdog.stamp'), String(now)); } catch {}

async function postCNCP(note, url) {
  const base = ENV.CNCP_URL || 'http://localhost:3333';
  try { const ac = new AbortController(); const t = setTimeout(() => ac.abort(), 8000);
    const r = await fetch(`${base}/api/parking-lot`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url, note }), signal: ac.signal });
    clearTimeout(t); return r.ok; } catch { return false; }
}
async function emailGeorge(subject, html) {
  let auth = ENV.GEORGE_AUTH;
  if (!auth) { try { const env = fs.readFileSync(path.join(os.homedir(), 'Projects/secrets-manager/.env'), 'utf8'); auth = (env.match(/^GEORGE_AUTH=(.*)$/m) || [])[1]; } catch {} }
  if (!auth) { log('email skipped (no GEORGE_AUTH)'); return false; }
  // George flipped to HTTPS-only 2026-06-17 — raw http://<ip> now 400s and the IP
  // has no valid cert; default to the Tailscale MagicDNS host (overridable).
  const base = ENV.GEORGE_URL || 'https://kamatera.tail79cb8e.ts.net:9850';
  try { const ac = new AbortController(); const t = setTimeout(() => ac.abort(), 15000);
    const r = await fetch(`${base}/api/send`, { method: 'POST', headers: { 'Authorization': auth, 'Content-Type': 'application/json; charset=utf-8' },
      body: JSON.stringify({ to: ENV.WATCHDOG_TO || 'steve@designerwallcoverings.com', subject, body: html }), signal: ac.signal });
    clearTimeout(t); const j = await r.json().catch(() => ({})); return !!j.success; } catch { return false; }
}

const beat = `[CANARY WATCHDOG ${new Date(now).toISOString().slice(0, 16).replace('T', ' ')}] alive · ${summary.ok}/${summary.total} OK${fail.length ? ` · ${fail.length} FAIL: ${fail.map(f => f.label).join(', ')}` : ''}${warn.length ? ` · ${warn.length} WARN` : ''}`;

if (fail.length) {
  const note = `[CANARY WATCHDOG ${new Date(now).toISOString().slice(0, 10)}] ${fail.length} canary(ies) NOT running: ${fail.map(f => `${f.label} (${f.reason})`).join('; ')}. A blind canary = a silent failure waiting to happen.`;
  await postCNCP(note, 'file://' + SKILL);
  const html = `<div style="font-family:-apple-system,sans-serif;color:#222"><h3 style="color:#b23b3b">⚠ Canary Watchdog — ${fail.length} canary(ies) not running</h3>`
    + fail.map(f => `<div style="margin:6px 0"><b>${f.label}</b> — ${f.reason} <span style="color:#888">(${f.cadence})</span></div>`).join('')
    + (warn.length ? `<hr><div style="color:#a07000">WARN: ${warn.map(w => w.label + ' — ' + w.reason).join('; ')}</div>` : '')
    + `</div>`;
  const emailed = await emailGeorge(`⚠ Canary Watchdog — ${fail.length} canary down — ${new Date(now).toISOString().slice(0, 10)}`, html);
  log(emailed ? 'FAIL alert emailed' : 'FAIL alert email not sent');
  process.exitCode = 1;
} else {
  // healthy → post a lightweight alive-beat so Steve can eyeball the watchdog itself on CNCP
  if (ENV.WATCHDOG_BEAT !== '0') { const ok = await postCNCP(beat, 'file://' + SKILL); log(ok ? 'CNCP alive-beat posted' : 'CNCP alive-beat failed'); }
  log('✓ all canaries running' + (warn.length ? ` (${warn.length} aging — logged, not alerted)` : ''));
}