← back to Dw Yolo Loop
Canary Meta-Watchdog: heartbeat verifier for com.steve.* jobs (read-only)
36a2b0fbb8b4961ef59110856378fcfc52fc329c · 2026-06-15 23:26:12 -0700 · Steve Abrams
Proves the canaries themselves ran; first run found 15 hard-signal FAILs incl
NEW silent deaths: dw-uptime-probe + yolo-watchdog (the watchers themselves
down), contact-mailer-daily (~20d), dw-price-coverage (4.5d), wallco-generator.
Plist staged (NOT bootstrapped). Closes the blind spot that hid the 12-day
pg_dump death.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Files touched
A scripts/canary-meta-watchdog/canary-meta-watchdog.jsA scripts/canary-meta-watchdog/com.steve.canary-meta-watchdog.plist
Diff
commit 36a2b0fbb8b4961ef59110856378fcfc52fc329c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jun 15 23:26:12 2026 -0700
Canary Meta-Watchdog: heartbeat verifier for com.steve.* jobs (read-only)
Proves the canaries themselves ran; first run found 15 hard-signal FAILs incl
NEW silent deaths: dw-uptime-probe + yolo-watchdog (the watchers themselves
down), contact-mailer-daily (~20d), dw-price-coverage (4.5d), wallco-generator.
Plist staged (NOT bootstrapped). Closes the blind spot that hid the 12-day
pg_dump death.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
.../canary-meta-watchdog/canary-meta-watchdog.js | 134 +++++++++++++++++++++
.../com.steve.canary-meta-watchdog.plist | 31 +++++
2 files changed, 165 insertions(+)
diff --git a/scripts/canary-meta-watchdog/canary-meta-watchdog.js b/scripts/canary-meta-watchdog/canary-meta-watchdog.js
new file mode 100644
index 0000000..613a6bb
--- /dev/null
+++ b/scripts/canary-meta-watchdog/canary-meta-watchdog.js
@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+/**
+ * Canary Meta-Watchdog — proves the canaries themselves actually RAN.
+ *
+ * The fleet's worst failures are silent monitoring deaths: the dw_unified
+ * pg_dump wrote 0 bytes for 12 days unnoticed, the Kamatera health-watchdog
+ * went down, and ≥6 launchd jobs are failing — each a case of "the thing that
+ * should have warned us is the thing that broke." This meta-watchdog reads
+ * every com.steve.* LaunchAgent, derives its expected cadence, and FAILs
+ * (report-only) if a job is unloaded, last-exited non-zero, or hasn't produced
+ * fresh log output within its expected interval (× a grace factor).
+ *
+ * READ-ONLY: parses plists + stats log files + reads `launchctl print`. Writes
+ * only a local report JSON. No prod writes, no network. Cost: $0 (local).
+ *
+ * node canary-meta-watchdog.js # human table + write report JSON
+ * node canary-meta-watchdog.js --json # machine JSON to stdout
+ */
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const HOME = process.env.HOME;
+const LA_DIR = path.join(HOME, 'Library/LaunchAgents');
+const UID = parseInt(execSync('id -u').toString().trim(), 10);
+const GRACE = 1.5; // allow 1.5× the expected interval before flagging stale
+const OUT = path.join(HOME, '.claude/yolo-queue', `canary-meta-watchdog-${new Date().toISOString().slice(0,10)}.json`);
+const JSON_ONLY = process.argv.includes('--json');
+
+const now = Date.now();
+const sec = (ms) => ms / 1000;
+
+function plistToObj(file) {
+ try { return JSON.parse(execSync(`plutil -convert json -o - ${JSON.stringify(file)}`, { stdio: ['ignore','pipe','ignore'] }).toString()); }
+ catch (e) { return null; }
+}
+
+// Expected period in seconds from StartInterval or StartCalendarInterval.
+function expectedPeriodSec(p) {
+ if (typeof p.StartInterval === 'number') return p.StartInterval;
+ const sci = p.StartCalendarInterval;
+ if (sci) {
+ const entries = Array.isArray(sci) ? sci : [sci];
+ // crude cadence inference: Minute-only => hourly; Hour set => daily; Weekday set => weekly; Day set => monthly
+ const e = entries[0] || {};
+ if ('Weekday' in e) return 7 * 86400;
+ if ('Day' in e) return 30 * 86400;
+ if ('Hour' in e) return 86400; // runs at a fixed hour daily
+ if ('Minute' in e) return 3600; // runs each hour at a minute
+ return 86400;
+ }
+ return null; // RunAtLoad-only / on-demand — no cadence to check
+}
+
+// Last-run proxy = newest mtime among the job's stdout/stderr log paths.
+function lastRunMs(p) {
+ const paths = [p.StandardOutPath, p.StandardErrorPath].filter(Boolean);
+ let newest = 0;
+ for (const f of paths) {
+ try { const st = fs.statSync(f); newest = Math.max(newest, st.mtimeMs); } catch (_) {}
+ }
+ return newest || null;
+}
+
+// launchctl state: loaded? last exit code? pid?
+function launchctlState(label) {
+ try {
+ const out = execSync(`launchctl print gui/${UID}/${label} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString();
+ const pid = (out.match(/pid = (\d+)/) || [])[1] || null;
+ const lastExit = (out.match(/last exit code = (\d+)/) || [])[1];
+ return { loaded: true, pid, lastExit: lastExit !== undefined ? parseInt(lastExit, 10) : null };
+ } catch (e) {
+ // fall back to `launchctl list` membership
+ try {
+ const line = execSync(`launchctl list | grep ${JSON.stringify(label)} 2>/dev/null`, { stdio: ['ignore','pipe','ignore'] }).toString().trim();
+ if (!line) return { loaded: false, pid: null, lastExit: null };
+ const cols = line.split(/\s+/);
+ return { loaded: true, pid: cols[0] === '-' ? null : cols[0], lastExit: cols[1] === '-' ? null : parseInt(cols[1], 10) };
+ } catch (_) { return { loaded: false, pid: null, lastExit: null }; }
+ }
+}
+
+const files = fs.readdirSync(LA_DIR).filter(f => /^com\.steve\..*\.plist$/.test(f));
+const results = [];
+
+for (const f of files) {
+ const full = path.join(LA_DIR, f);
+ const p = plistToObj(full);
+ if (!p || !p.Label) { results.push({ label: f.replace(/\.plist$/, ''), verdict: 'WARN', why: 'unparseable plist' }); continue; }
+ const label = p.Label;
+ const period = expectedPeriodSec(p);
+ const last = lastRunMs(p);
+ const st = launchctlState(label);
+ const ageSec = last ? sec(now - last) : null;
+
+ let verdict = 'PASS', why = [];
+ if (!st.loaded) { verdict = 'FAIL'; why.push('NOT LOADED in launchd'); }
+ if (st.lastExit && st.lastExit !== 0) { verdict = 'FAIL'; why.push(`last exit = ${st.lastExit}`); }
+ if (period && ageSec !== null) {
+ if (ageSec > period * GRACE) { verdict = 'FAIL'; why.push(`STALE: last log ${(ageSec/3600).toFixed(1)}h ago vs ${(period/3600).toFixed(1)}h cadence`); }
+ } else if (period && last === null) {
+ if (verdict === 'PASS') verdict = 'WARN'; why.push('no log output found (never ran or logs elsewhere)');
+ } else if (!period) {
+ if (verdict === 'PASS') verdict = 'INFO'; why.push('on-demand/RunAtLoad — no cadence to check');
+ }
+ results.push({
+ label, verdict,
+ cadence_h: period ? +(period/3600).toFixed(1) : null,
+ last_run_age_h: ageSec !== null ? +(ageSec/3600).toFixed(1) : null,
+ loaded: st.loaded, last_exit: st.lastExit, pid: st.pid,
+ why: why.join('; ') || 'ok',
+ });
+}
+
+const rank = { FAIL: 0, WARN: 1, INFO: 2, PASS: 3 };
+results.sort((a, b) => (rank[a.verdict] - rank[b.verdict]) || a.label.localeCompare(b.label));
+const summary = results.reduce((m, r) => (m[r.verdict] = (m[r.verdict]||0)+1, m), {});
+const report = { generated_at: new Date().toISOString(), launch_agents_dir: LA_DIR, grace_factor: GRACE, total: results.length, summary, results };
+
+fs.writeFileSync(OUT, JSON.stringify(report, null, 2));
+
+if (JSON_ONLY) { console.log(JSON.stringify(report, null, 2)); process.exit(0); }
+
+console.log(`\nCanary Meta-Watchdog — ${results.length} com.steve.* jobs (${new Date().toISOString().slice(0,16)})`);
+console.log(`Summary: ${Object.entries(summary).map(([k,v])=>`${k}=${v}`).join(' ')}`);
+console.log('─'.repeat(96));
+for (const r of results) {
+ const tag = { FAIL:'🔴', WARN:'🟠', INFO:'⚪', PASS:'🟢' }[r.verdict];
+ console.log(`${tag} ${r.verdict.padEnd(4)} ${r.label.padEnd(42)} ${r.why}`);
+}
+console.log('─'.repeat(96));
+console.log(`Report: ${OUT}`);
+const fails = results.filter(r => r.verdict === 'FAIL');
+if (fails.length) console.log(`\n⚠️ ${fails.length} canary FAIL(s) — these monitors are not provably running.`);
diff --git a/scripts/canary-meta-watchdog/com.steve.canary-meta-watchdog.plist b/scripts/canary-meta-watchdog/com.steve.canary-meta-watchdog.plist
new file mode 100644
index 0000000..46617dd
--- /dev/null
+++ b/scripts/canary-meta-watchdog/com.steve.canary-meta-watchdog.plist
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!-- STAGED — NOT installed. Enable with:
+ launchctl bootstrap gui/$(id -u) scripts/canary-meta-watchdog/com.steve.canary-meta-watchdog.plist
+ Runs the meta-watchdog daily at 06:30 (after the overnight jobs settle). -->
+<plist version="1.0">
+<dict>
+ <key>Label</key>
+ <string>com.steve.canary-meta-watchdog</string>
+ <key>ProgramArguments</key>
+ <array>
+ <string>/usr/local/bin/node</string>
+ <string>/Users/stevestudio2/Projects/designerwallcoverings/scripts/canary-meta-watchdog/canary-meta-watchdog.js</string>
+ </array>
+ <key>WorkingDirectory</key>
+ <string>/Users/stevestudio2/Projects/designerwallcoverings/scripts/canary-meta-watchdog</string>
+ <key>StartCalendarInterval</key>
+ <dict>
+ <key>Hour</key>
+ <integer>6</integer>
+ <key>Minute</key>
+ <integer>30</integer>
+ </dict>
+ <key>StandardOutPath</key>
+ <string>/Users/stevestudio2/Projects/designerwallcoverings/scripts/canary-meta-watchdog/cron.out.log</string>
+ <key>StandardErrorPath</key>
+ <string>/Users/stevestudio2/Projects/designerwallcoverings/scripts/canary-meta-watchdog/cron.err.log</string>
+ <key>RunAtLoad</key>
+ <false/>
+</dict>
+</plist>
← f5311b1 chore: gitignore runtime artifacts + un-track gmc-425-result
·
back to Dw Yolo Loop
·
Add blocked-vendor cost-coverage ledger (2026-06-15): 7 vend d6bb2f3 →