[object Object]

← back to Dw Yolo Loop

harden 3 read-only canaries (DTD-A 3/3): dynamic-dated reports + best-effort CNCP alert on FAIL

3343eea8fe5e0fd550e08a4dd624e224e287d4c6 · 2026-06-16 07:47:47 -0700 · Steve Abrams

- Fix hardcoded 2026-06-16 output filename → dynamic ${today} so a scheduled
  run can't clobber a prior day's report or stamp a stale date.
- Wire a timeout-bounded, never-throws CNCP parking-lot alert on FAIL (these
  canaries previously only console.log'd — a 3am FAIL was silently written to
  a JSON nobody reads).
- queue-slo alert gated on !STALE && live-backup so the by-design Mac2 frozen-
  mirror STALE doesn't cry-wolf every run.
Smoke-tested tls-expiry: PASS, 15 hosts, dynamic date confirmed.

Files touched

Diff

commit 3343eea8fe5e0fd550e08a4dd624e224e287d4c6
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jun 16 07:47:47 2026 -0700

    harden 3 read-only canaries (DTD-A 3/3): dynamic-dated reports + best-effort CNCP alert on FAIL
    
    - Fix hardcoded 2026-06-16 output filename → dynamic ${today} so a scheduled
      run can't clobber a prior day's report or stamp a stale date.
    - Wire a timeout-bounded, never-throws CNCP parking-lot alert on FAIL (these
      canaries previously only console.log'd — a 3am FAIL was silently written to
      a JSON nobody reads).
    - queue-slo alert gated on !STALE && live-backup so the by-design Mac2 frozen-
      mirror STALE doesn't cry-wolf every run.
    Smoke-tested tls-expiry: PASS, 15 hosts, dynamic date confirmed.
---
 scripts/handle-freshness/handle-freshness-canary.mjs | 15 +++++++++++++--
 scripts/queue-slo-snapshot/queue-slo-snapshot.mjs    | 18 ++++++++++++++++--
 scripts/tls-expiry/tls-expiry-canary.mjs             | 16 ++++++++++++++--
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/scripts/handle-freshness/handle-freshness-canary.mjs b/scripts/handle-freshness/handle-freshness-canary.mjs
index e8843d6..835c9fb 100644
--- a/scripts/handle-freshness/handle-freshness-canary.mjs
+++ b/scripts/handle-freshness/handle-freshness-canary.mjs
@@ -24,8 +24,9 @@ const N = parseInt(args.find((_,i,a)=>a[i-1]==='--n') || '300', 10) || 300;
 const PER_VENDOR = parseInt(args.find((_,i,a)=>a[i-1]==='--per-vendor') || '0', 10) || 0; // 0 = flat random across all
 const WARN = parseFloat(args.find((_,i,a)=>a[i-1]==='--warn') || '5') || 5;   // %
 const FAIL = parseFloat(args.find((_,i,a)=>a[i-1]==='--fail') || '15') || 15; // %
-const OUT = `${process.env.HOME}/.claude/yolo-queue/handle-freshness-2026-06-16.json`;
-const MD  = `${process.env.HOME}/.claude/yolo-queue/handle-freshness-2026-06-16.md`;
+const today = new Date().toISOString().slice(0,10); // dynamic — a scheduled run must not clobber a prior day's report
+const OUT = `${process.env.HOME}/.claude/yolo-queue/handle-freshness-${today}.json`;
+const MD  = `${process.env.HOME}/.claude/yolo-queue/handle-freshness-${today}.md`;
 
 function q(sql) {
   const out = execFileSync(PSQL, [DB, '-At', '-F', '|', '-c', sql], { encoding: 'utf8', maxBuffer: 64*1024*1024 });
@@ -33,6 +34,15 @@ function q(sql) {
 }
 const sleep = (ms) => new Promise(r => setTimeout(r, ms));
 
+// best-effort local CNCP alert — timeout-bounded, NEVER throws (canary must still finish).
+async function alertCNCP(note) {
+  try { const ac = new AbortController(); const t = setTimeout(()=>ac.abort(), 6000);
+    const r = await fetch(`${process.env.CNCP_URL||'http://localhost:3333'}/api/parking-lot`,
+      { method:'POST', headers:{'Content-Type':'application/json'},
+        body: JSON.stringify({ url:'file://handle-freshness-canary', note }), signal: ac.signal });
+    clearTimeout(t); return r.ok; } catch { return false; }
+}
+
 // stratified or flat sample. random() ordering = representative draw (not oldest-biased like cycle 21's first run).
 let rows;
 if (PER_VENDOR > 0) {
@@ -95,5 +105,6 @@ async function check(p) {
 
   console.log(`[handle-freshness] ${emoji} ${verdict} · drift=${driftPct.toFixed(1)}% ±${ci.toFixed(1)} · OK=${ok} STALE=${stale.length} UNK=${unknown} (n=${results.length})`);
   console.log(`Report: ${MD}`);
+  if (verdict === 'FAIL') await alertCNCP(`[HANDLE-FRESHNESS 🔴 FAIL] storefront drift ${driftPct.toFixed(1)}% (${stale.length}/${decided}) — mirror-ACTIVE handles returning 404; shoppers can't reach them`);
   process.exit(verdict==='FAIL'?2:0);
 })().catch(e => { console.error('FATAL', e.message); process.exit(1); });
diff --git a/scripts/queue-slo-snapshot/queue-slo-snapshot.mjs b/scripts/queue-slo-snapshot/queue-slo-snapshot.mjs
index 5e764c7..6864be2 100644
--- a/scripts/queue-slo-snapshot/queue-slo-snapshot.mjs
+++ b/scripts/queue-slo-snapshot/queue-slo-snapshot.mjs
@@ -24,8 +24,9 @@ const PSQL = [ '/opt/homebrew/opt/postgresql@14/bin/psql', '/usr/local/opt/postg
 const DB = process.env.DW_UNIFIED_URL || 'postgresql:///dw_unified?host=/tmp';
 const args = process.argv.slice(2);
 const MAX_STALE_H = parseFloat(args.find((_,i,a)=>a[i-1]==='--max-stale-hours') || '6') || 6;
-const OUT = `${process.env.HOME}/.claude/yolo-queue/queue-slo-snapshot-2026-06-16.json`;
-const MD  = `${process.env.HOME}/.claude/yolo-queue/queue-slo-snapshot-2026-06-16.md`;
+const today = new Date().toISOString().slice(0,10); // dynamic — a scheduled run must not clobber a prior day's report
+const OUT = `${process.env.HOME}/.claude/yolo-queue/queue-slo-snapshot-${today}.json`;
+const MD  = `${process.env.HOME}/.claude/yolo-queue/queue-slo-snapshot-${today}.md`;
 
 function q(sql) {
   const out = execFileSync(PSQL, [DB, '-At', '-F', '|', '-c', sql], { encoding: 'utf8', maxBuffer: 64*1024*1024 });
@@ -78,4 +79,17 @@ fs.writeFileSync(MD, md);
 
 console.log(`[queue-slo] ${STALE?'🔴 STALE':'🟢 LIVE'} · last_completed=${lastCompleted} · tput24=${tput24} · pending=${report.by_status.pending||0}`);
 console.log(`Report: ${MD}`);
+
+// Alert ONLY on a genuine LIVE backup/error — NEVER on the by-design Mac2 frozen-mirror
+// STALE (that's expected here and would cry-wolf nightly). best-effort, never throws.
+const liveBackup = !STALE && ((report.oldest_pending_age_h ?? 0) > 12 || report.errors.exhausted > 0);
+if (liveBackup) {
+  try { const ac = new AbortController(); const t = setTimeout(()=>ac.abort(), 6000);
+    await fetch(`${process.env.CNCP_URL||'http://localhost:3333'}/api/parking-lot`,
+      { method:'POST', headers:{'Content-Type':'application/json'},
+        body: JSON.stringify({ url:'file://queue-slo-snapshot',
+          note:`[QUEUE-SLO 🔴] LIVE write-queue backed up: oldest_pending=${report.oldest_pending_age_h?.toFixed(1)}h · retry-exhausted=${report.errors.exhausted} · pending=${report.by_status.pending||0}` }),
+        signal: ac.signal });
+    clearTimeout(t); } catch { /* dashboard down — report still written */ }
+}
 process.exit(STALE ? 2 : 0);
diff --git a/scripts/tls-expiry/tls-expiry-canary.mjs b/scripts/tls-expiry/tls-expiry-canary.mjs
index d391ee0..49173ae 100644
--- a/scripts/tls-expiry/tls-expiry-canary.mjs
+++ b/scripts/tls-expiry/tls-expiry-canary.mjs
@@ -21,8 +21,9 @@ const HOSTS = (args.find((_,i,a)=>a[i-1]==='--hosts') || [
   'corkwallcovering.com','silkwallpaper.com','linenwallpaper.com','grasscloth.com',
   'raffiawallpaper.com','glitterwallpaper.com','hospitalitywallcoverings.com',
 ].join(',')).split(',').map(h=>h.trim()).filter(Boolean);
-const OUT = `${process.env.HOME}/.claude/yolo-queue/tls-expiry-2026-06-16.json`;
-const MD  = `${process.env.HOME}/.claude/yolo-queue/tls-expiry-2026-06-16.md`;
+const today = new Date().toISOString().slice(0,10); // dynamic — a scheduled run must not clobber a prior day's report
+const OUT = `${process.env.HOME}/.claude/yolo-queue/tls-expiry-${today}.json`;
+const MD  = `${process.env.HOME}/.claude/yolo-queue/tls-expiry-${today}.md`;
 
 function checkHost(host) {
   return new Promise((resolve) => {
@@ -41,6 +42,16 @@ function checkHost(host) {
   });
 }
 
+// best-effort local CNCP alert — timeout-bounded, NEVER throws (the canary must
+// still finish + write its report even if the dashboard is down/slow).
+async function alertCNCP(note) {
+  try { const ac = new AbortController(); const t = setTimeout(()=>ac.abort(), 6000);
+    const r = await fetch(`${process.env.CNCP_URL||'http://localhost:3333'}/api/parking-lot`,
+      { method:'POST', headers:{'Content-Type':'application/json'},
+        body: JSON.stringify({ url:'file://tls-expiry-canary', note }), signal: ac.signal });
+    clearTimeout(t); return r.ok; } catch { return false; }
+}
+
 (async () => {
   const results = [];
   for (const h of HOSTS) results.push(await checkHost(h));
@@ -73,5 +84,6 @@ function checkHost(host) {
   console.log(`[tls-expiry] ${emoji} ${verdict} · checked=${results.length} · expired=${expired.length} crit=${critical.length} warn=${warning.length} unreachable=${unreachable.length}`);
   for (const r of [...expired,...critical,...warning]) console.log(`  ⚠️  ${r.host}: ${r.days_to_expiry}d`);
   console.log(`Report: ${MD}`);
+  if (verdict === 'FAIL') await alertCNCP(`[TLS-EXPIRY 🔴 FAIL] ${[...expired,...critical].map(r=>`${r.host}(${r.days_to_expiry}d)`).join(', ')} — cert expired/expiring; renew now`);
   process.exit(verdict==='FAIL'?2:0);
 })().catch(e => { console.error('FATAL', e.message); process.exit(1); });

← 587c4e4 tls-expiry canary: read-only fleet cert-expiry check (DTD 3/  ·  back to Dw Yolo Loop  ·  metafield-coverage audit: repped-vendor specs/images/color c acaeef7 →