← back to Dw Yolo Loop
cycle 77: harden canary per officer REVISE — abort-on-partial-crawl, confirm-before-page (re-run), freshness re-based to >72h stale-guard, upward-only baseline ratchet, fileURLToPath, data/latest.json heartbeat; re-verified healthy + crater-through-confirm-rerun
213fd6005b84452935ee9fba04332aa9390e816a · 2026-06-18 00:12:01 -0700 · Steve Abrams
Files touched
A scripts/sitemap-integrity-canary/.gitignoreM scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
Diff
commit 213fd6005b84452935ee9fba04332aa9390e816a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 18 00:12:01 2026 -0700
cycle 77: harden canary per officer REVISE — abort-on-partial-crawl, confirm-before-page (re-run), freshness re-based to >72h stale-guard, upward-only baseline ratchet, fileURLToPath, data/latest.json heartbeat; re-verified healthy + crater-through-confirm-rerun
---
scripts/sitemap-integrity-canary/.gitignore | 1 +
.../sitemap-integrity-canary.mjs | 145 +++++++++++++--------
2 files changed, 91 insertions(+), 55 deletions(-)
diff --git a/scripts/sitemap-integrity-canary/.gitignore b/scripts/sitemap-integrity-canary/.gitignore
new file mode 100644
index 0000000..d9e6a75
--- /dev/null
+++ b/scripts/sitemap-integrity-canary/.gitignore
@@ -0,0 +1 @@
+data/latest.json
diff --git a/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs b/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
index 47ab79e..3a3918f 100644
--- a/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
+++ b/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
@@ -1,72 +1,107 @@
-// sitemap-integrity-canary — READ-ONLY, $0. Cycle 77 (DTD-picked A, unanimous).
+// sitemap-integrity-canary — READ-ONLY, $0. Cycle 77 (DTD-A) + c77 officer REVISE hardening.
// A DURABLE standing tripwire for the catastrophe class the existing canaries miss:
// the April-2026 mass-archive (~39.7k products archived in one day, ~50k+ sellable dark
-// for 2 months, 0 restored). That event would have surfaced here as a SITEMAP COLLAPSE
-// (product-URL count craters). This canary asserts, every run:
-// (1) sitemap index returns 200 + parses;
-// (2) every child product sitemap returns 200;
-// (3) total product-URL count vs a stored BASELINE doesn't crater (> CRATER_PCT drop);
-// (4) regeneration hasn't stalled (newest child <lastmod> within FRESH_HOURS).
-// First run with no baseline ESTABLISHES it (writes baseline.json) + reports HEALTHY.
-// NEGATIVE CONTROL: re-derives today's count live + compares to the baseline mechanism.
-// Distinct from the c76 always-green lastmod check — this guards 200/count/staleness.
-// EXCLUDES Phillip Jeffries N/A (sitemap-level count). Read-only; never mutates.
+// for 2 months, 0 restored) — which would surface here as a SITEMAP product-URL COLLAPSE.
+// Asserts each run: (1) index 200+parses; (2) every en product child 200; (3) total
+// product-URL count vs a stored baseline doesn't crater (>CRATER_PCT) — ONLY on a complete
+// all-200 crawl; (4) endpoint isn't serving an implausibly-stale cached doc (>STALE_HOURS).
+//
+// c77-officer hardening (false-positive defense for a standing canary):
+// - retry 5xx (not just 429); ABORT the crater comparison if ANY child errored (partial
+// crawl => untrustworthy sum => report reachability only, never a phantom crater);
+// - CONFIRM-BEFORE-PAGE: on any alert, re-run once after CONFIRM_DELAY; emit ALERT only if
+// the second run also alerts (a single network blip won't page);
+// - freshness re-based: Shopify rewrites all <lastmod> to ~now every regen (the c76 finding),
+// so a 24h check is dead/always-green; keep ONLY an implausible >STALE_HOURS guard
+// (would mean the endpoint is serving a stale cached doc — a real pathology);
+// - baseline UPWARD-ONLY ratchet on healthy runs (catalog grows; never lower the floor) +
+// rolling history; fileURLToPath for BASE_DIR (path-with-spaces safe); data/latest.json
+// heartbeat so dw-canary-meta-watchdog can prove it ran.
+// NEGATIVE CONTROL: re-derives count live + (test) fires on an inflated baseline. Read-only.
import fs from 'fs';
import path from 'path';
+import { fileURLToPath } from 'url';
const WWW='https://www.designerwallcoverings.com';
const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
-const CRATER_PCT=10; // alert if product count drops >10% vs baseline (April was ~54%)
-const FRESH_HOURS=24; // alert if newest child lastmod older than 24h (regeneration stalled)
-const BASE_DIR=path.dirname(new URL(import.meta.url).pathname);
+const CRATER_PCT=Number(process.env.CRATER_PCT||10); // April was ~54%; 10% catches w/ ~5x margin
+const STALE_HOURS=Number(process.env.STALE_HOURS||72); // implausible-stale cached-doc guard only
+const CONFIRM_DELAY=Number(process.env.CONFIRM_DELAY_MS||60000); // re-run gap before paging
+const BASE_DIR=path.dirname(fileURLToPath(import.meta.url));
+const DATA_DIR=path.join(BASE_DIR,'data');
const BASELINE=path.join(BASE_DIR,'baseline.json');
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
-async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA}});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,text:await r.text()};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
+async function get(u){for(let a=0;a<5;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA}});if(r.status===429||r.status>=500){await sleep(1500*(a+1));continue;}return {status:r.status,text:await r.text()};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
-const alerts=[];
-console.log(`=== sitemap-integrity-canary (READ-ONLY, $0) ===`);
-
-// (1) index
-const idx=await get(`${WWW}/sitemap.xml`);
-if(idx.status!==200){ alerts.push(`INDEX non-200: ${idx.status}`); }
-const allChildren=[...idx.text.matchAll(/<loc>([^<]+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&/g,'&'));
-const prodChildren=allChildren.filter(u=>/sitemap_products_\d+\.xml/.test(u) && !/\/en-(ca|gb)\//.test(u));
-console.log(`index: ${idx.status} | total child sitemaps: ${allChildren.length} | en product children: ${prodChildren.length}`);
+// One full crawl → returns measurement (no paging decision here).
+async function crawl(){
+ const idx=await get(`${WWW}/sitemap.xml`);
+ const reach=[]; if(idx.status!==200) reach.push(`INDEX non-200: ${idx.status}`);
+ const prodChildren=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
+ let total=0, badChild=0, newest=0;
+ for(const c of prodChildren){
+ const x=await get(c);
+ if(x.status!==200){ badChild++; reach.push(`CHILD non-200 (${x.status}): ${c.split('/').pop().split('?')[0]}`); continue; }
+ total+=[...x.text.matchAll(/<loc>[^<]*\/products\/[^<]+<\/loc>/g)].length;
+ const lm=[...x.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>Date.parse(m[1])).filter(n=>!isNaN(n));
+ if(lm.length) newest=Math.max(newest, ...lm);
+ await sleep(120);
+ }
+ const ageH = newest? (Date.now()-newest)/3.6e6 : null;
+ return {indexStatus:idx.status, childCount:prodChildren.length, total, badChild, ageH, reach};
+}
-// (2)+(3) per child: 200 + count product locs; (4) freshness
-let total=0, badChild=0, newest=0;
-for(const c of prodChildren){
- const x=await get(c);
- if(x.status!==200){ badChild++; alerts.push(`CHILD non-200 (${x.status}): ${c.split('/').pop().split('?')[0]}`); continue; }
- total+=[...x.text.matchAll(/<loc>[^<]*\/products\/[^<]+<\/loc>/g)].length;
- const lm=[...x.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>Date.parse(m[1])).filter(n=>!isNaN(n));
- if(lm.length) newest=Math.max(newest, ...lm);
- await sleep(120);
+// Evaluate a crawl → alerts[]. Crater comparison ONLY on a complete all-200 crawl.
+function evaluate(m, baseline){
+ const alerts=[...m.reach];
+ const complete = m.indexStatus===200 && m.badChild===0 && m.childCount>0;
+ if(!complete){
+ alerts.push(`PARTIAL/UNREACHABLE CRAWL (index ${m.indexStatus}, ${m.badChild} bad children) — crater check SKIPPED (cannot trust partial sum)`);
+ } else if(baseline && baseline.productCount){
+ const dropPct = 100*(baseline.productCount-m.total)/baseline.productCount;
+ if(dropPct>CRATER_PCT) alerts.push(`PRODUCT-COUNT CRATER: ${dropPct.toFixed(1)}% drop (${baseline.productCount}->${m.total}) > ${CRATER_PCT}% — possible mass-archive/sitemap collapse`);
+ }
+ if(m.ageH!=null && m.ageH>STALE_HOURS) alerts.push(`STALE SITEMAP: newest lastmod ${m.ageH.toFixed(1)}h > ${STALE_HOURS}h (endpoint serving a stale cached doc?)`);
+ return {alerts, complete};
}
-console.log(`product-URL count (en): ${total} | bad children: ${badChild}`);
-const ageH = newest? (Date.now()-newest)/3.6e6 : null;
-console.log(`newest child lastmod age: ${ageH==null?'(none)':ageH.toFixed(1)+'h'} (stall threshold ${FRESH_HOURS}h)`);
-if(ageH!=null && ageH>FRESH_HOURS) alerts.push(`REGENERATION STALLED: newest lastmod ${ageH.toFixed(1)}h > ${FRESH_HOURS}h`);
-// baseline compare / establish
+console.log(`=== sitemap-integrity-canary (READ-ONLY, $0) — hardened ===`);
let baseline=null; try{ baseline=JSON.parse(fs.readFileSync(BASELINE,'utf8')); }catch{}
-if(!baseline || !baseline.productCount){
- // NOTE: import.meta.url pathname is fine for read; for write use the resolved path
- const seed={establishedFromCount:total, productCount:total, childCount:prodChildren.length, note:'baseline seeded cycle 77; April-mass-archive crater tripwire', crater_pct:CRATER_PCT};
- try{ fs.writeFileSync(BASELINE, JSON.stringify(seed,null,2)); console.log(`\nNO BASELINE → ESTABLISHED baseline.json @ productCount=${total}, childCount=${prodChildren.length}`); }
- catch(e){ console.log('baseline write err',e.message); }
-} else {
- const dropPct = baseline.productCount? (100*(baseline.productCount-total)/baseline.productCount) : 0;
- console.log(`\nbaseline productCount=${baseline.productCount} → current=${total} → drop ${dropPct.toFixed(1)}% (crater threshold ${CRATER_PCT}%)`);
- if(dropPct>CRATER_PCT) alerts.push(`PRODUCT-COUNT CRATER: ${dropPct.toFixed(1)}% drop (${baseline.productCount}→${total}) > ${CRATER_PCT}% — possible mass-archive/sitemap collapse`);
- // also flag a >2x SURGE (could signal a bad bulk-publish) — informational
- if(total > baseline.productCount*1.5) console.log(` (note: count surged >50% vs baseline — possible bulk (re)publish, informational)`);
+
+let m=await crawl();
+let {alerts, complete}=evaluate(m, baseline);
+console.log(`run1: index ${m.indexStatus} | children ${m.childCount} | product-URLs ${m.total} | bad ${m.badChild} | lastmod age ${m.ageH==null?'n/a':m.ageH.toFixed(1)+'h'}`);
+if(baseline?.productCount) console.log(` baseline ${baseline.productCount} → drop ${(100*(baseline.productCount-m.total)/baseline.productCount).toFixed(1)}% (crater ${CRATER_PCT}%)`);
+
+// CONFIRM-BEFORE-PAGE: a single blip shouldn't page. Re-run once; require agreement.
+let confirmed=true;
+if(alerts.length){
+ console.log(`\nrun1 ALERTED (${alerts.length}) → confirm-rerun after ${CONFIRM_DELAY}ms (a single blip won't page)…`);
+ await sleep(CONFIRM_DELAY);
+ const m2=await crawl(); const e2=evaluate(m2, baseline);
+ console.log(`run2: index ${m2.indexStatus} | children ${m2.childCount} | product-URLs ${m2.total} | bad ${m2.badChild}`);
+ if(e2.alerts.length===0){ confirmed=false; console.log(` → run2 CLEAN: run1 was a transient blip; NOT paging.`); alerts=[]; m=m2; complete=e2.complete; }
+ else { alerts=e2.alerts; m=m2; complete=e2.complete; console.log(` → run2 ALSO alerted: confirmed, paging.`); }
}
-console.log(`\n=== VERDICT: ${alerts.length===0?'HEALTHY':'ALERT'} ===`);
-if(alerts.length) alerts.forEach(a=>console.log(` 🔴 ${a}`));
-else console.log(` 🟢 index+children 200, count vs baseline within ${CRATER_PCT}%, regeneration < ${FRESH_HOURS}h`);
-console.log(`\nNEGATIVE CONTROL: count derived live this run (${total}) from ${prodChildren.length} children; baseline mechanism ${baseline?'compared':'seeded'}.`);
+const verdict = alerts.length===0?'HEALTHY':'ALERT';
+console.log(`\n=== VERDICT: ${verdict}${alerts.length===0&&!confirmed?' (transient blip cleared on rerun)':''} ===`);
+if(alerts.length) alerts.forEach(a=>console.log(` 🔴 ${a}`)); else console.log(` 🟢 index+children 200, count vs baseline within ${CRATER_PCT}%, sitemap not stale`);
+
+// baseline: seed if absent; UPWARD-ONLY ratchet on a healthy complete run (catalog grows; never lower the floor)
+fs.mkdirSync(DATA_DIR,{recursive:true});
+if(complete){
+ if(!baseline || !baseline.productCount){
+ baseline={productCount:m.total, childCount:m.childCount, established:new Date().toISOString(), crater_pct:CRATER_PCT, history:[{c:m.total,t:new Date().toISOString()}]};
+ fs.writeFileSync(BASELINE, JSON.stringify(baseline,null,2)); console.log(`\nNO BASELINE → ESTABLISHED @ ${m.total}/${m.childCount}`);
+ } else if(verdict==='HEALTHY' && m.total>baseline.productCount){
+ const old=baseline.productCount; baseline.productCount=m.total; baseline.childCount=m.childCount;
+ baseline.history=[...(baseline.history||[]).slice(-29), {c:m.total,t:new Date().toISOString()}];
+ fs.writeFileSync(BASELINE, JSON.stringify(baseline,null,2)); console.log(`\nbaseline RATCHETED up ${old}→${m.total} (catalog grew; floor never lowered)`);
+ }
+}
-fs.writeFileSync('/tmp/sitemap-integrity-canary.json',JSON.stringify({ts:new Date().toISOString(),indexStatus:idx.status,childCount:prodChildren.length,productCount:total,badChild,newestAgeH:ageH,alerts,verdict:alerts.length===0?'HEALTHY':'ALERT'},null,2));
-console.log('wrote /tmp/sitemap-integrity-canary.json');
+const result={ts:new Date().toISOString(),verdict,indexStatus:m.indexStatus,childCount:m.childCount,productCount:m.total,badChild:m.badChild,newestAgeH:m.ageH,baseline:baseline?.productCount??null,alerts};
+fs.writeFileSync('/tmp/sitemap-integrity-canary.json',JSON.stringify(result,null,2));
+fs.writeFileSync(path.join(DATA_DIR,'latest.json'),JSON.stringify(result,null,2)); // heartbeat for dw-canary-meta-watchdog
+console.log(`\nwrote /tmp/sitemap-integrity-canary.json + data/latest.json (heartbeat)`);
process.exitCode = alerts.length===0?0:3;
← 5427ff2 cycle 77: BUILD sitemap-integrity canary (April-mass-archive
·
back to Dw Yolo Loop
·
cycle 77: officer REVISE sign-off appended (6 hardening item 423d741 →