[object Object]

← back to Dw Yolo Loop

cycle 77: BUILD sitemap-integrity canary (April-mass-archive early-warning) — baseline 72462 product URLs/29 children; asserts index+children 200 + count-crater vs baseline (>10pct) + regen-stall (>24h); HEALTHY now, negative-control fires on 63.8pct crater

5427ff202a1c691a7a3c27324faaea0242fc18bc · 2026-06-18 00:07:35 -0700 · Steve Abrams

Files touched

Diff

commit 5427ff202a1c691a7a3c27324faaea0242fc18bc
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jun 18 00:07:35 2026 -0700

    cycle 77: BUILD sitemap-integrity canary (April-mass-archive early-warning) — baseline 72462 product URLs/29 children; asserts index+children 200 + count-crater vs baseline (>10pct) + regen-stall (>24h); HEALTHY now, negative-control fires on 63.8pct crater
---
 scripts/sitemap-integrity-canary/baseline.json     |  7 +++
 .../sitemap-integrity-canary.mjs                   | 72 ++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/scripts/sitemap-integrity-canary/baseline.json b/scripts/sitemap-integrity-canary/baseline.json
new file mode 100644
index 0000000..868e597
--- /dev/null
+++ b/scripts/sitemap-integrity-canary/baseline.json
@@ -0,0 +1,7 @@
+{
+  "establishedFromCount": 72462,
+  "productCount": 72462,
+  "childCount": 29,
+  "note": "baseline seeded cycle 77; April-mass-archive crater tripwire",
+  "crater_pct": 10
+}
\ No newline at end of file
diff --git a/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs b/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
new file mode 100644
index 0000000..47ab79e
--- /dev/null
+++ b/scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs
@@ -0,0 +1,72 @@
+// sitemap-integrity-canary — READ-ONLY, $0. Cycle 77 (DTD-picked A, unanimous).
+// A DURABLE standing tripwire for the catastrophe class the existing canaries miss:
+// the April-2026 mass-archive (~39.7k products archived in one day, ~50k+ sellable dark
+// for 2 months, 0 restored). That event would have surfaced here as a SITEMAP COLLAPSE
+// (product-URL count craters). This canary asserts, every run:
+//   (1) sitemap index returns 200 + parses;
+//   (2) every child product sitemap returns 200;
+//   (3) total product-URL count vs a stored BASELINE doesn't crater (> CRATER_PCT drop);
+//   (4) regeneration hasn't stalled (newest child <lastmod> within FRESH_HOURS).
+// First run with no baseline ESTABLISHES it (writes baseline.json) + reports HEALTHY.
+// NEGATIVE CONTROL: re-derives today's count live + compares to the baseline mechanism.
+// Distinct from the c76 always-green lastmod check — this guards 200/count/staleness.
+// EXCLUDES Phillip Jeffries N/A (sitemap-level count). Read-only; never mutates.
+import fs from 'fs';
+import path from 'path';
+const WWW='https://www.designerwallcoverings.com';
+const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const CRATER_PCT=10;     // alert if product count drops >10% vs baseline (April was ~54%)
+const FRESH_HOURS=24;    // alert if newest child lastmod older than 24h (regeneration stalled)
+const BASE_DIR=path.dirname(new URL(import.meta.url).pathname);
+const BASELINE=path.join(BASE_DIR,'baseline.json');
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA}});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,text:await r.text()};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
+
+const alerts=[];
+console.log(`=== sitemap-integrity-canary (READ-ONLY, $0) ===`);
+
+// (1) index
+const idx=await get(`${WWW}/sitemap.xml`);
+if(idx.status!==200){ alerts.push(`INDEX non-200: ${idx.status}`); }
+const allChildren=[...idx.text.matchAll(/<loc>([^<]+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&'));
+const prodChildren=allChildren.filter(u=>/sitemap_products_\d+\.xml/.test(u) && !/\/en-(ca|gb)\//.test(u));
+console.log(`index: ${idx.status} | total child sitemaps: ${allChildren.length} | en product children: ${prodChildren.length}`);
+
+// (2)+(3) per child: 200 + count product locs; (4) freshness
+let total=0, badChild=0, newest=0;
+for(const c of prodChildren){
+  const x=await get(c);
+  if(x.status!==200){ badChild++; alerts.push(`CHILD non-200 (${x.status}): ${c.split('/').pop().split('?')[0]}`); continue; }
+  total+=[...x.text.matchAll(/<loc>[^<]*\/products\/[^<]+<\/loc>/g)].length;
+  const lm=[...x.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>Date.parse(m[1])).filter(n=>!isNaN(n));
+  if(lm.length) newest=Math.max(newest, ...lm);
+  await sleep(120);
+}
+console.log(`product-URL count (en): ${total} | bad children: ${badChild}`);
+const ageH = newest? (Date.now()-newest)/3.6e6 : null;
+console.log(`newest child lastmod age: ${ageH==null?'(none)':ageH.toFixed(1)+'h'} (stall threshold ${FRESH_HOURS}h)`);
+if(ageH!=null && ageH>FRESH_HOURS) alerts.push(`REGENERATION STALLED: newest lastmod ${ageH.toFixed(1)}h > ${FRESH_HOURS}h`);
+
+// baseline compare / establish
+let baseline=null; try{ baseline=JSON.parse(fs.readFileSync(BASELINE,'utf8')); }catch{}
+if(!baseline || !baseline.productCount){
+  // NOTE: import.meta.url pathname is fine for read; for write use the resolved path
+  const seed={establishedFromCount:total, productCount:total, childCount:prodChildren.length, note:'baseline seeded cycle 77; April-mass-archive crater tripwire', crater_pct:CRATER_PCT};
+  try{ fs.writeFileSync(BASELINE, JSON.stringify(seed,null,2)); console.log(`\nNO BASELINE → ESTABLISHED baseline.json @ productCount=${total}, childCount=${prodChildren.length}`); }
+  catch(e){ console.log('baseline write err',e.message); }
+} else {
+  const dropPct = baseline.productCount? (100*(baseline.productCount-total)/baseline.productCount) : 0;
+  console.log(`\nbaseline productCount=${baseline.productCount} → current=${total} → drop ${dropPct.toFixed(1)}% (crater threshold ${CRATER_PCT}%)`);
+  if(dropPct>CRATER_PCT) alerts.push(`PRODUCT-COUNT CRATER: ${dropPct.toFixed(1)}% drop (${baseline.productCount}→${total}) > ${CRATER_PCT}% — possible mass-archive/sitemap collapse`);
+  // also flag a >2x SURGE (could signal a bad bulk-publish) — informational
+  if(total > baseline.productCount*1.5) console.log(`  (note: count surged >50% vs baseline — possible bulk (re)publish, informational)`);
+}
+
+console.log(`\n=== VERDICT: ${alerts.length===0?'HEALTHY':'ALERT'} ===`);
+if(alerts.length) alerts.forEach(a=>console.log(`  🔴 ${a}`));
+else console.log(`  🟢 index+children 200, count vs baseline within ${CRATER_PCT}%, regeneration < ${FRESH_HOURS}h`);
+console.log(`\nNEGATIVE CONTROL: count derived live this run (${total}) from ${prodChildren.length} children; baseline mechanism ${baseline?'compared':'seeded'}.`);
+
+fs.writeFileSync('/tmp/sitemap-integrity-canary.json',JSON.stringify({ts:new Date().toISOString(),indexStatus:idx.status,childCount:prodChildren.length,productCount:total,badChild,newestAgeH:ageH,alerts,verdict:alerts.length===0?'HEALTHY':'ALERT'},null,2));
+console.log('wrote /tmp/sitemap-integrity-canary.json');
+process.exitCode = alerts.length===0?0:3;

← bb81aed cycle 76: officer CONFIRMED (reproduced live) — de-escalatio  ·  back to Dw Yolo Loop  ·  cycle 77: harden canary per officer REVISE — abort-on-partia 213fd60 →