`scripts/sitemap-integrity-canary/sitemap-integrity-canary.mjs`

108 lines
// sitemap-integrity-canary — READ-ONLY, $0. Cycle 77 (DTD-A) + c77 officer REVISE hardening.
// A DURABLE standing tripwire for the catastrophe class the existing canaries miss:
// the April-2026 mass-archive (~39.7k products archived in one day, ~50k+ sellable dark
// for 2 months, 0 restored) — which would surface here as a SITEMAP product-URL COLLAPSE.
// Asserts each run: (1) index 200+parses; (2) every en product child 200; (3) total
// product-URL count vs a stored baseline doesn't crater (>CRATER_PCT) — ONLY on a complete
// all-200 crawl; (4) endpoint isn't serving an implausibly-stale cached doc (>STALE_HOURS).
//
// c77-officer hardening (false-positive defense for a standing canary):
//  - retry 5xx (not just 429); ABORT the crater comparison if ANY child errored (partial
//    crawl => untrustworthy sum => report reachability only, never a phantom crater);
//  - CONFIRM-BEFORE-PAGE: on any alert, re-run once after CONFIRM_DELAY; emit ALERT only if
//    the second run also alerts (a single network blip won't page);
//  - freshness re-based: Shopify rewrites all <lastmod> to ~now every regen (the c76 finding),
//    so a 24h check is dead/always-green; keep ONLY an implausible >STALE_HOURS guard
//    (would mean the endpoint is serving a stale cached doc — a real pathology);
//  - baseline UPWARD-ONLY ratchet on healthy runs (catalog grows; never lower the floor) +
//    rolling history; fileURLToPath for BASE_DIR (path-with-spaces safe); data/latest.json
//    heartbeat so dw-canary-meta-watchdog can prove it ran.
// NEGATIVE CONTROL: re-derives count live + (test) fires on an inflated baseline. Read-only.
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
const WWW='https://www.designerwallcoverings.com';
const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
const CRATER_PCT=Number(process.env.CRATER_PCT||10);   // April was ~54%; 10% catches w/ ~5x margin
const STALE_HOURS=Number(process.env.STALE_HOURS||72); // implausible-stale cached-doc guard only
const CONFIRM_DELAY=Number(process.env.CONFIRM_DELAY_MS||60000); // re-run gap before paging
const BASE_DIR=path.dirname(fileURLToPath(import.meta.url));
const DATA_DIR=path.join(BASE_DIR,'data');
const BASELINE=path.join(BASE_DIR,'baseline.json');
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<5;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA}});if(r.status===429||r.status>=500){await sleep(1500*(a+1));continue;}return {status:r.status,text:await r.text()};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}

// One full crawl → returns measurement (no paging decision here).
async function crawl(){
  const idx=await get(`${WWW}/sitemap.xml`);
  const reach=[]; if(idx.status!==200) reach.push(`INDEX non-200: ${idx.status}`);
  const prodChildren=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
  let total=0, badChild=0, newest=0;
  for(const c of prodChildren){
    const x=await get(c);
    if(x.status!==200){ badChild++; reach.push(`CHILD non-200 (${x.status}): ${c.split('/').pop().split('?')[0]}`); continue; }
    total+=[...x.text.matchAll(/<loc>[^<]*\/products\/[^<]+<\/loc>/g)].length;
    const lm=[...x.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>Date.parse(m[1])).filter(n=>!isNaN(n));
    if(lm.length) newest=Math.max(newest, ...lm);
    await sleep(120);
  }
  const ageH = newest? (Date.now()-newest)/3.6e6 : null;
  return {indexStatus:idx.status, childCount:prodChildren.length, total, badChild, ageH, reach};
}

// Evaluate a crawl → alerts[]. Crater comparison ONLY on a complete all-200 crawl.
function evaluate(m, baseline){
  const alerts=[...m.reach];
  const complete = m.indexStatus===200 && m.badChild===0 && m.childCount>0;
  if(!complete){
    alerts.push(`PARTIAL/UNREACHABLE CRAWL (index ${m.indexStatus}, ${m.badChild} bad children) — crater check SKIPPED (cannot trust partial sum)`);
  } else if(baseline && baseline.productCount){
    const dropPct = 100*(baseline.productCount-m.total)/baseline.productCount;
    if(dropPct>CRATER_PCT) alerts.push(`PRODUCT-COUNT CRATER: ${dropPct.toFixed(1)}% drop (${baseline.productCount}->${m.total}) > ${CRATER_PCT}% — possible mass-archive/sitemap collapse`);
  }
  if(m.ageH!=null && m.ageH>STALE_HOURS) alerts.push(`STALE SITEMAP: newest lastmod ${m.ageH.toFixed(1)}h > ${STALE_HOURS}h (endpoint serving a stale cached doc?)`);
  return {alerts, complete};
}

console.log(`=== sitemap-integrity-canary (READ-ONLY, $0) — hardened ===`);
let baseline=null; try{ baseline=JSON.parse(fs.readFileSync(BASELINE,'utf8')); }catch{}

let m=await crawl();
let {alerts, complete}=evaluate(m, baseline);
console.log(`run1: index ${m.indexStatus} | children ${m.childCount} | product-URLs ${m.total} | bad ${m.badChild} | lastmod age ${m.ageH==null?'n/a':m.ageH.toFixed(1)+'h'}`);
if(baseline?.productCount) console.log(`  baseline ${baseline.productCount} → drop ${(100*(baseline.productCount-m.total)/baseline.productCount).toFixed(1)}% (crater ${CRATER_PCT}%)`);

// CONFIRM-BEFORE-PAGE: a single blip shouldn't page. Re-run once; require agreement.
let confirmed=true;
if(alerts.length){
  console.log(`\nrun1 ALERTED (${alerts.length}) → confirm-rerun after ${CONFIRM_DELAY}ms (a single blip won't page)…`);
  await sleep(CONFIRM_DELAY);
  const m2=await crawl(); const e2=evaluate(m2, baseline);
  console.log(`run2: index ${m2.indexStatus} | children ${m2.childCount} | product-URLs ${m2.total} | bad ${m2.badChild}`);
  if(e2.alerts.length===0){ confirmed=false; console.log(`  → run2 CLEAN: run1 was a transient blip; NOT paging.`); alerts=[]; m=m2; complete=e2.complete; }
  else { alerts=e2.alerts; m=m2; complete=e2.complete; console.log(`  → run2 ALSO alerted: confirmed, paging.`); }
}

const verdict = alerts.length===0?'HEALTHY':'ALERT';
console.log(`\n=== VERDICT: ${verdict}${alerts.length===0&&!confirmed?' (transient blip cleared on rerun)':''} ===`);
if(alerts.length) alerts.forEach(a=>console.log(`  🔴 ${a}`)); else console.log(`  🟢 index+children 200, count vs baseline within ${CRATER_PCT}%, sitemap not stale`);

// baseline: seed if absent; UPWARD-ONLY ratchet on a healthy complete run (catalog grows; never lower the floor)
fs.mkdirSync(DATA_DIR,{recursive:true});
if(complete){
  if(!baseline || !baseline.productCount){
    baseline={productCount:m.total, childCount:m.childCount, established:new Date().toISOString(), crater_pct:CRATER_PCT, history:[{c:m.total,t:new Date().toISOString()}]};
    fs.writeFileSync(BASELINE, JSON.stringify(baseline,null,2)); console.log(`\nNO BASELINE → ESTABLISHED @ ${m.total}/${m.childCount}`);
  } else if(verdict==='HEALTHY' && m.total>baseline.productCount){
    const old=baseline.productCount; baseline.productCount=m.total; baseline.childCount=m.childCount;
    baseline.history=[...(baseline.history||[]).slice(-29), {c:m.total,t:new Date().toISOString()}];
    fs.writeFileSync(BASELINE, JSON.stringify(baseline,null,2)); console.log(`\nbaseline RATCHETED up ${old}→${m.total} (catalog grew; floor never lowered)`);
  }
}

const result={ts:new Date().toISOString(),verdict,indexStatus:m.indexStatus,childCount:m.childCount,productCount:m.total,badChild:m.badChild,newestAgeH:m.ageH,baseline:baseline?.productCount??null,alerts};
fs.writeFileSync('/tmp/sitemap-integrity-canary.json',JSON.stringify(result,null,2));
fs.writeFileSync(path.join(DATA_DIR,'latest.json'),JSON.stringify(result,null,2)); // heartbeat for dw-canary-meta-watchdog
console.log(`\nwrote /tmp/sitemap-integrity-canary.json + data/latest.json (heartbeat)`);
process.exitCode = alerts.length===0?0:3;