`scripts/collection-canonical-audit/collection-canonical-audit.mjs`

94 lines
// collection-canonical-audit — READ-ONLY, $0. Cycle 64 (DTD-picked A, unanimous).
// c63 confirmed PDP canonical/robots are clean. This audits the OTHER half the SEO
// officer flagged as the #1 index-bloat risk on a 74k-SKU store: COLLECTION pages +
// FACETED/PARAMETERIZED URLs (Shopify ?sort_by= / ?filter.* + Boost ?pf_*/?_=pf).
// The defect we hunt: a param URL that DOES NOT collapse to the clean base canonical
// (i.e. self-canonicals WITH the param, or points cross/missing) → Google indexes
// thousands of near-duplicate facet permutations, wasting crawl budget + diluting signal.
// Healthy = every sort/filter param URL canonicals to the param-stripped base.
// Built-in NEGATIVE CONTROL: ?page=2 self-canonicals WITH the page param (Google's
// recommended pagination behavior) — proving the parser faithfully reports a
// param-RETAINING canonical, so a real bloat (retained sort/filter param) WOULD show.
// EXCLUDES Phillip Jeffries + en-ca/en-gb locale sitemaps.
const WWW='https://www.designerwallcoverings.com';
const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA},redirect:'manual'});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,text:r.status===200?await r.text():''};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
const PJ=/phillip[- ]?jeffries|phillip-jeffries/i;
function canonOf(html){const m=html.match(/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i);return m?m[1]:null;}
function norm(u){ try{ const x=new URL(u, WWW); return (x.origin+x.pathname+x.search).replace(/\/(\?|$)/,'$1').replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').toLowerCase(); } }
const base=u=>{ try{ const x=new URL(u,WWW); return (x.origin+x.pathname).replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').toLowerCase(); } };

// collection handles from the canonical (en) collections sitemap only
const idx=await get(`${WWW}/sitemap.xml`);
const collMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_collections_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
let handles=[];
for(const sm of collMaps){ const x=await get(sm); handles.push(...[...x.text.matchAll(/<loc>[^<]*\/collections\/([^<\/?]+)/g)].map(m=>m[1])); }
handles=[...new Set(handles)].filter(h=>!PJ.test(h));
// stratified spread of ~40 collections
const N=40, step=Math.max(1,Math.floor(handles.length/N));
let sample=[]; for(let i=0;i<handles.length && sample.length<N;i+=step) sample.push(handles[i]);
console.log(`=== collection-canonical-audit (READ-ONLY, $0) ===\n${handles.length} collections advertised; sampling ${sample.length} (PJ + en-ca/en-gb excluded)\n`);

// param patterns that SHOULD collapse to the param-stripped base
const collapseParams=['?sort_by=price-ascending','?sort_by=best-selling','?filter.v.price.gte=10','?pf_v_color=Blue','?_=pf&pf_v_availability=1'];

let baseSelf=0, baseBad=0;            // base collection page canonical health
let collapseGood=0, collapseBad=0;    // param URLs collapsing to clean base
let pageCtrlRetained=0, pageCtrlOther=0; // ?page=2 negative control
const problems=[]; const samples=[];
let done=0, fetchErr=0;
for(const h of sample){
  const baseUrl=`${WWW}/collections/${h}`;
  const r=await get(baseUrl);
  if(r.status!==200){ fetchErr++; done++; continue; }
  const baseCanon=canonOf(r.text);
  const baseOk = baseCanon && base(baseCanon)===base(baseUrl) && norm(baseCanon)===norm(baseUrl);
  if(baseOk) baseSelf++; else { baseBad++; problems.push({h,kind:'base',canon:baseCanon?norm(baseCanon):'(none)'}); }

  // test each collapse-param: canonical MUST equal the param-stripped base
  for(const p of collapseParams){
    const r2=await get(`${baseUrl}${p}`);
    if(r2.status!==200){ continue; }
    const c=canonOf(r2.text);
    const collapsed = c && norm(c)===base(baseUrl); // param fully stripped, lands on clean base
    if(collapsed) collapseGood++;
    else { collapseBad++; if(problems.length<40) problems.push({h,kind:'param',param:p,canon:c?norm(c):'(none)'}); }
    await sleep(120);
  }
  // negative control: ?page=2 — expect canonical to RETAIN ?page=2 (self), proving parser sees params
  const rp=await get(`${baseUrl}?page=2`);
  if(rp.status===200){ const cp=canonOf(rp.text);
    if(cp && /[?&]page=2/.test(cp)) pageCtrlRetained++; else pageCtrlOther++;
  }
  if(samples.length<6) samples.push({h,baseOk,baseCanon:baseCanon?norm(baseCanon):'(none)'});
  done++;
  if(done%10===0) console.log(`  ${done}/${sample.length} (collapse good=${collapseGood} bad=${collapseBad}; base bad=${baseBad})`);
  await sleep(140);
}

const totalCollapse=collapseGood+collapseBad;
console.log(`\n=== RESULTS ===`);
console.log(`collections evaluated: ${done} | fetch-err: ${fetchErr}`);
console.log(`\n-- base collection-page canonical --`);
console.log(`  SELF-referencing (healthy): ${baseSelf}/${baseSelf+baseBad}`);
console.log(`  bad (cross/missing/param-retained): ${baseBad}`);
console.log(`\n-- faceted/param URLs (should collapse to param-stripped base) --`);
console.log(`  COLLAPSE-to-base (healthy): ${collapseGood}/${totalCollapse}  (${totalCollapse?(100*collapseGood/totalCollapse).toFixed(1):0}%)`);
console.log(`  did NOT collapse (BLOAT risk): ${collapseBad}`);
console.log(`\n-- NEGATIVE CONTROL (?page=2 must self-canonical WITH the param) --`);
console.log(`  retained ?page=2 (parser sees params ✓): ${pageCtrlRetained}`);
console.log(`  did not retain: ${pageCtrlOther}  ${pageCtrlRetained>0?'→ control PASSED (a retained sort/filter param WOULD have been caught)':'→ control INCONCLUSIVE'}`);
if(totalCollapse>0){
  const bad=collapseBad+baseBad, n=totalCollapse+baseSelf+baseBad;
  const p=bad/n, se=Math.sqrt(p*(1-p)/n), lo=Math.max(0,p-1.96*se)*100, hi=Math.min(1,p+1.96*se)*100;
  console.log(`\n*** collection index-bloat DEFECT rate: ${bad}/${n} = ${(100*p).toFixed(1)}%  (95% CI ${lo.toFixed(1)}–${hi.toFixed(1)}%, n=${n}) ***`);
}
if(problems.length){ console.log('\nproblems (first 15):'); problems.slice(0,15).forEach(x=>console.log(`  ${x.kind} /collections/${x.h}${x.param||''} → canon=${x.canon}`)); }
else console.log('\nno collection-canonical bloat in sample.');
console.log('\nsample (first 6):'); samples.forEach(s=>console.log(`  /collections/${s.h} baseOk=${s.baseOk} canon=${s.baseCanon}`));

import fs from 'fs';
fs.writeFileSync('/tmp/collection-canonical-audit.json',JSON.stringify({ts:new Date().toISOString(),advertised:handles.length,sampled:sample.length,evaluated:done,baseSelf,baseBad,collapseGood,collapseBad,pageCtrlRetained,pageCtrlOther,problems,samples},null,2));
console.log('\nwrote /tmp/collection-canonical-audit.json');