[object Object]

← back to Dw Yolo Loop

cycle 64: collection + faceted-URL canonical audit (0/231 bloat, negative-control validated)

719c31f878e4bb50cce0e84f26a161b1d346ec22 · 2026-06-17 14:04:59 -0700 · Steve Abrams

Files touched

Diff

commit 719c31f878e4bb50cce0e84f26a161b1d346ec22
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 17 14:04:59 2026 -0700

    cycle 64: collection + faceted-URL canonical audit (0/231 bloat, negative-control validated)
---
 .../collection-canonical-audit.mjs                 | 93 ++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/scripts/collection-canonical-audit/collection-canonical-audit.mjs b/scripts/collection-canonical-audit/collection-canonical-audit.mjs
new file mode 100644
index 0000000..7b62ff8
--- /dev/null
+++ b/scripts/collection-canonical-audit/collection-canonical-audit.mjs
@@ -0,0 +1,93 @@
+// collection-canonical-audit — READ-ONLY, $0. Cycle 64 (DTD-picked A, unanimous).
+// c63 confirmed PDP canonical/robots are clean. This audits the OTHER half the SEO
+// officer flagged as the #1 index-bloat risk on a 74k-SKU store: COLLECTION pages +
+// FACETED/PARAMETERIZED URLs (Shopify ?sort_by= / ?filter.* + Boost ?pf_*/?_=pf).
+// The defect we hunt: a param URL that DOES NOT collapse to the clean base canonical
+// (i.e. self-canonicals WITH the param, or points cross/missing) → Google indexes
+// thousands of near-duplicate facet permutations, wasting crawl budget + diluting signal.
+// Healthy = every sort/filter param URL canonicals to the param-stripped base.
+// Built-in NEGATIVE CONTROL: ?page=2 self-canonicals WITH the page param (Google's
+// recommended pagination behavior) — proving the parser faithfully reports a
+// param-RETAINING canonical, so a real bloat (retained sort/filter param) WOULD show.
+// EXCLUDES Phillip Jeffries + en-ca/en-gb locale sitemaps.
+const WWW='https://www.designerwallcoverings.com';
+const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA},redirect:'manual'});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,text:r.status===200?await r.text():''};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
+const PJ=/phillip[- ]?jeffries|phillip-jeffries/i;
+function canonOf(html){const m=html.match(/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i);return m?m[1]:null;}
+function norm(u){ try{ const x=new URL(u, WWW); return (x.origin+x.pathname+x.search).replace(/\/(\?|$)/,'$1').replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').toLowerCase(); } }
+const base=u=>{ try{ const x=new URL(u,WWW); return (x.origin+x.pathname).replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').toLowerCase(); } };
+
+// collection handles from the canonical (en) collections sitemap only
+const idx=await get(`${WWW}/sitemap.xml`);
+const collMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_collections_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
+let handles=[];
+for(const sm of collMaps){ const x=await get(sm); handles.push(...[...x.text.matchAll(/<loc>[^<]*\/collections\/([^<\/?]+)/g)].map(m=>m[1])); }
+handles=[...new Set(handles)].filter(h=>!PJ.test(h));
+// stratified spread of ~40 collections
+const N=40, step=Math.max(1,Math.floor(handles.length/N));
+let sample=[]; for(let i=0;i<handles.length && sample.length<N;i+=step) sample.push(handles[i]);
+console.log(`=== collection-canonical-audit (READ-ONLY, $0) ===\n${handles.length} collections advertised; sampling ${sample.length} (PJ + en-ca/en-gb excluded)\n`);
+
+// param patterns that SHOULD collapse to the param-stripped base
+const collapseParams=['?sort_by=price-ascending','?sort_by=best-selling','?filter.v.price.gte=10','?pf_v_color=Blue','?_=pf&pf_v_availability=1'];
+
+let baseSelf=0, baseBad=0;            // base collection page canonical health
+let collapseGood=0, collapseBad=0;    // param URLs collapsing to clean base
+let pageCtrlRetained=0, pageCtrlOther=0; // ?page=2 negative control
+const problems=[]; const samples=[];
+let done=0, fetchErr=0;
+for(const h of sample){
+  const baseUrl=`${WWW}/collections/${h}`;
+  const r=await get(baseUrl);
+  if(r.status!==200){ fetchErr++; done++; continue; }
+  const baseCanon=canonOf(r.text);
+  const baseOk = baseCanon && base(baseCanon)===base(baseUrl) && norm(baseCanon)===norm(baseUrl);
+  if(baseOk) baseSelf++; else { baseBad++; problems.push({h,kind:'base',canon:baseCanon?norm(baseCanon):'(none)'}); }
+
+  // test each collapse-param: canonical MUST equal the param-stripped base
+  for(const p of collapseParams){
+    const r2=await get(`${baseUrl}${p}`);
+    if(r2.status!==200){ continue; }
+    const c=canonOf(r2.text);
+    const collapsed = c && norm(c)===base(baseUrl); // param fully stripped, lands on clean base
+    if(collapsed) collapseGood++;
+    else { collapseBad++; if(problems.length<40) problems.push({h,kind:'param',param:p,canon:c?norm(c):'(none)'}); }
+    await sleep(120);
+  }
+  // negative control: ?page=2 — expect canonical to RETAIN ?page=2 (self), proving parser sees params
+  const rp=await get(`${baseUrl}?page=2`);
+  if(rp.status===200){ const cp=canonOf(rp.text);
+    if(cp && /[?&]page=2/.test(cp)) pageCtrlRetained++; else pageCtrlOther++;
+  }
+  if(samples.length<6) samples.push({h,baseOk,baseCanon:baseCanon?norm(baseCanon):'(none)'});
+  done++;
+  if(done%10===0) console.log(`  ${done}/${sample.length} (collapse good=${collapseGood} bad=${collapseBad}; base bad=${baseBad})`);
+  await sleep(140);
+}
+
+const totalCollapse=collapseGood+collapseBad;
+console.log(`\n=== RESULTS ===`);
+console.log(`collections evaluated: ${done} | fetch-err: ${fetchErr}`);
+console.log(`\n-- base collection-page canonical --`);
+console.log(`  SELF-referencing (healthy): ${baseSelf}/${baseSelf+baseBad}`);
+console.log(`  bad (cross/missing/param-retained): ${baseBad}`);
+console.log(`\n-- faceted/param URLs (should collapse to param-stripped base) --`);
+console.log(`  COLLAPSE-to-base (healthy): ${collapseGood}/${totalCollapse}  (${totalCollapse?(100*collapseGood/totalCollapse).toFixed(1):0}%)`);
+console.log(`  did NOT collapse (BLOAT risk): ${collapseBad}`);
+console.log(`\n-- NEGATIVE CONTROL (?page=2 must self-canonical WITH the param) --`);
+console.log(`  retained ?page=2 (parser sees params ✓): ${pageCtrlRetained}`);
+console.log(`  did not retain: ${pageCtrlOther}  ${pageCtrlRetained>0?'→ control PASSED (a retained sort/filter param WOULD have been caught)':'→ control INCONCLUSIVE'}`);
+if(totalCollapse>0){
+  const bad=collapseBad+baseBad, n=totalCollapse+baseSelf+baseBad;
+  const p=bad/n, se=Math.sqrt(p*(1-p)/n), lo=Math.max(0,p-1.96*se)*100, hi=Math.min(1,p+1.96*se)*100;
+  console.log(`\n*** collection index-bloat DEFECT rate: ${bad}/${n} = ${(100*p).toFixed(1)}%  (95% CI ${lo.toFixed(1)}–${hi.toFixed(1)}%, n=${n}) ***`);
+}
+if(problems.length){ console.log('\nproblems (first 15):'); problems.slice(0,15).forEach(x=>console.log(`  ${x.kind} /collections/${x.h}${x.param||''} → canon=${x.canon}`)); }
+else console.log('\nno collection-canonical bloat in sample.');
+console.log('\nsample (first 6):'); samples.forEach(s=>console.log(`  /collections/${s.h} baseOk=${s.baseOk} canon=${s.baseCanon}`));
+
+import fs from 'fs';
+fs.writeFileSync('/tmp/collection-canonical-audit.json',JSON.stringify({ts:new Date().toISOString(),advertised:handles.length,sampled:sample.length,evaluated:done,baseSelf,baseBad,collapseGood,collapseBad,pageCtrlRetained,pageCtrlOther,problems,samples},null,2));
+console.log('\nwrote /tmp/collection-canonical-audit.json');

← f80814f cycle 63: canonical+robots indexability audit (347/347 clean  ·  back to Dw Yolo Loop  ·  cycle 64: officer sign-off + in-cycle stacked-param & tag-pa 2c7c6b5 →