← back to Dw Yolo Loop

cycle 63: canonical+robots indexability audit (347/347 clean, negative-control validated)

f80814fc1926fc488eed5337e4fa52bf83acc81b · 2026-06-17 13:24:30 -0700 · Steve Abrams

Files touched

• A scripts/canonical-robots-audit/canonical-robots-audit.mjs

Diff

commit f80814fc1926fc488eed5337e4fa52bf83acc81b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 17 13:24:30 2026 -0700

    cycle 63: canonical+robots indexability audit (347/347 clean, negative-control validated)
---
 .../canonical-robots-audit.mjs                     | 101 +++++++++++++++++++++
 1 file changed, 101 insertions(+)

diff --git a/scripts/canonical-robots-audit/canonical-robots-audit.mjs b/scripts/canonical-robots-audit/canonical-robots-audit.mjs
new file mode 100644
index 0000000..249c167
--- /dev/null
+++ b/scripts/canonical-robots-audit/canonical-robots-audit.mjs
@@ -0,0 +1,101 @@
+// canonical-robots-audit — READ-ONLY, $0. Cycle 63 (DTD-picked A, unanimous).
+// Audits INDEXABILITY of live PDPs — the gate that governs whether ~74k products
+// are eligible for Google's index at all. For each sampled live PDP we read:
+//   - the response X-Robots-Tag header (header-level noindex beats any on-page tag)
+//   - <meta name="robots"> / <meta name="googlebot"> content (noindex/nofollow leak?)
+//   - <link rel="canonical" href>  — is it SELF-referencing, CROSS-pointing, or MISSING?
+// A stray noindex or a canonical that points away silently de-lists products with
+// ZERO on-page symptom — exactly the invisible fleet-wide failure this loop catches.
+// EXCLUDES Phillip Jeffries (phillip-jeffries-display-excluded) + en-ca locale dupes.
+const WWW='https://www.designerwallcoverings.com';
+const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+async function get(u,method='GET'){for(let a=0;a<4;a++){try{const r=await fetch(u,{method,headers:{'User-Agent':UA},redirect:'manual'});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,headers:r.headers,text:method==='GET'?await r.text():''};}catch(e){await sleep(800*(a+1));}}return {status:0,headers:new Headers(),text:''};}
+const PJ=/phillip[- ]?jeffries|phillip-jeffries/i;
+
+// stratified handle pool across ALL non-en-ca product sitemaps
+const idx=await get(`${WWW}/sitemap.xml`);
+const prodMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-ca\//.test(u));
+let handles=[];
+for(const sm of prodMaps){
+  const x=await get(sm);
+  const hs=[...x.text.matchAll(/<loc>[^<]*\/products\/([^<\/?]+)/g)].map(m=>m[1]);
+  const step=Math.max(1,Math.floor(hs.length/12));
+  for(let i=0;i<hs.length && handles.length<400;i+=step) handles.push(hs[i]);
+  await sleep(100);
+}
+handles=[...new Set(handles)].slice(0,360);
+console.log(`=== canonical-robots-audit (READ-ONLY, $0) ===\nstratified sample: ${handles.length} live PDPs across ${prodMaps.length} sitemaps (PJ + en-ca excluded)\n`);
+
+function headTag(html,re){ const m=html.match(re); return m?m[1]:null; }
+// normalize a URL for self-ref comparison: lowercase host, strip trailing slash, strip query/hash
+function norm(u){ try{ const x=new URL(u, WWW); return (x.origin+x.pathname).replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').replace(/\/$/,'').toLowerCase(); } }
+
+let selfCanon=0, crossCanon=0, missingCanon=0;
+let noindexMeta=0, nofollowMeta=0, xRobotsNoindex=0, pjSkip=0, fetchErr=0, redirected=0;
+const problems=[]; // canonical-cross or noindex anything
+const samples=[];
+let done=0;
+for(const h of handles){
+  const expectUrl=norm(`${WWW}/products/${h}`);
+  const r=await get(`${WWW}/products/${h}`,'GET');
+  if(r.status===0){ fetchErr++; done++; continue; }
+  if(r.status>=300 && r.status<400){ redirected++; done++; await sleep(110); continue; } // archived/redirect — not an indexable PDP
+  if(r.status!==200){ fetchErr++; done++; continue; }
+  const html=r.text;
+  // vendor (from product json) for PJ exclusion
+  let vendor='';
+  const ldMatch=html.match(/"vendor"\s*:\s*"([^"]+)"/); if(ldMatch) vendor=ldMatch[1];
+  if(PJ.test(vendor)||PJ.test(h)){ pjSkip++; done++; await sleep(80); continue; }
+
+  // robots signals
+  const xr=(r.headers.get('x-robots-tag')||'').toLowerCase();
+  const metaRobots=(headTag(html,/<meta[^>]+name=["']robots["'][^>]*content=["']([^"']*)["']/i)||'').toLowerCase();
+  const metaGbot=(headTag(html,/<meta[^>]+name=["']googlebot["'][^>]*content=["']([^"']*)["']/i)||'').toLowerCase();
+  const robotsBlob=`${xr} | ${metaRobots} | ${metaGbot}`;
+  const hasNoindex=/noindex/.test(robotsBlob);
+  const hasNofollow=/nofollow/.test(robotsBlob);
+  if(/noindex/.test(xr)) xRobotsNoindex++;
+  if(/noindex/.test(metaRobots)||/noindex/.test(metaGbot)) noindexMeta++;
+  if(hasNofollow) nofollowMeta++;
+
+  // canonical
+  const canonHref=headTag(html,/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i)
+                 || headTag(html,/<link[^>]+href=["']([^"']+)["'][^>]*rel=["']canonical["']/i);
+  let canonClass;
+  if(!canonHref){ missingCanon++; canonClass='MISSING'; }
+  else if(norm(canonHref)===expectUrl){ selfCanon++; canonClass='SELF'; }
+  else { crossCanon++; canonClass='CROSS'; }
+
+  if(hasNoindex || canonClass==='CROSS' || canonClass==='MISSING'){
+    if(problems.length<40) problems.push({h,vendor,canonClass,canonHref:canonHref?norm(canonHref):null,noindex:hasNoindex,nofollow:hasNofollow,xr:xr||null,metaRobots:metaRobots||null});
+  }
+  if(samples.length<6) samples.push({h,canonClass,robots:metaRobots||xr||'(none)'});
+  done++;
+  if(done%50===0) console.log(`  ${done}/${handles.length} (self=${selfCanon} cross=${crossCanon} miss=${missingCanon} noindex=${noindexMeta+xRobotsNoindex})`);
+  await sleep(130);
+}
+
+const evaluated=selfCanon+crossCanon+missingCanon;
+console.log(`\n=== RESULTS ===`);
+console.log(`evaluated PDPs (200-OK, non-PJ): ${evaluated} | PJ-skipped: ${pjSkip} | redirected(3xx, archived): ${redirected} | fetch-err: ${fetchErr}`);
+console.log(`\n-- canonical --`);
+console.log(`  SELF-referencing (healthy): ${selfCanon}  (${evaluated?(100*selfCanon/evaluated).toFixed(1):0}%)`);
+console.log(`  CROSS-pointing (canonical → other URL): ${crossCanon}`);
+console.log(`  MISSING canonical: ${missingCanon}`);
+console.log(`\n-- robots (indexability) --`);
+console.log(`  meta robots/googlebot NOINDEX: ${noindexMeta}`);
+console.log(`  X-Robots-Tag header NOINDEX: ${xRobotsNoindex}`);
+console.log(`  NOFOLLOW present: ${nofollowMeta}`);
+if(evaluated>0){
+  const bad=crossCanon+missingCanon+noindexMeta+xRobotsNoindex;
+  const p=bad/evaluated, se=Math.sqrt(p*(1-p)/evaluated), lo=Math.max(0,p-1.96*se)*100, hi=Math.min(1,p+1.96*se)*100;
+  console.log(`\n*** indexability-DEFECT rate (cross/missing canonical OR noindex): ${bad}/${evaluated} = ${(100*p).toFixed(1)}%  (95% CI ${lo.toFixed(1)}–${hi.toFixed(1)}%, n=${evaluated}) ***`);
+}
+if(problems.length){ console.log('\nproblems (first 15):'); problems.slice(0,15).forEach(x=>console.log(`  ${x.h.slice(0,44)} [${x.vendor}] canon=${x.canonClass}${x.canonHref?'→'+x.canonHref.replace(WWW,''):''} noindex=${x.noindex} nofollow=${x.nofollow}`)); }
+else console.log('\nno indexability problems in sample.');
+console.log('\nsample (first 6):'); samples.forEach(s=>console.log(`  ${s.h.slice(0,44)} canon=${s.canonClass} robots="${s.robots}"`));
+
+import fs from 'fs';
+fs.writeFileSync('/tmp/canonical-robots-audit.json',JSON.stringify({ts:new Date().toISOString(),sample:handles.length,evaluated,selfCanon,crossCanon,missingCanon,noindexMeta,xRobotsNoindex,nofollowMeta,pjSkip,redirected,fetchErr,problems,samples},null,2));
+console.log('\nwrote /tmp/canonical-robots-audit.json');

← 0f99d97 jsonld-425-scan: sized the $4.25 JSON-LD leak — 87.8% of mul  ·  back to Dw Yolo Loop  ·  cycle 64: collection + faceted-URL canonical audit (0/231 bl 719c31f →