`scripts/canonical-robots-audit/canonical-robots-audit.mjs`

102 lines
// canonical-robots-audit — READ-ONLY, $0. Cycle 63 (DTD-picked A, unanimous).
// Audits INDEXABILITY of live PDPs — the gate that governs whether ~74k products
// are eligible for Google's index at all. For each sampled live PDP we read:
//   - the response X-Robots-Tag header (header-level noindex beats any on-page tag)
//   - <meta name="robots"> / <meta name="googlebot"> content (noindex/nofollow leak?)
//   - <link rel="canonical" href>  — is it SELF-referencing, CROSS-pointing, or MISSING?
// A stray noindex or a canonical that points away silently de-lists products with
// ZERO on-page symptom — exactly the invisible fleet-wide failure this loop catches.
// EXCLUDES Phillip Jeffries (phillip-jeffries-display-excluded) + en-ca locale dupes.
const WWW='https://www.designerwallcoverings.com';
const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u,method='GET'){for(let a=0;a<4;a++){try{const r=await fetch(u,{method,headers:{'User-Agent':UA},redirect:'manual'});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,headers:r.headers,text:method==='GET'?await r.text():''};}catch(e){await sleep(800*(a+1));}}return {status:0,headers:new Headers(),text:''};}
const PJ=/phillip[- ]?jeffries|phillip-jeffries/i;

// stratified handle pool across ALL non-en-ca product sitemaps
const idx=await get(`${WWW}/sitemap.xml`);
const prodMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-ca\//.test(u));
let handles=[];
for(const sm of prodMaps){
  const x=await get(sm);
  const hs=[...x.text.matchAll(/<loc>[^<]*\/products\/([^<\/?]+)/g)].map(m=>m[1]);
  const step=Math.max(1,Math.floor(hs.length/12));
  for(let i=0;i<hs.length && handles.length<400;i+=step) handles.push(hs[i]);
  await sleep(100);
}
handles=[...new Set(handles)].slice(0,360);
console.log(`=== canonical-robots-audit (READ-ONLY, $0) ===\nstratified sample: ${handles.length} live PDPs across ${prodMaps.length} sitemaps (PJ + en-ca excluded)\n`);

function headTag(html,re){ const m=html.match(re); return m?m[1]:null; }
// normalize a URL for self-ref comparison: lowercase host, strip trailing slash, strip query/hash
function norm(u){ try{ const x=new URL(u, WWW); return (x.origin+x.pathname).replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').replace(/\/$/,'').toLowerCase(); } }

let selfCanon=0, crossCanon=0, missingCanon=0;
let noindexMeta=0, nofollowMeta=0, xRobotsNoindex=0, pjSkip=0, fetchErr=0, redirected=0;
const problems=[]; // canonical-cross or noindex anything
const samples=[];
let done=0;
for(const h of handles){
  const expectUrl=norm(`${WWW}/products/${h}`);
  const r=await get(`${WWW}/products/${h}`,'GET');
  if(r.status===0){ fetchErr++; done++; continue; }
  if(r.status>=300 && r.status<400){ redirected++; done++; await sleep(110); continue; } // archived/redirect — not an indexable PDP
  if(r.status!==200){ fetchErr++; done++; continue; }
  const html=r.text;
  // vendor (from product json) for PJ exclusion
  let vendor='';
  const ldMatch=html.match(/"vendor"\s*:\s*"([^"]+)"/); if(ldMatch) vendor=ldMatch[1];
  if(PJ.test(vendor)||PJ.test(h)){ pjSkip++; done++; await sleep(80); continue; }

  // robots signals
  const xr=(r.headers.get('x-robots-tag')||'').toLowerCase();
  const metaRobots=(headTag(html,/<meta[^>]+name=["']robots["'][^>]*content=["']([^"']*)["']/i)||'').toLowerCase();
  const metaGbot=(headTag(html,/<meta[^>]+name=["']googlebot["'][^>]*content=["']([^"']*)["']/i)||'').toLowerCase();
  const robotsBlob=`${xr} | ${metaRobots} | ${metaGbot}`;
  const hasNoindex=/noindex/.test(robotsBlob);
  const hasNofollow=/nofollow/.test(robotsBlob);
  if(/noindex/.test(xr)) xRobotsNoindex++;
  if(/noindex/.test(metaRobots)||/noindex/.test(metaGbot)) noindexMeta++;
  if(hasNofollow) nofollowMeta++;

  // canonical
  const canonHref=headTag(html,/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i)
                 || headTag(html,/<link[^>]+href=["']([^"']+)["'][^>]*rel=["']canonical["']/i);
  let canonClass;
  if(!canonHref){ missingCanon++; canonClass='MISSING'; }
  else if(norm(canonHref)===expectUrl){ selfCanon++; canonClass='SELF'; }
  else { crossCanon++; canonClass='CROSS'; }

  if(hasNoindex || canonClass==='CROSS' || canonClass==='MISSING'){
    if(problems.length<40) problems.push({h,vendor,canonClass,canonHref:canonHref?norm(canonHref):null,noindex:hasNoindex,nofollow:hasNofollow,xr:xr||null,metaRobots:metaRobots||null});
  }
  if(samples.length<6) samples.push({h,canonClass,robots:metaRobots||xr||'(none)'});
  done++;
  if(done%50===0) console.log(`  ${done}/${handles.length} (self=${selfCanon} cross=${crossCanon} miss=${missingCanon} noindex=${noindexMeta+xRobotsNoindex})`);
  await sleep(130);
}

const evaluated=selfCanon+crossCanon+missingCanon;
console.log(`\n=== RESULTS ===`);
console.log(`evaluated PDPs (200-OK, non-PJ): ${evaluated} | PJ-skipped: ${pjSkip} | redirected(3xx, archived): ${redirected} | fetch-err: ${fetchErr}`);
console.log(`\n-- canonical --`);
console.log(`  SELF-referencing (healthy): ${selfCanon}  (${evaluated?(100*selfCanon/evaluated).toFixed(1):0}%)`);
console.log(`  CROSS-pointing (canonical → other URL): ${crossCanon}`);
console.log(`  MISSING canonical: ${missingCanon}`);
console.log(`\n-- robots (indexability) --`);
console.log(`  meta robots/googlebot NOINDEX: ${noindexMeta}`);
console.log(`  X-Robots-Tag header NOINDEX: ${xRobotsNoindex}`);
console.log(`  NOFOLLOW present: ${nofollowMeta}`);
if(evaluated>0){
  const bad=crossCanon+missingCanon+noindexMeta+xRobotsNoindex;
  const p=bad/evaluated, se=Math.sqrt(p*(1-p)/evaluated), lo=Math.max(0,p-1.96*se)*100, hi=Math.min(1,p+1.96*se)*100;
  console.log(`\n*** indexability-DEFECT rate (cross/missing canonical OR noindex): ${bad}/${evaluated} = ${(100*p).toFixed(1)}%  (95% CI ${lo.toFixed(1)}–${hi.toFixed(1)}%, n=${evaluated}) ***`);
}
if(problems.length){ console.log('\nproblems (first 15):'); problems.slice(0,15).forEach(x=>console.log(`  ${x.h.slice(0,44)} [${x.vendor}] canon=${x.canonClass}${x.canonHref?'→'+x.canonHref.replace(WWW,''):''} noindex=${x.noindex} nofollow=${x.nofollow}`)); }
else console.log('\nno indexability problems in sample.');
console.log('\nsample (first 6):'); samples.forEach(s=>console.log(`  ${s.h.slice(0,44)} canon=${s.canonClass} robots="${s.robots}"`));

import fs from 'fs';
fs.writeFileSync('/tmp/canonical-robots-audit.json',JSON.stringify({ts:new Date().toISOString(),sample:handles.length,evaluated,selfCanon,crossCanon,missingCanon,noindexMeta,xRobotsNoindex,nofollowMeta,pjSkip,redirected,fetchErr,problems,samples},null,2));
console.log('\nwrote /tmp/canonical-robots-audit.json');