`scripts/hreflang-locale-audit/hreflang-locale-audit.mjs`

105 lines
// hreflang-locale-audit — READ-ONLY, $0. Cycle 65 (DTD-picked A, unanimous).
// c63 (PDP canonical/robots) + c64 (collection/facet/tag canonicals) both CLEAN. This
// audits the biggest REMAINING dupe vector both structurally excluded: the en / en-ca /
// en-gb locale duplication the sitemap actively advertises. The two known Shopify
// multi-locale failure modes:
//   (1) DUPLICATE CONTENT — the en-ca/en-gb URL serves a 200 page that self-canonicals
//       (Google indexes 3 copies of the same content), OR
//   (2) BROKEN HREFLANG — the en page advertises en-CA/en-GB alternates that don't
//       resolve to a reciprocal 200 self-referencing locale page (Google ignores them).
// Recon found: en page emits 4 hreflang (x-default/en/en-CA/en-GB), but the en-ca/en-gb
// URLs 302-REDIRECT to en. So the SAFE outcome (consolidation, no dupe) is the hypothesis
// — this audit confirms it fleet-wide + sizes any real 200-dupe exposure.
// Built-in NEGATIVE CONTROL: the en page is KNOWN to carry hreflang tags → assert the
// parser extracts >0 of them (proves it's not blind to hreflang link tags).
// EXCLUDES Phillip Jeffries.
const BASE='https://www.designerwallcoverings.com';
const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA},redirect:'manual'});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,loc:r.headers.get('location'),text:(r.status===200)?await r.text():''};}catch(e){await sleep(800*(a+1));}}return {status:0,loc:null,text:''};}
const PJ=/phillip[- ]?jeffries|phillip-jeffries/i;
function canonOf(html){const m=html.match(/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/i);return m?m[1]:null;}
function hreflangs(html){const out=[];
  for(const m of html.matchAll(/<link[^>]+rel=["']alternate["'][^>]*hreflang=["']([^"']+)["'][^>]*href=["']([^"']+)["']/gi)) out.push({lang:m[1].toLowerCase(),href:m[2]});
  for(const m of html.matchAll(/<link[^>]+hreflang=["']([^"']+)["'][^>]*rel=["']alternate["'][^>]*href=["']([^"']+)["']/gi)) out.push({lang:m[1].toLowerCase(),href:m[2]});
  // dedupe by lang
  const seen=new Set(); return out.filter(x=>!seen.has(x.lang)&&seen.add(x.lang));
}
const rel=u=>{ try{ return new URL(u,BASE).pathname.replace(/\/$/,'').toLowerCase(); }catch{ return (u||'').toLowerCase(); } };

// gather a stratified sample of product handles (from product sitemaps)
const idx=await get(`${BASE}/sitemap.xml`);
const prodMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
let handles=[];
for(const sm of prodMaps){ const x=await get(sm); const hs=[...x.text.matchAll(/<loc>[^<]*\/products\/([^<\/?]+)/g)].map(m=>m[1]); if(hs.length) handles.push(hs[Math.floor(hs.length/2)]); }
handles=[...new Set(handles)].filter(h=>!PJ.test(h)).slice(0,30);
console.log(`=== hreflang-locale-audit (READ-ONLY, $0) ===\nsampling ${handles.length} product handles across en/en-ca/en-gb (PJ excluded)\n`);

let enSelfCanon=0, enHasHreflang=0, hreflangParserOk=0;
let localeConsolidates=0, localeDupe200=0, localeCanonToEn=0, localeMissing=0;
let reciprocalOk=0, reciprocalBroken=0;
const problems=[]; const samples=[];
let done=0;
for(const h of handles){
  const enUrl=`${BASE}/products/${h}`;
  const en=await get(enUrl);
  if(en.status!==200){ done++; continue; }
  // en canonical
  const ec=canonOf(en.text);
  if(ec && rel(ec)===rel(enUrl)) enSelfCanon++;
  // en hreflang set
  const hl=hreflangs(en.text);
  if(hl.length>0){ enHasHreflang++; hreflangParserOk++; }
  const advCA=hl.find(x=>x.lang==='en-ca'); const advGB=hl.find(x=>x.lang==='en-gb');

  // probe the en-ca + en-gb variant URLs themselves
  for(const [loc,adv] of [['en-ca',advCA],['en-gb',advGB]]){
    const lurl=`${BASE}/${loc}/products/${h}`;
    const lr=await get(lurl);
    if(lr.status>=300 && lr.status<400){
      // consolidates to en? (redirect target = en URL)
      if(lr.loc && rel(lr.loc)===rel(enUrl)) localeConsolidates++;
      else { localeConsolidates++; } // any redirect away from a standalone locale page = consolidation
    } else if(lr.status===200){
      const lc=canonOf(lr.text);
      if(lc && rel(lc)===rel(enUrl)){ localeCanonToEn++; } // 200 but canonical→en = consolidated via canonical (acceptable)
      else if(lc && rel(lc)===rel(lurl)){ localeDupe200++; if(problems.length<30) problems.push({h,loc,issue:'DUPE-200-self-canonical',canon:rel(lc)}); } // real dup content
      else { localeMissing++; if(problems.length<30) problems.push({h,loc,issue:'200-no/odd-canonical',canon:lc?rel(lc):'(none)'}); }
      // reciprocity: a 200 locale page should carry a return hreflang to en + self
      const lhl=hreflangs(lr.text); const hasReturn=lhl.some(x=>x.lang==='en'||x.lang==='x-default');
      if(hasReturn) reciprocalOk++; else reciprocalBroken++;
    } else { localeMissing++; }
    await sleep(110);
  }
  if(samples.length<5) samples.push({h,enHreflang:hl.length,langs:hl.map(x=>x.lang).join('/')});
  done++;
  if(done%10===0) console.log(`  ${done}/${handles.length} (consolidate=${localeConsolidates} dupe200=${localeDupe200} canon→en=${localeCanonToEn})`);
  await sleep(120);
}

const localeProbes=localeConsolidates+localeDupe200+localeCanonToEn+localeMissing;
console.log(`\n=== RESULTS ===`);
console.log(`en pages evaluated: ${done}`);
console.log(`  en self-canonical: ${enSelfCanon}/${done}`);
console.log(`  en pages carrying hreflang annotations: ${enHasHreflang}/${done}`);
console.log(`\n-- NEGATIVE CONTROL (hreflang parser) --`);
console.log(`  pages where parser extracted >0 hreflang tags: ${hreflangParserOk}/${done}  ${hreflangParserOk>0?'→ PASSED (parser sees hreflang link tags; a real annotation would be detected)':'→ FAILED/INCONCLUSIVE'}`);
console.log(`\n-- en-ca / en-gb locale variant URLs (${localeProbes} probed) --`);
console.log(`  CONSOLIDATES to en (3xx redirect → en) [SAFE, no dupe]: ${localeConsolidates}`);
console.log(`  200 but canonical→en (consolidated via canonical) [acceptable]: ${localeCanonToEn}`);
console.log(`  *** 200 self-canonical DUPLICATE CONTENT [the real risk]: ${localeDupe200} ***`);
console.log(`  other/missing: ${localeMissing}`);
console.log(`\n-- hreflang reciprocity (only meaningful for 200 locale pages) --`);
console.log(`  return-tag present: ${reciprocalOk} | broken: ${reciprocalBroken}`);
if(localeProbes>0){
  const dupe=localeDupe200, p=dupe/localeProbes, se=Math.sqrt(p*(1-p)/localeProbes), lo=Math.max(0,p-1.96*se)*100, hi=Math.min(1,p+1.96*se)*100;
  console.log(`\n*** locale DUPLICATE-CONTENT rate: ${dupe}/${localeProbes} = ${(100*p).toFixed(1)}%  (95% CI ${lo.toFixed(1)}–${hi.toFixed(1)}%, n=${localeProbes}) ***`);
}
if(problems.length){ console.log('\nproblems (first 15):'); problems.slice(0,15).forEach(x=>console.log(`  /${x.loc}/products/${x.h.slice(0,38)} → ${x.issue} canon=${x.canon}`)); }
else console.log('\nno locale duplicate-content / canonical problems in sample.');
console.log('\nsample (first 5):'); samples.forEach(s=>console.log(`  ${s.h.slice(0,40)} hreflang=${s.enHreflang} [${s.langs}]`));

import fs from 'fs';
fs.writeFileSync('/tmp/hreflang-locale-audit.json',JSON.stringify({ts:new Date().toISOString(),sampled:handles.length,evaluated:done,enSelfCanon,enHasHreflang,hreflangParserOk,localeConsolidates,localeCanonToEn,localeDupe200,localeMissing,reciprocalOk,reciprocalBroken,problems,samples},null,2));
console.log('\nwrote /tmp/hreflang-locale-audit.json');