`scripts/sitemap-health/sitemap-health.mjs`

68 lines
// sitemap-health — READ-ONLY, $0. Extends c58: are the ~66k archived (soft-404)
// product URLs ACTIVELY ADVERTISED in the product sitemaps? Shopify natively lists
// only active+published products, so the expected/healthy answer is NO. We verify,
// not assume. Also checks the Sitemap: directive in robots.txt + sitemap freshness.
import { execSync } from 'child_process';
const PSQL='/opt/homebrew/opt/postgresql@14/bin/psql';
const DB='postgresql:///dw_unified?host=/tmp';
const WWW='https://www.designerwallcoverings.com';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u);if(!r.ok){await sleep(800*(a+1));continue;}return await r.text();}catch(e){await sleep(800*(a+1));}}return '';}

console.log('=== sitemap-health (READ-ONLY, $0) ===\n');

// 1. robots Sitemap: directive present?
const robots=await get(`${WWW}/robots.txt`);
const smDirective=(robots.match(/^sitemap:\s*(.+)$/im)||[])[1];
console.log(`[robots] Allow:/ present: ${/^allow:\s*\/\s*$/im.test(robots)} | Disallow products/collections: ${/disallow:\s*\/(products|collections)\s*$/im.test(robots)}`);
console.log(`[robots] Sitemap: directive: ${smDirective||'MISSING'}`);

// 2. enumerate product sitemaps from the index (US locale only — skip en-ca dupes)
const idx=await get(`${WWW}/sitemap.xml`);
const childLocs=[...idx.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&'));
const prodMaps=childLocs.filter(u=>/\/sitemap_products_\d+\.xml/.test(u) && !/\/en-ca\//.test(u));
console.log(`\n[index] ${childLocs.length} child sitemaps; ${prodMaps.length} US product sitemaps`);

// 3. pull every product-URL handle advertised across the US product sitemaps
const advertised=new Set();
let fetched=0;
for(const sm of prodMaps){
  const xml=await get(sm);
  const handles=[...xml.matchAll(/<loc>[^<]*\/products\/([^<\/?]+)/g)].map(m=>m[1]);
  handles.forEach(h=>advertised.add(h));
  fetched++;
  await sleep(120);
}
console.log(`[sitemaps] fetched ${fetched}/${prodMaps.length}; ${advertised.size} distinct product handles advertised`);

// 4. cross-reference: how many ARCHIVED handles appear in the advertised set?
//    pull a large random sample of archived handles from the mirror
let archived=[];
try{
  const out=execSync(`${PSQL} "${DB}" -At -F'|' -c "select handle from shopify_products where status='ARCHIVED' and handle<>'' order by md5(handle) limit 2000"`,{encoding:'utf8'});
  archived=out.trim().split('\n').filter(Boolean);
}catch(e){ console.log('[mirror] archived read failed: '+e.message.slice(0,80)); }
const archivedAdvertised=archived.filter(h=>advertised.has(h));
console.log(`\n[cross-ref] archived handles sampled: ${archived.length}`);
console.log(`[cross-ref] archived handles ALSO in sitemap (dead URL advertised): ${archivedAdvertised.length}`);

// 5. also confirm direction: how many ACTIVE handles are advertised (sanity that sitemap isn't empty/broken)
let active=[];
try{
  const out=execSync(`${PSQL} "${DB}" -At -F'|' -c "select handle from shopify_products where status='ACTIVE' and handle<>'' order by md5(handle) limit 2000"`,{encoding:'utf8'});
  active=out.trim().split('\n').filter(Boolean);
}catch(e){}
const activeAdvertised=active.filter(h=>advertised.has(h));
console.log(`[cross-ref] active handles sampled: ${active.length} | of those advertised in sitemap: ${activeAdvertised.length} (${active.length?(100*activeAdvertised.length/active.length).toFixed(0):0}%)`);

if(archivedAdvertised.length){ console.log('\n  sample archived-but-advertised (DEAD URLs in sitemap):'); archivedAdvertised.slice(0,12).forEach(h=>console.log('    - '+h)); }

const out={ ts:new Date().toISOString(), robots:{allowRoot:/^allow:\s*\/\s*$/im.test(robots),sitemapDirective:smDirective||null}, sitemaps:{childCount:childLocs.length,usProductMaps:prodMaps.length,advertisedHandles:advertised.size}, crossRef:{archivedSampled:archived.length,archivedAdvertised:archivedAdvertised.length,activeSampled:active.length,activeAdvertised:activeAdvertised.length,sampleDead:archivedAdvertised.slice(0,20)} };
import fs from 'fs';
fs.writeFileSync('/tmp/sitemap-health.json',JSON.stringify(out,null,2));
console.log('\nwrote /tmp/sitemap-health.json');
console.log('\n=== TELL ===');
console.log(archivedAdvertised.length===0
  ? 'CLEAN: sitemap advertises 0 archived/dead URLs (Shopify native behavior holds) — c58 soft-404s are NOT being actively fed via sitemap.'
  : `DEFECT: ${archivedAdvertised.length}/${archived.length} sampled archived URLs ARE in the sitemap — the soft-404s ARE being actively submitted to Googlebot.`);