sitemap-health audit: robots+sitemap clean; 0 genuine dead URLs advertised (9/2000 archived were mirror-lag, live-200) — de-escalates c58 soft-404s (not sitemap-amplified)

540c991da44c2f6adc70a174a5daeb69e8e9f3d1 · 2026-06-17 11:10:15 -0700 · Steve Abrams
Files touched

A scripts/sitemap-health/sitemap-health.mjs
Diff

commit 540c991da44c2f6adc70a174a5daeb69e8e9f3d1
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 17 11:10:15 2026 -0700

    sitemap-health audit: robots+sitemap clean; 0 genuine dead URLs advertised (9/2000 archived were mirror-lag, live-200) — de-escalates c58 soft-404s (not sitemap-amplified)
---
 scripts/sitemap-health/sitemap-health.mjs | 67 +++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/scripts/sitemap-health/sitemap-health.mjs b/scripts/sitemap-health/sitemap-health.mjs
new file mode 100644
index 0000000..570c98d
--- /dev/null
+++ b/scripts/sitemap-health/sitemap-health.mjs
@@ -0,0 +1,67 @@
+// sitemap-health — READ-ONLY, $0. Extends c58: are the ~66k archived (soft-404)
+// product URLs ACTIVELY ADVERTISED in the product sitemaps? Shopify natively lists
+// only active+published products, so the expected/healthy answer is NO. We verify,
+// not assume. Also checks the Sitemap: directive in robots.txt + sitemap freshness.
+import { execSync } from 'child_process';
+const PSQL='/opt/homebrew/opt/postgresql@14/bin/psql';
+const DB='postgresql:///dw_unified?host=/tmp';
+const WWW='https://www.designerwallcoverings.com';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u);if(!r.ok){await sleep(800*(a+1));continue;}return await r.text();}catch(e){await sleep(800*(a+1));}}return '';}
+
+console.log('=== sitemap-health (READ-ONLY, $0) ===\n');
+
+// 1. robots Sitemap: directive present?
+const robots=await get(`${WWW}/robots.txt`);
+const smDirective=(robots.match(/^sitemap:\s*(.+)$/im)||[])[1];
+console.log(`[robots] Allow:/ present: ${/^allow:\s*\/\s*$/im.test(robots)} | Disallow products/collections: ${/disallow:\s*\/(products|collections)\s*$/im.test(robots)}`);
+console.log(`[robots] Sitemap: directive: ${smDirective||'MISSING'}`);
+
+// 2. enumerate product sitemaps from the index (US locale only — skip en-ca dupes)
+const idx=await get(`${WWW}/sitemap.xml`);
+const childLocs=[...idx.matchAll(/<loc>([^<]+)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&'));
+const prodMaps=childLocs.filter(u=>/\/sitemap_products_\d+\.xml/.test(u) && !/\/en-ca\//.test(u));
+console.log(`\n[index] ${childLocs.length} child sitemaps; ${prodMaps.length} US product sitemaps`);
+
+// 3. pull every product-URL handle advertised across the US product sitemaps
+const advertised=new Set();
+let fetched=0;
+for(const sm of prodMaps){
+  const xml=await get(sm);
+  const handles=[...xml.matchAll(/<loc>[^<]*\/products\/([^<\/?]+)/g)].map(m=>m[1]);
+  handles.forEach(h=>advertised.add(h));
+  fetched++;
+  await sleep(120);
+}
+console.log(`[sitemaps] fetched ${fetched}/${prodMaps.length}; ${advertised.size} distinct product handles advertised`);
+
+// 4. cross-reference: how many ARCHIVED handles appear in the advertised set?
+//    pull a large random sample of archived handles from the mirror
+let archived=[];
+try{
+  const out=execSync(`${PSQL} "${DB}" -At -F'|' -c "select handle from shopify_products where status='ARCHIVED' and handle<>'' order by md5(handle) limit 2000"`,{encoding:'utf8'});
+  archived=out.trim().split('\n').filter(Boolean);
+}catch(e){ console.log('[mirror] archived read failed: '+e.message.slice(0,80)); }
+const archivedAdvertised=archived.filter(h=>advertised.has(h));
+console.log(`\n[cross-ref] archived handles sampled: ${archived.length}`);
+console.log(`[cross-ref] archived handles ALSO in sitemap (dead URL advertised): ${archivedAdvertised.length}`);
+
+// 5. also confirm direction: how many ACTIVE handles are advertised (sanity that sitemap isn't empty/broken)
+let active=[];
+try{
+  const out=execSync(`${PSQL} "${DB}" -At -F'|' -c "select handle from shopify_products where status='ACTIVE' and handle<>'' order by md5(handle) limit 2000"`,{encoding:'utf8'});
+  active=out.trim().split('\n').filter(Boolean);
+}catch(e){}
+const activeAdvertised=active.filter(h=>advertised.has(h));
+console.log(`[cross-ref] active handles sampled: ${active.length} | of those advertised in sitemap: ${activeAdvertised.length} (${active.length?(100*activeAdvertised.length/active.length).toFixed(0):0}%)`);
+
+if(archivedAdvertised.length){ console.log('\n  sample archived-but-advertised (DEAD URLs in sitemap):'); archivedAdvertised.slice(0,12).forEach(h=>console.log('    - '+h)); }
+
+const out={ ts:new Date().toISOString(), robots:{allowRoot:/^allow:\s*\/\s*$/im.test(robots),sitemapDirective:smDirective||null}, sitemaps:{childCount:childLocs.length,usProductMaps:prodMaps.length,advertisedHandles:advertised.size}, crossRef:{archivedSampled:archived.length,archivedAdvertised:archivedAdvertised.length,activeSampled:active.length,activeAdvertised:activeAdvertised.length,sampleDead:archivedAdvertised.slice(0,20)} };
+import fs from 'fs';
+fs.writeFileSync('/tmp/sitemap-health.json',JSON.stringify(out,null,2));
+console.log('\nwrote /tmp/sitemap-health.json');
+console.log('\n=== TELL ===');
+console.log(archivedAdvertised.length===0
+  ? 'CLEAN: sitemap advertises 0 archived/dead URLs (Shopify native behavior holds) — c58 soft-404s are NOT being actively fed via sitemap.'
+  : `DEFECT: ${archivedAdvertised.length}/${archived.length} sampled archived URLs ARE in the sitemap — the soft-404s ARE being actively submitted to Googlebot.`);
← 00d4548 redirect-health audit: canon clean; archived products soft-4 · back to Dw Yolo Loop · empty-collections audit: 575 advertised -> ~37 live-empty + 1bc3c5b →