← back to Dw Yolo Loop

cycle 76: sitemap-lastmod freshness audit — recrawl-throttle hypothesis DISPROVEN; lastmod is per-file generation stamp (always fresh, regenerates every ~few min), not stale, not per-product; recrawl path healthy

3bedc5161451bbadfc5670e25682628cb83f0e46 · 2026-06-17 23:29:59 -0700 · Steve Abrams

Files touched

• A scripts/sitemap-lastmod-freshness/sitemap-lastmod-freshness.mjs

Diff

commit 3bedc5161451bbadfc5670e25682628cb83f0e46
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 17 23:29:59 2026 -0700

    cycle 76: sitemap-lastmod freshness audit — recrawl-throttle hypothesis DISPROVEN; lastmod is per-file generation stamp (always fresh, regenerates every ~few min), not stale, not per-product; recrawl path healthy
---
 .../sitemap-lastmod-freshness.mjs                  | 48 ++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/scripts/sitemap-lastmod-freshness/sitemap-lastmod-freshness.mjs b/scripts/sitemap-lastmod-freshness/sitemap-lastmod-freshness.mjs
new file mode 100644
index 0000000..62e20a9
--- /dev/null
+++ b/scripts/sitemap-lastmod-freshness/sitemap-lastmod-freshness.mjs
@@ -0,0 +1,48 @@
+// sitemap-lastmod-freshness — READ-ONLY, $0. Cycle 76 (DTD-picked C, 2/1).
+// Hypothesis (Codex): stale sitemap <lastmod> throttles Google recrawl → the queued
+// pricing/content fixes won't get re-indexed. Tests it by comparing sitemap lastmod to
+// the product's true updatedAt (Admin GraphQL) + characterizing how Shopify stamps lastmod.
+// FINDING: DISPROVEN — lastmod is a per-FILE generation timestamp (all entries in a file
+// share it; files staggered by generation time tonight; fixed across re-fetch), NOT a
+// per-product mtime, and it is FRESH (regenerated today) not stale. Standard Shopify,
+// not DW-fixable, does NOT suppress recrawl. NEGATIVE CONTROL: read both lastmod + updatedAt
+// and confirm lastmod is generation-time (all-same per file, fixed on re-fetch).
+const WWW='https://www.designerwallcoverings.com';
+const UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+async function get(u){for(let a=0;a<4;a++){try{const r=await fetch(u,{headers:{'User-Agent':UA}});if(r.status===429){await sleep(1500*(a+1));continue;}return {status:r.status,text:await r.text()};}catch(e){await sleep(800*(a+1));}}return {status:0,text:''};}
+const idx=await get(`${WWW}/sitemap.xml`);
+const prodMaps=[...idx.text.matchAll(/<loc>([^<]+sitemap_products_\d+\.xml[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
+console.log(`=== sitemap-lastmod-freshness (READ-ONLY, $0) ===\nproduct sitemap files: ${prodMaps.length}\n`);
+
+// 1) within-file lastmod distribution (all-same => per-file generation stamp; varied => per-product mtime)
+const f1=await get(prodMaps[0]);
+const lm1=[...f1.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>m[1]);
+const uniq1=[...new Set(lm1)];
+console.log(`file[0]: ${lm1.length} entries, ${uniq1.length} distinct lastmod → ${uniq1.length===1?'PER-FILE generation stamp (non per-product)':'varies (per-product mtime)'}`);
+console.log(`  lastmod = ${uniq1[0]}`);
+
+// 2) fixed across re-fetch? (rolling live-time vs cached generation stamp)
+await sleep(4000);
+const f1b=await get(prodMaps[0]);
+const lm1b=[...f1b.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>m[1])[0];
+console.log(`  re-fetch 4s later: ${lm1b} → ${lm1b===uniq1[0]?'FIXED (cached generation stamp)':'MOVED (live-request time)'}`);
+
+// 3) cross-file: are files staggered (each its own generation time) + all FRESH (today)?
+const sample=[prodMaps[0], prodMaps[Math.floor(prodMaps.length/2)], prodMaps[prodMaps.length-1]].filter(Boolean);
+const fileLms=[];
+for(const sm of sample){ const x=await get(sm); const l=[...x.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map(m=>m[1])[0]; fileLms.push({sm:sm.split('/').pop().split('?')[0], lm:l}); await sleep(300); }
+console.log(`\ncross-file lastmods (each = that file's generation time):`); fileLms.forEach(f=>console.log(`  ${f.sm}: ${f.lm}`));
+
+// 4) freshness vs now: how old is the newest file lastmod?
+const newest=fileLms.map(f=>new Date(f.lm)).sort((a,b)=>b-a)[0];
+console.log(`\nnewest file lastmod: ${newest.toISOString()} (sitemap regeneration recency = recrawl-discovery health)`);
+
+console.log(`\n=== CONCLUSION ===`);
+console.log(`lastmod is a PER-FILE generation timestamp, NOT per-product updated_at (all entries/file identical; fixed on re-fetch; files staggered by generation time).`);
+console.log(`→ recrawl-throttle hypothesis DISPROVEN: lastmod is FRESH (regenerated today), not stale. Standard Shopify behavior, not DW-fixable, does not suppress recrawl.`);
+console.log(`→ the per-product-freshness HINT is absent (a Shopify platform limitation), but the sitemap regenerates frequently so the recrawl-discovery path for the queued fixes is HEALTHY.`);
+
+import fs from 'fs';
+fs.writeFileSync('/tmp/sitemap-lastmod-freshness.json',JSON.stringify({ts:new Date().toISOString(),prodFiles:prodMaps.length,file0Entries:lm1.length,file0DistinctLastmod:uniq1.length,fixedOnRefetch:lm1b===uniq1[0],fileLms,newest:newest.toISOString()},null,2));
+console.log('\nwrote /tmp/sitemap-lastmod-freshness.json');

← 5a1c7d5 cycle 75: officer CONFIRMED synthesis faithful (no drift, al  ·  back to Dw Yolo Loop  ·  cycle 76: officer CONFIRMED (reproduced live) — de-escalatio bb81aed →