`product-sitemap-sampler.mjs`

32 lines
// TK-10658 — READ-ONLY: sample product URLs from the DW sitemap (default locale) and HEAD
// each to measure status distribution (200 good vs 301/404 = crawl-budget waste / stale sitemap).
const BASE='https://www.designerwallcoverings.com';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<3;a++){try{const r=await fetch(u,{headers:{'User-Agent':'Mozilla/5.0 dw-audit'}});if(r.status>=500||r.status===429){await sleep(1000*(a+1));continue;}return r;}catch(e){await sleep(800*(a+1));}}return null;}
// get raw index, product sub-sitemaps (default locale only, with params)
const idx=await (await get(`${BASE}/sitemap.xml`)).text();
const subs=[...idx.matchAll(/<loc>([^<]*sitemap_products_[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&'))
  .filter(u=>!/\/en-(ca|gb)\//.test(u)); // default locale only
// pick ~12 spread across the range
const pick=[]; const step=Math.max(1,Math.floor(subs.length/12));
for(let i=0;i<subs.length;i+=step) pick.push(subs[i]);
let allUrls=[];
for(const s of pick){ const xml=await (await get(s)).text(); const us=[...xml.matchAll(/<loc>([^<]*\/products\/[^<]+)<\/loc>/g)].map(m=>m[1]); allUrls.push(...us); }
// dedupe + random sample 250
allUrls=[...new Set(allUrls)];
for(let i=allUrls.length-1;i>0;i--){const j=Math.floor(Math.random()*(i+1));[allUrls[i],allUrls[j]]=[allUrls[j],allUrls[i]];}
const sample=allUrls.slice(0,250);
const status={}; const bad=[];
for(const u of sample){
  const r=await get(u); const code=r?r.status:0;
  // follow to final for 3xx
  let finalCode=code, loc=null;
  if(code>=300&&code<400){ loc=r.headers.get('location'); const rf=await get(u); const rl=await fetch(u,{redirect:'manual'}).catch(()=>null); }
  status[code]=(status[code]||0)+1;
  if(code!==200){ if(bad.length<40) bad.push({url:u.replace(BASE,''),code,loc}); }
  await sleep(60);
}
console.log(JSON.stringify({product_sitemaps_total:subs.length, sampled_sitemaps:pick.length,
  unique_product_urls_gathered:allUrls.length, urls_headed:sample.length,
  status_distribution:status, non200_sample:bad.slice(0,20)},null,2));