`leak-rate.mjs`

28 lines
// TK-10661 — tighter measure: JSON-LD $4.25-in-offer rate across a larger REGULAR-product sample.
const BASE='https://www.designerwallcoverings.com';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function get(u){for(let a=0;a<3;a++){try{const r=await fetch(u,{headers:{'User-Agent':'Mozilla/5.0 dw-audit'}});if(r.status>=500||r.status===429){await sleep(900*(a+1));continue;}return r;}catch(e){await sleep(700*(a+1));}}return null;}
const idx=await (await get(`${BASE}/sitemap.xml`)).text();
const subs=[...idx.matchAll(/<loc>([^<]*sitemap_products_[^<]*)<\/loc>/g)].map(m=>m[1].replace(/&amp;/g,'&')).filter(u=>!/\/en-(ca|gb)\//.test(u));
const pick=[]; const step=Math.max(1,Math.floor(subs.length/8)); for(let i=0;i<subs.length;i+=step) pick.push(subs[i]);
let urls=[]; for(const s of pick){ const xml=await (await get(s)).text(); urls.push(...[...xml.matchAll(/<loc>([^<]*\/products\/[^<]+)<\/loc>/g)].map(m=>m[1])); }
urls=[...new Set(urls)].filter(u=>!/-sample-|memo-sample/i.test(u)); // regular only
for(let i=urls.length-1;i>0;i--){const j=Math.floor(Math.random()*(i+1));[urls[i],urls[j]]=[urls[j],urls[i]];}
const set=urls.slice(0,120);
let leak=0, real=0, noprod=0, err=0; const realEx=[];
for(const u of set){ const r=await get(u); if(!r||r.status!==200){err++;continue;}
  const html=await r.text();
  const blocks=[...html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)].map(m=>m[1]);
  let price=null,found=false;
  for(const b of blocks){try{const j=JSON.parse(b.trim());const arr=Array.isArray(j)?j:(j['@graph']||[j]);
    for(const n of arr){if(/product/i.test(n['@type']||'')){found=true;const o=Array.isArray(n.offers)?n.offers[0]:n.offers;if(o&&o.price!=null)price=Number(o.price);}}}catch{}}
  if(!found){noprod++;continue;}
  if(price!=null&&price<=5) leak++; else { real++; if(realEx.length<5)realEx.push({url:u.replace(BASE,''),price}); }
  await sleep(70);
}
const n=leak+real;
// Wilson 95% CI on leak rate
const p=leak/n, z=1.96, denom=1+z*z/n, lo=(p+z*z/(2*n)-z*Math.sqrt((p*(1-p)+z*z/(4*n))/n))/denom, hi=(p+z*z/(2*n)+z*Math.sqrt((p*(1-p)+z*z/(4*n))/n))/denom;
console.log(JSON.stringify({regular_tested:n, price_4_25_in_schema:leak, real_price:real, no_product_schema:noprod, fetch_err:err,
  leak_rate:+p.toFixed(3), wilson95:[+lo.toFixed(3),+hi.toFixed(3)], real_price_examples:realEx},null,2));