`tk10978-rebelwalls/rescrape-39.mjs`

40 lines
#!/usr/bin/env node
// TK-10978 — focused re-scrape of the 39 NO-TWIN candidates (READ-ONLY, $0).
// Proper JSON-LD parse + 2s delay + 1 retry (avoids the soft-block the 494 sweep hit).
// Classifies each: recoverable_live | oos | discontinued_410 | no_vendor_match.
// Writes ONLY local artifacts. NO writes to Shopify or dw_unified. Makes nothing buyable.
import { readFileSync, writeFileSync } from 'node:fs';
import { execSync } from 'node:child_process';
const HERE = new URL('.', import.meta.url).pathname;
const rows = readFileSync(`${HERE}data/investigate-39-no-twin.csv`,'utf8').trim().split('\n').slice(1)
  .map(l=>{const m=l.match(/^(\d+),(".*?"|[^,]*),(.*)$/);return m?{pid:m[1],title:m[2].replace(/^"|"$/g,''),sku:m[3]}:null;}).filter(Boolean);
const slugify = t => t.replace(/ \| Rebel Walls\s*$/i,'').toLowerCase().replace(/['’]/g,'').replace(/[^a-z0-9]+/g,'-').replace(/^-+|-+$/g,'');
const sh = s => `"${String(s??'').replace(/"/g,'""')}"`;
async function fetchPage(slug){
  const html = execSync(`curl -s -m 25 -w '\\n<<HTTP:%{http_code}>>' -A 'Mozilla/5.0 (compatible; DW-catalog/1.0)' "https://rebelwalls.com/${slug}"`,{encoding:'utf8',maxBuffer:16*1024*1024});
  const http=(html.match(/<<HTTP:(\d+)>>/)||[])[1]||'000';
  let ldsku='',price='',avail='';
  const m=html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/);
  if(m){try{const d=JSON.parse(m[1]);const it=(Array.isArray(d)?d:[d]).find(x=>x&&x['@type']==='Product');if(it){ldsku=it.sku||'';const off=it.offers||{};price=off.price??'';avail=String(off.availability||'').split('/').pop();}}catch{}}
  return {http,ldsku,price,avail};
}
const results=[];
for(const r of rows){
  const slug=slugify(r.title);
  let {http,ldsku,price,avail}=await fetchPage(slug);
  if(http==='200'&&!ldsku){ execSync('sleep 2'); ({http,ldsku,price,avail}=await fetchPage(slug)); } // retry once
  let cls;
  if(http==='410')cls='discontinued_410';
  else if(http==='200'&&ldsku&&Number(price)>10&&Number(price)<10000&&avail==='InStock')cls='recoverable_live';
  else if(http==='200'&&ldsku&&avail&&avail!=='InStock')cls='oos';
  else cls='no_vendor_match';
  results.push({pid:r.pid,title:r.title,sku:r.sku,slug,http,ldsku,per_sqm:price,avail,cls});
  execSync('sleep 2');
}
const cols=['pid','title','sku','slug','http','ldsku','per_sqm','avail','cls'];
writeFileSync(`${HERE}data/rescrape-39.csv`,[cols.join(',')].concat(results.map(x=>cols.map(c=>sh(x[c])).join(','))).join('\n'));
const by=results.reduce((a,x)=>{a[x.cls]=(a[x.cls]||0)+1;return a;},{});
const recov=results.filter(x=>x.cls==='recoverable_live');
writeFileSync(`${HERE}data/rescrape-39-summary.json`,JSON.stringify({ticket:'TK-10978',mode:'READ-ONLY',total:results.length,by_class:by,recoverable_live:recov.map(x=>({title:x.title,sku:x.sku,per_sqm:x.per_sqm}))},null,2));
console.log(JSON.stringify({total:results.length,by_class:by,recoverable_live:recov.length},null,2));