`tk10978-rebelwalls/rescrape-recon.mjs`

77 lines
#!/usr/bin/env node
// TK-10978 — Rebel Walls RS-494 corrected re-scrape RECON (READ-ONLY, $0, feed-first).
// Fetches the vendor's TRUE per-m2 price from JSON-LD, keyed RS<n> -> vendor R<n>.
// SAFETY NET: only accept a price when JSON-LD sku DIGITS == our RS digits (rejects wrong-slug hits).
// Writes ONLY local artifacts (data/rescrape-recon.csv + data/rescrape-summary.json).
// NO writes to Shopify or dw_unified. Gentle: 1 request / 1.5s, single-threaded (skill's stated rate).
import { writeFileSync, mkdirSync } from 'node:fs';
import { execSync } from 'node:child_process';
const HERE = new URL('.', import.meta.url).pathname;
mkdirSync(`${HERE}data`, { recursive: true });

const SQL = `BEGIN READ ONLY;
SELECT json_agg(t) FROM (
  SELECT regexp_replace(shopify_id,'.*/','') AS pid, title, variant_sku AS sample_sku, mfr_sku
  FROM shopify_products
  WHERE vendor='Rebel Walls' AND status='ACTIVE'
    AND NOT coalesce(has_product_variant,false) AND variant_sku ILIKE '%-Sample'
) t;
ROLLBACK;`;
const out = execSync('psql -h /tmp -d dw_unified -tA -v ON_ERROR_STOP=1', { input: SQL, encoding: 'utf8', maxBuffer: 64*1024*1024 });
const rows = JSON.parse(out.slice(out.indexOf('['), out.lastIndexOf(']')+1) || '[]');

const slugify = t => t.replace(/ \| Rebel Walls\s*$/i,'').toLowerCase()
  .replace(/['’]/g,'').replace(/[^a-z0-9]+/g,'-').replace(/^-+|-+$/g,'');
const digits = s => (String(s||'').match(/\d+/)||[''])[0];
const sh = s => `"${String(s??'').replace(/"/g,'""')}"`;

const results = [];
for (let i=0;i<rows.length;i++){
  const r = rows[i];
  const slug = slugify(r.title);
  const rsDig = digits(r.mfr_sku);
  let http='ERR', ldsku='', price='', avail='', cls='needs_scraper';
  try {
    const html = execSync(`curl -s -m 25 -w '\\n<<HTTP:%{http_code}>>' -A 'Mozilla/5.0 (compatible; DW-catalog/1.0)' "https://rebelwalls.com/${slug}"`, {encoding:'utf8', maxBuffer:16*1024*1024});
    http = (html.match(/<<HTTP:(\d+)>>/)||[])[1] || '000';
    if (http==='200'){
      const m = html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/);
      if (m){
        try {
          const d = JSON.parse(m[1]);
          const it = (Array.isArray(d)?d:[d]).find(x=>x&&x['@type']==='Product');
          if (it){ ldsku=it.sku||''; const off=it.offers||{}; price=off.price??''; avail=String(off.availability||'').split('/').pop(); }
        } catch {}
      }
      if (!ldsku) cls='slug_unresolved';
      else if (digits(ldsku)!==rsDig) cls='sku_mismatch';        // wrong product -> reject
      else if (avail && avail!=='InStock') cls='oos_defer';
      else if (Number(price)>10 && Number(price)<10000) cls='recoverable';
      else cls='no_price';
    } else if (http==='410') cls='discontinued_410';
    else if (http==='404') cls='slug_404';
    else cls='http_'+http;
  } catch(e){ cls='fetch_error'; }
  results.push({ pid:r.pid, title:r.title, rs_sku:r.mfr_sku, r_map:'R'+rsDig, slug, http, ldsku, per_sqm:price, avail, cls });
  if ((i+1)%25===0) process.stderr.write(`  ...${i+1}/${rows.length}\n`);
  execSync('sleep 1.5');
}

const cols=['pid','title','rs_sku','r_map','slug','http','ldsku','per_sqm','avail','cls'];
writeFileSync(`${HERE}data/rescrape-recon.csv`, [cols.join(',')].concat(results.map(x=>cols.map(c=>sh(x[c])).join(','))).join('\n'));
const by = results.reduce((a,x)=>{a[x.cls]=(a[x.cls]||0)+1;return a;},{});
const recov = results.filter(x=>x.cls==='recoverable');
const prices = recov.map(x=>Number(x.per_sqm));
const summary = {
  ticket:'TK-10978', mode:'READ-ONLY recon — NO writes', generated_at:new Date().toISOString(),
  total:results.length, by_class:by,
  recoverable:recov.length,
  per_sqm_min: prices.length?Math.min(...prices):null,
  per_sqm_max: prices.length?Math.max(...prices):null,
  per_sqm_distinct: [...new Set(prices)].sort((a,b)=>a-b),
  defer_discontinued_or_unresolved: results.length - recov.length,
  note:'recoverable = live 200 + JSON-LD sku digits == RS digits + InStock + price in range. Everything else defers to the scraper or is vendor-discontinued.',
};
writeFileSync(`${HERE}data/rescrape-summary.json`, JSON.stringify(summary,null,2));
console.log(JSON.stringify(summary,null,2));