`scripts/harvest_sdg_feed.mjs`

152 lines
#!/usr/bin/env node
// Generalized SDG .design feed harvester — FULL catalog (wallpaper AND fabric), $0, staging-only.
//
// USAGE:  node harvest_sdg_feed.mjs --domain sanderson.design --brand sanderson [--limit N]
//
// Method (feed-first, verified):
//   1. ENUMERATE the full SKU universe via /us/api/n/find?type=product&verbosity=1&limit=1000&skip=N
//      (paginates by `skip`; `result.catalog[].sku` = colorway SKUs, prefix xxW=wallpaper-code / xxF=fabric-code).
//      We collapse to DISTINCT priceable colorway base codes  ^[A-Z]{2}[WF][0-9]{4}-[0-9]{2}.
//   2. PRICE each base code at verbosity=3 via filter={"sku":"<base>"}: returns real US SSP in USD (`price`)
//      + type discriminator sdb_product_group_code_data[].label (Wallpaper|Fabric) + category_path + metadata.
//   INVARIANT (verified 1948/1948): SSP = 2 x TRADE  =>  trade = SSP/2 ; retail = trade/0.65/0.85. Money -> 2dp.
//
// HARD RULES: NEVER fabricate a price — NULL on miss (price<=0 or no US price). Checkpoint continuously to
//   pilot/<brand>_feed_harvest.jsonl (resumes, loses nothing on rate-limit/crash). Staging only — writes NO DB here;
//   emits the JSONL checkpoint which a loader turns into <brand>_catalog rows.
import https from 'https';
import fs from 'fs';

const args = Object.fromEntries(process.argv.slice(2).reduce((a,v,i,arr)=>{ if(v.startsWith('--')) a.push([v.slice(2), arr[i+1]&&!arr[i+1].startsWith('--')?arr[i+1]:true]); return a; }, []));
const DOMAIN = args.domain; const BRAND = args.brand;
if (!DOMAIN || !BRAND) { console.error('need --domain and --brand'); process.exit(1); }
const PAGE = 1000;
const DIR = new URL('..', import.meta.url).pathname;
const CKPT = `${DIR}pilot/${BRAND}_feed_harvest.jsonl`;
const ENUM_CACHE = `${DIR}pilot/${BRAND}_enum.json`;
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
const sleep = ms => new Promise(r => setTimeout(r, ms));

function get(url) {
  return new Promise(resolve => {
    const req = https.get(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } }, res => {
      let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch { resolve(null); } });
    });
    req.on('error', () => resolve(null));
    req.setTimeout(60000, () => { req.destroy(); resolve(null); });
  });
}
const enc = o => encodeURIComponent(JSON.stringify(o));
const money = n => Math.round(n * 100) / 100;

// group label -> normalized product_type. Robust to object|string|json-string shapes.
function typeOf(it) {
  let g = it.sdb_product_group_code_data;
  if (typeof g === 'string') { try { g = JSON.parse(g); } catch {} }
  let label = Array.isArray(g) && g[0] ? g[0].label : (g && g.label);
  if (!label) {
    const cp = String(it.category_path || '').toLowerCase();
    if (cp.includes('wallpaper')) label = 'Wallpaper';
    else if (cp.includes('fabric')) label = 'Fabric';
  }
  const l = String(label || '').toLowerCase();
  if (l.includes('wallpaper') || l.includes('wallcovering')) return 'wallcovering';
  if (l.includes('fabric')) return 'fabric';
  // fall back to SKU letter: xxW = wallcovering, xxF = fabric
  const m = String(it.sku || '').match(/^[A-Z]{2}([WF])/);
  if (m) return m[1] === 'W' ? 'wallcovering' : 'fabric';
  return null;
}

async function enumerate() {
  if (fs.existsSync(ENUM_CACHE)) {
    const cached = JSON.parse(fs.readFileSync(ENUM_CACHE, 'utf8'));
    console.log(`[${BRAND}] using cached enumeration: ${cached.length} base codes`);
    return cached;
  }
  // Collect PATTERN-level base codes: xxW####/xxF#### (design patterns). The feed enumerates both
  // CN_<PATTERN> pattern rows AND <PATTERN>-<NN>[UC] colorway rows; both collapse to the same pattern base.
  // Each pattern base is then fetch-expanded (verbosity=3) into ALL its priced colorway SKUs downstream.
  const base = new Set(); let skip = 0, rows = 0;
  while (true) {
    const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=1&limit=${PAGE}&skip=${skip}`);
    const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
    if (!cat.length) break;
    rows += cat.length;
    for (const c of cat) {
      const s = String(c.sku||'').replace(/^CN_/, '');
      const m = s.match(/^([A-Z]{2}[WF][0-9]{4})/);   // pattern base, colorway/CN suffix stripped
      if (m) base.add(m[1]);
    }
    skip += PAGE;
    process.stdout.write(`\r[${BRAND}] enumerated ${rows} rows -> ${base.size} distinct pattern codes`);
    if (cat.length < PAGE) break;
    await sleep(150);
  }
  console.log('');
  const arr = [...base].sort();
  fs.writeFileSync(ENUM_CACHE, JSON.stringify(arr));
  return arr;
}

(async () => {
  const patterns = await enumerate();     // pattern-level base codes (xxW####/xxF####)
  const limit = args.limit ? Math.min(Number(args.limit), patterns.length) : patterns.length;
  // resume by PATTERN: skip a pattern only if we already checkpointed at least one row for it
  const donePat = new Set();
  if (fs.existsSync(CKPT)) for (const l of fs.readFileSync(CKPT,'utf8').split('\n').filter(Boolean)) {
    try { const r = JSON.parse(l); if (r.pattern_base) donePat.add(r.pattern_base); } catch {}
  }
  const todo = patterns.slice(0, limit).filter(p => !donePat.has(p));
  console.log(`[${BRAND}] ${patterns.length} pattern codes, ${donePat.size} checkpointed, ${todo.length} to harvest`);

  const out = fs.createWriteStream(CKPT, { flags: 'a' });
  let patn=0, cwCount=0, hit=0, miss=0, wp=0, fab=0;
  for (const pbase of todo) {
    patn++;
    // Expand the pattern into ALL its colorway rows. The feed's sku-filter matches the CN_<pattern>
    // parent row, which returns every colorway child (verified: bare pattern base returns 0). limit=60
    // covers the widest colorway sets.
    const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=3&filter=${enc({sku:'CN_'+pbase})}&limit=60`);
    const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
    // real colorway rows = SKU form xxW####-## / xxF####-## (drop the CN_ pattern parent & any non-colorway).
    // De-dup by colorway code, preferring the PRICED (non-UC) variant over a $0/discontinued duplicate.
    const byCw = new Map();
    for (const x of cat) {
      const sku = String(x.sku||'');
      const m = sku.match(/^([A-Z]{2}[WF][0-9]{4}-[0-9]{2})/);
      if (!m) continue;                       // skip CN_ parent + malformed
      const cw = m[1];
      const priced = typeof x.price==='number' && x.price>0;
      const prev = byCw.get(cw);
      if (!prev || (priced && !(typeof prev.price==='number' && prev.price>0))) byCw.set(cw, x);
    }
    if (byCw.size === 0) miss++;              // pattern yielded no colorway rows at all
    for (const [cw, it] of byCw) {
      cwCount++;
      let ssp=null, trade=null, retail=null;
      if (typeof it.price==='number' && it.price>0) { ssp=money(it.price); trade=money(ssp/2); retail=money(trade/0.65/0.85); hit++; }
      else miss++;
      const ptype = typeOf(it);
      if (ptype==='wallcovering') wp++; else if (ptype==='fabric') fab++;
      out.write(JSON.stringify({
        base_code: cw,                        // colorway code = unique staging key
        pattern_base: pbase,
        mfr_sku: it.sku || cw,
        product_type: ptype,
        pattern: it.sdb_design_name || it.name || null,
        color: it.sdb_desc_colour || it.sdb_design_colour_description || it.sdb_colour_variant || null,
        collection: it.sdb_collection_name || it.sdb_product_collection || null,
        width: it.sdb_useable_width || it.sdb_usable_width_inches || null,
        length: it.sdb_standard_length_inches || it.sdb_standard_length || null,
        ssp_usd: ssp, trade_usd: trade, retail_usd: retail,
        image: it.image || null
      }) + '\n');
    }
    if (patn%25===0 || patn===todo.length) console.log(`[${BRAND}] pat ${patn}/${todo.length} colorways=${cwCount} priced=${hit} miss=${miss} wp=${wp} fab=${fab}`);
    await sleep(120);
  }
  out.end();
  console.log(`[${BRAND}] DONE patterns=${patn} colorways=${cwCount} priced=${hit} miss=${miss} wp=${wp} fab=${fab}. Checkpoint: ${CKPT}`);
})();