`scripts/expand-diff.mjs`

129 lines
#!/usr/bin/env node
// EXPAND step: re-scrape the live Cork(40) line, explode detail.skus[] colorways
// (same logic as scrape.mjs), then DIFF against the mfr_skus already in
// greenland_catalog. Emits data/expand-new.ndjson (rows NOT yet in DB) + prints
// counts. NO DB WRITE — pure discovery. $0 (open API).
import { writeFileSync, readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const API = process.env.GL_API || 'https://api.greenlandwallcoverings.com/api';
const CATEGORY_ID = 1, SUBCAT_ID = 40, PAGE_SIZE = 200;

const sleep = ms => new Promise(r => setTimeout(r, ms));
async function j(url, tries = 4) {
  for (let i = 0; i < tries; i++) {
    try {
      const r = await fetch(url, { headers: { accept: 'application/json' } });
      if (!r.ok) throw new Error('HTTP ' + r.status);
      return await r.json();
    } catch (e) { if (i === tries - 1) throw e; await sleep(600 * (i + 1)); }
  }
}
const SKU_RE = /^[A-Z]{1,4}\d{2,}[A-Z0-9-]*$/;
function splitNameCode(name, code) {
  const n = (name || '').trim(), c = (code || '').trim();
  const nSku = SKU_RE.test(n.replace(/\s+/g, '')), cSku = SKU_RE.test(c.replace(/\s+/g, ''));
  if (nSku && !cSku) return { sku: n, label: c };
  if (cSku && !nSku) return { sku: c, label: n };
  return { sku: n || c, label: (n && c && n !== c) ? c : '' };
}
function widthInches(w) {
  if (!w) return null;
  let m = w.match(/([\d.]+)\s*(?:''|"|in\b|inch)/i); if (m) return parseFloat(m[1]);
  m = w.match(/([\d.]+)\s*cm/i); if (m) return +(parseFloat(m[1]) / 2.54).toFixed(2);
  return null;
}
const norm = s => (s || '').trim().toUpperCase().replace(/\s+/g, '');

async function main() {
  // existing skus from DB (via stdin file)
  const existRaw = readFileSync('/tmp/gl_existing_skus.txt', 'utf8').split('\n').map(x => x.trim()).filter(Boolean);
  const existNorm = new Set(existRaw.map(norm));
  console.log(`existing DB mfr_skus: ${existRaw.length} (${existNorm.size} normalized)`);

  const first = await j(`${API}/product/page?categoryId=${CATEGORY_ID}&subCategoryIds=${SUBCAT_ID}&page=1&pageSize=${PAGE_SIZE}`);
  const total = first.data.total; let all = [...(first.data.list || [])];
  for (let p = 2; all.length < total; p++) {
    const d = await j(`${API}/product/page?categoryId=${CATEGORY_ID}&subCategoryIds=${SUBCAT_ID}&page=${p}&pageSize=${PAGE_SIZE}`);
    const l = d.data.list || []; if (!l.length) break; all.push(...l);
  }
  console.log(`live cork groups: ${all.length} (total=${total})`);

  const rows = []; const seen = new Set();
  let parents = 0, exploded = 0;
  for (let i = 0; i < all.length; i++) {
    const li = all[i];
    let det = {};
    try { det = (await j(`${API}/product/${li.ID}`)).data || {}; }
    catch (e) { console.warn(`  ! detail ${li.ID} failed: ${e.message}`); }
    const { sku: parentSku, label: parentLabel } = splitNameCode(det.name ?? li.name, det.code ?? li.code);
    const patternName = parentLabel || parentSku;
    const subs = (det.subCategories || []).filter(s => s.category_name && s.category_name !== '是');
    const collection = subs.find(s => s.category_type === 'wallType')?.category_name || null;
    const subcats = subs.map(s => s.category_name).join(', ') || null;
    const compName = (det.compositions || []).map(c => c.compositionName).filter(Boolean).join(', ');
    const compPct = (det.compositions?.[0]?.percentage) || null;
    const material = compName || 'Cork';
    const composition = compName ? [compName, compPct && `${compPct}%`].filter(Boolean).join(' ')
                                 : (compPct ? `Cork ${compPct}%` : 'Cork');
    const shared = {
      collection, category_name: det.categoryName || li.categoryName || 'Wallcovering',
      subcategory_names: subcats, product_type: 'Wallcovering', use: det.use || null,
      width: det.width || null, width_inches: widthInches(det.width),
      minimum_order: det.minimumOrder || null, pattern_repeat: det.repeat || null,
      fire_rating_us: det.fireRating || null, backing: det.backing || null,
      weight: det.weight != null ? String(det.weight) : null,
      material, composition,
      is_publish: det.isPublish ?? li.isPublish ?? 0,
      specs: { width: det.width, minimumOrder: det.minimumOrder, fireRating: det.fireRating, backing: det.backing, weight: det.weight, use: det.use, unitPriceCode: det.unitPrice },
    };
    const push = (r) => { if (r.mfr_sku && !seen.has(norm(r.mfr_sku))) { seen.add(norm(r.mfr_sku)); rows.push(r); } };
    const kids = Array.isArray(det.skus) ? det.skus.filter(k => k && k.model) : [];
    if (kids.length) {
      exploded++;
      for (const k of kids) {
        const img = k.thumbnail || det.coverImg || li.coverImg || null;
        push({
          gl_id: 10000000 + (k.id || 0), mfr_sku: k.model, pattern_name: patternName,
          color_name: k.colorDescription || null, ...shared, image_url: img,
          all_images: [img].filter(Boolean).join('|') || null,
          product_url: `https://m.greenlandwallcoverings.com/#/pages/detail/index?id=${li.ID}`,
          raw: { parent_id: li.ID, parent_sku: parentSku, sku: k },
        });
      }
    } else {
      parents++;
      push({
        gl_id: li.ID, mfr_sku: parentSku, pattern_name: patternName, color_name: null, ...shared,
        image_url: det.coverImg || li.coverImg || null,
        all_images: [det.coverImg || li.coverImg, det.rotograph].filter(Boolean).join('|') || null,
        product_url: `https://m.greenlandwallcoverings.com/#/pages/detail/index?id=${li.ID}`,
        raw: det && det.id ? det : li,
      });
    }
    await sleep(100);
  }
  console.log(`live scrape: ${parents} leaf + ${exploded} groups exploded → ${rows.length} total colorways`);

  // diff
  const newRows = rows.filter(r => !existNorm.has(norm(r.mfr_sku)));
  console.log(`\n>>> NEW (not in DB): ${newRows.length}`);
  console.log(`>>> already in DB : ${rows.length - newRows.length}`);
  // also: which existing DB skus are NOT in the live scrape (removed upstream)?
  const liveNorm = new Set(rows.map(r => norm(r.mfr_sku)));
  const goneFromLive = existRaw.filter(s => !liveNorm.has(norm(s)));
  console.log(`>>> in DB but NOT in live scrape: ${goneFromLive.length}`);
  if (goneFromLive.length) console.log('    (kept, not deleted):', goneFromLive.slice(0, 20).join(', '), goneFromLive.length > 20 ? '…' : '');

  writeFileSync(join(ROOT, 'data', 'expand-new.ndjson'), newRows.map(r => JSON.stringify(r)).join('\n') + (newRows.length ? '\n' : ''));
  writeFileSync(join(ROOT, 'data', 'expand-live-all.ndjson'), rows.map(r => JSON.stringify(r)).join('\n') + '\n');
  if (newRows.length) {
    console.log('\nNEW sample:');
    for (const r of newRows.slice(0, 15)) console.log(`  ${r.mfr_sku}  | ${r.pattern_name} | color=${r.color_name || '—'} | ${r.collection || '—'}`);
  }
  console.log(`\nwrote data/expand-new.ndjson (${newRows.length}) + data/expand-live-all.ndjson (${rows.length})`);
}
main().catch(e => { console.error(e); process.exit(1); });