`scripts/scrape-full.mjs`

117 lines
#!/usr/bin/env node
// Authoritative full-catalog scrape (feed-first, $0). Two passes:
//  (1) Build joinKey -> material map from the "Wallcovering > Category" branch leaves
//      (the real material taxonomy) via the per-material /product/page listing. The
//      trailing [A-Z]{2}\d+ token (TF3492, NQ8248, DP1172, LA3001...) is the physical
//      colorway key that is stable across a collection's G0### code and a material
//      series' S### code.
//  (2) Walk detail /product/<id> for the whole catalog, explode skus[] colorways,
//      classify each by joinKey->material (fallbacks: detail wallClass, then unknown),
//      dedupe colorways by joinKey to unique physical SKUs.
// Emits data/full-catalog-skus.ndjson (one line per unique colorway) + prints
// per-material unique counts. No pricing, no names yet — pure inventory truth.
import { writeFileSync, readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const API = 'https://api.greenlandwallcoverings.com/api';
const KEY_RE = /([A-Z]{2}\d+)\s*$/i;
const joinKey = (code) => { const m = (code || '').match(KEY_RE); return m ? m[1].toUpperCase() : null; };

// ---- category tree -> leaves with full path
const tree = (await (await fetch(`${API}/category/index`)).json()).data;
const leaves = [];
(function walk(ns, path) { for (const n of ns) { const p = [...path, n.label]; if (n.children && n.children.length) walk(n.children, p); else leaves.push({ id: n.id, label: n.label, path: p }); } })(tree, []);

// real material leaves = under "Wallcovering > Category ..." (+ Engineering wallpaper for XPE)
const matLeaves = leaves.filter(l => (l.path[0] === 'Wallcovering' && l.path[1] === 'Category') || l.path[0] === 'Engineering wallpaper');

// ---- pass 1: joinKey -> material
const key2mat = new Map();
async function pageAll(catId) {
  const rows = [];
  for (let pg = 1; pg <= 30; pg++) {
    const j = await (await fetch(`${API}/product/page?categoryId=1&subCategoryIds=${catId}&pageSize=200&pageNum=${pg}`)).json();
    const list = j?.data?.list || [];
    rows.push(...list);
    if (rows.length >= (j?.data?.total || 0) || list.length === 0) break;
  }
  return rows;
}
for (const lf of matLeaves) {
  const rows = await pageAll(lf.id);
  for (const r of rows) {
    const k = joinKey(r.code);
    if (k && !key2mat.has(k)) key2mat.set(k, lf.label);
  }
}
console.log(`pass1: ${key2mat.size} colorway keys classified across ${matLeaves.length} material leaves`);

// ---- pass 2: walk detail, explode, dedupe by joinKey
// multi-signal material classifier (subcat VALUES + install-manual filename + name + code)
const MAT_KW = [
  [/paper\s*weave/i, 'Paper Weave'], [/water\s*hyacinth/i, 'Water hyacinth'], [/cotton/i, 'Cotton Yarn'],
  [/arrowroot/i, 'Arrowroot'], [/grass\s*cloth|grasscloth|\bgrass\b/i, 'Grass'], [/jacquard/i, 'Jacquard'],
  [/\bsilk\b/i, 'Silk'], [/suede/i, 'Suede'], [/velvet/i, 'Velvet'], [/raffia/i, 'Raffia'], [/hemp/i, 'Hemp'],
  [/\bcork\b/i, 'Cork'], [/sisal/i, 'Sisal'], [/abaca/i, 'Abaca'], [/linen/i, 'Linen'], [/\bjute\b/i, 'Jute'],
  [/\bwool\b/i, 'Wool'], [/wood\s*bark|\bwood\b/i, 'Wood'], [/mica/i, 'Mica'],
  [/\bxpe\b/i, 'XPE Functional Wallcoverings'], [/pe\s*weav|pe\s*woven/i, 'PE Weaving / PE Woven Material'],
  [/vinyl/i, 'Vinyl'],
];
const kw = (s) => { for (const [re, m] of MAT_KW) if (re.test(s || '')) return m; return null; };
const classifyDetail = (d) => {
  for (const s of (d.subCategories || [])) { const m = kw(s.category_name); if (m) return m; }
  const man = kw((d.installationManual || '').replace(/^.*downloadFile\?fileName=/, ''));
  if (man) return man;
  return kw(d.name) || (/^S\d/.test(d.code) ? 'Silk' : /^V\d/.test(d.code) ? 'Vinyl' : /^P\d/.test(d.code) ? 'Paper Weave' : null);
};
async function fetchId(id) {
  try { const r = await fetch(`${API}/product/${id}`, { signal: AbortSignal.timeout(20000) }); if (!r.ok) return null; const j = await r.json(); return (j.code === 0 && j.data && j.data.id) ? j.data : null; } catch { return null; }
}
const uniq = new Map(); // joinKey -> record
let id = 1, HI = 340; const CONC = 8;
async function worker() {
  while (id <= HI) {
    const d = await fetchId(id++);
    if (!d) continue;
    const detailMat = classifyDetail(d);
    const specs = {
      use: (d.use || '').trim(), width: (d.width || '').trim(), minimumOrder: (d.minimumOrder || '').trim(),
      repeat: (d.repeat || '').trim(), fireRating: (d.fireRating || '').trim(), backing: (d.backing || '').trim(),
      weight: (d.weight || '').toString().trim(),
      installUrl: d.installationManual || '', catalogUrl: d.catalog || '',
    };
    for (const s of (d.skus || [])) {
      const k = joinKey(s.model);
      if (!k) continue;
      // priority: category-branch map (authoritative) > detail multi-signal > Unknown
      const material = key2mat.get(k) || detailMat || 'Unknown';
      if (!uniq.has(k)) {
        uniq.set(k, {
          joinKey: k, mfrModel: (s.model || '').trim(), color: (s.colorDescription || '').trim(),
          material, collectionId: d.id, collectionCode: d.code, collectionName: (d.name || '').trim(),
          thumb: s.thumbnail, coverImg: d.coverImg, seenIn: [d.id], ...specs,
        });
      } else { uniq.get(k).seenIn.push(d.id); }
    }
  }
}
await Promise.all(Array.from({ length: CONC }, worker));

const rows = [...uniq.values()];
writeFileSync(join(ROOT, 'data', 'full-catalog-skus.ndjson'), rows.map(r => JSON.stringify(r)).join('\n') + '\n');

// per-material counts
const prefixMap = JSON.parse(readFileSync(join(ROOT, 'data', 'material-prefix-map.json'), 'utf8'));
const byMat = {};
for (const r of rows) { byMat[r.material] = (byMat[r.material] || 0) + 1; }
console.log(`\npass2: ${rows.length} UNIQUE physical colorways (deduped by joinKey)\n`);
console.log('MATERIAL                              PREFIX   UNIQUE SKUS');
console.log('------------------------------------- -------  -----------');
for (const [m, c] of Object.entries(byMat).sort((a, b) => b[1] - a[1])) {
  const px = prefixMap[m] || '???';
  console.log(`${m.padEnd(37)} ${px.padEnd(7)}  ${String(c).padStart(11)}`);
}
console.log(`\nwrote data/full-catalog-skus.ndjson (${rows.length} rows)`);