`scripts/scrape.mjs`

150 lines
#!/usr/bin/env node
// Feed-first Greenland cork scraper. $0 — plain fetch of the open JSON API,
// no browser / no auth. Emits data/upsert.sql (piped to psql by run.sh).
import { writeFileSync } from 'node:fs';

const API = process.env.GL_API || 'https://api.greenlandwallcoverings.com/api';
const CATEGORY_ID = 1;        // Wallcovering tab
const SUBCAT_ID = 40;         // Crafted > Handmade > Cork
const PAGE_SIZE = 200;

const sleep = ms => new Promise(r => setTimeout(r, ms));
async function j(url, tries = 4) {
  for (let i = 0; i < tries; i++) {
    try {
      const r = await fetch(url, { headers: { 'accept': 'application/json' } });
      if (!r.ok) throw new Error('HTTP ' + r.status);
      return await r.json();
    } catch (e) {
      if (i === tries - 1) throw e;
      await sleep(600 * (i + 1));
    }
  }
}

// name/code are inconsistently swapped in their data: one is a SKU code
// (e.g. S300NQ8319 / V020P1455 — leading letters + digits, no spaces),
// the other is a human color/pattern name (e.g. "Warm White").
const SKU_RE = /^[A-Z]{1,4}\d{2,}[A-Z0-9-]*$/;
function splitNameCode(name, code) {
  const n = (name || '').trim(), c = (code || '').trim();
  const nSku = SKU_RE.test(n.replace(/\s+/g, '')), cSku = SKU_RE.test(c.replace(/\s+/g, ''));
  if (nSku && !cSku) return { sku: n, label: c };
  if (cSku && !nSku) return { sku: c, label: n };
  // both or neither look SKU-ish: prefer `name` as SKU (matches their PDP header)
  return { sku: n || c, label: (n && c && n !== c) ? c : '' };
}
function widthInches(w) {
  if (!w) return null;
  let m = w.match(/([\d.]+)\s*(?:''|"|in\b|inch)/i); if (m) return parseFloat(m[1]);
  m = w.match(/([\d.]+)\s*cm/i); if (m) return +(parseFloat(m[1]) / 2.54).toFixed(2);
  return null;
}
const esc = v => v === null || v === undefined ? 'NULL' : `'${String(v).replace(/'/g, "''")}'`;
const jesc = o => o === null || o === undefined ? 'NULL' : `'${JSON.stringify(o).replace(/'/g, "''")}'::jsonb`;

async function main() {
  console.log('→ fetching Cork list (categoryId=%d subCat=%d)…', CATEGORY_ID, SUBCAT_ID);
  const first = await j(`${API}/product/page?categoryId=${CATEGORY_ID}&subCategoryIds=${SUBCAT_ID}&page=1&pageSize=${PAGE_SIZE}`);
  const total = first.data.total, list = first.data.list || [];
  console.log(`   total=${total}, page1=${list.length}`);
  // pull remaining pages if any
  let all = [...list];
  for (let p = 2; all.length < total; p++) {
    const d = await j(`${API}/product/page?categoryId=${CATEGORY_ID}&subCategoryIds=${SUBCAT_ID}&page=${p}&pageSize=${PAGE_SIZE}`);
    const l = d.data.list || []; if (!l.length) break; all.push(...l);
  }
  console.log(`   collected ${all.length} list rows; fetching details…`);

  const rows = [];
  const seen = new Set();
  let parents = 0, exploded = 0;
  for (let i = 0; i < all.length; i++) {
    const li = all[i];
    let det = {};
    try { det = (await j(`${API}/product/${li.ID}`)).data || {}; }
    catch (e) { console.warn(`   ! detail ${li.ID} failed: ${e.message}`); }
    const { sku: parentSku, label: parentLabel } = splitNameCode(det.name ?? li.name, det.code ?? li.code);
    const patternName = parentLabel || parentSku;
    const subs = (det.subCategories || []).filter(s => s.category_name && s.category_name !== '是');
    const collection = subs.find(s => s.category_type === 'wallType')?.category_name || null;
    const subcats = subs.map(s => s.category_name).join(', ') || null;
    // material: compositions rarely name the fiber; this is the Cork leaf (subCat 40) → material IS Cork.
    const compName = (det.compositions || []).map(c => c.compositionName).filter(Boolean).join(', ');
    const compPct = (det.compositions?.[0]?.percentage) || null;
    const material = compName || 'Cork';
    const composition = compName ? [compName, compPct && `${compPct}%`].filter(Boolean).join(' ')
                                 : (compPct ? `Cork ${compPct}%` : 'Cork');
    const shared = {
      collection, category_name: det.categoryName || li.categoryName || 'Wallcovering',
      subcategory_names: subcats, product_type: 'Wallcovering', use: det.use || null,
      width: det.width || null, width_inches: widthInches(det.width),
      minimum_order: det.minimumOrder || null, pattern_repeat: det.repeat || null,
      fire_rating_us: det.fireRating || null, backing: det.backing || null,
      weight: det.weight != null ? String(det.weight) : null,
      material, composition,
      is_publish: det.isPublish ?? li.isPublish ?? 0,
      specs: { width: det.width, minimumOrder: det.minimumOrder, fireRating: det.fireRating, backing: det.backing, weight: det.weight, use: det.use, unitPriceCode: det.unitPrice },
    };
    const push = (r) => { if (r.mfr_sku && !seen.has(r.mfr_sku)) { seen.add(r.mfr_sku); rows.push(r); } };
    const kids = Array.isArray(det.skus) ? det.skus.filter(k => k && k.model) : [];
    if (kids.length) {
      // pattern-group → explode each colorway child into its own sellable SKU
      exploded++;
      for (const k of kids) {
        const img = k.thumbnail || det.coverImg || li.coverImg || null;
        push({
          gl_id: 10000000 + (k.id || 0),          // offset to never collide with parent product IDs
          mfr_sku: k.model,
          pattern_name: patternName,
          color_name: k.colorDescription || null,
          ...shared,
          image_url: img,
          all_images: [img].filter(Boolean).join('|') || null,
          product_url: `https://m.greenlandwallcoverings.com/#/pages/detail/index?id=${li.ID}`,
          raw: { parent_id: li.ID, parent_sku: parentSku, sku: k },
        });
      }
    } else {
      // leaf product — the group IS the SKU
      parents++;
      push({
        gl_id: li.ID, mfr_sku: parentSku, pattern_name: patternName, color_name: null,
        ...shared,
        image_url: det.coverImg || li.coverImg || null,
        all_images: [det.coverImg || li.coverImg, det.rotograph].filter(Boolean).join('|') || null,
        product_url: `https://m.greenlandwallcoverings.com/#/pages/detail/index?id=${li.ID}`,
        raw: det && det.id ? det : li,
      });
    }
    if ((i + 1) % 20 === 0) console.log(`   …${i + 1}/${all.length} groups (${rows.length} SKUs so far)`);
    await sleep(120);
  }
  console.log(`   ${parents} leaf SKUs + ${exploded} pattern-groups exploded → ${rows.length} total SKUs`);

  writeFileSync(new URL('../data/greenland_cork.ndjson', import.meta.url), rows.map(r => JSON.stringify(r)).join('\n'));
  const cols = ['gl_id','mfr_sku','pattern_name','color_name','collection','category_name','subcategory_names','product_type','use','width','width_inches','minimum_order','pattern_repeat','fire_rating_us','backing','weight','material','composition','image_url','all_images','product_url','is_publish','specs','raw'];
  const values = rows.map(r => '(' + cols.map(c => {
    const v = r[c];
    if (c === 'specs' || c === 'raw') return jesc(v);
    if (c === 'width_inches' || c === 'is_publish' || c === 'gl_id') return v == null ? 'NULL' : v;
    return esc(v);
  }).join(',') + ')').join(',\n');
  const sql = `BEGIN;
INSERT INTO greenland_catalog (${cols.join(',')}) VALUES
${values}
ON CONFLICT (gl_id) DO UPDATE SET
  mfr_sku=EXCLUDED.mfr_sku, pattern_name=EXCLUDED.pattern_name, color_name=EXCLUDED.color_name,
  collection=EXCLUDED.collection, subcategory_names=EXCLUDED.subcategory_names, use=EXCLUDED.use,
  width=EXCLUDED.width, width_inches=EXCLUDED.width_inches, minimum_order=EXCLUDED.minimum_order,
  pattern_repeat=EXCLUDED.pattern_repeat, fire_rating_us=EXCLUDED.fire_rating_us, backing=EXCLUDED.backing,
  weight=EXCLUDED.weight, material=EXCLUDED.material, composition=EXCLUDED.composition, image_url=EXCLUDED.image_url,
  all_images=EXCLUDED.all_images, product_url=EXCLUDED.product_url, is_publish=EXCLUDED.is_publish,
  specs=EXCLUDED.specs, raw=EXCLUDED.raw, updated_at=now();
COMMIT;
`;
  writeFileSync(new URL('../data/upsert.sql', import.meta.url), sql);
  console.log(`✓ wrote ${rows.length} rows → data/greenland_cork.ndjson + data/upsert.sql`);
}
main().catch(e => { console.error(e); process.exit(1); });