`scripts/scrape.js`

178 lines
#!/usr/bin/env node
/**
 * Greenland Wallcoverings feed-first scraper (LOCAL ONLY, read-only).
 *
 * Discovered API (Vue/uni-app SPA backend, no anti-bot on the API itself):
 *   BASE = https://api.greenlandwallcoverings.com/api
 *   GET /category/0                          -> full category tree
 *   GET /product/page?page&pageSize&categoryId&subCategoryIds&colors  -> paginated list {list,total,totalPage}
 *   GET /product/{id}                        -> full detail (width, repeat, fireRating, backing, weight,
 *                                               compositions, subCategories, skus[colorways], rotograph[banners])
 *   images: /downloadFile?fileName=...       -> JPEG bytes
 *
 * Enumerates ALL products via /product/page, then enriches each with /product/{id}.
 * Captures FULL page data + ALL images per Steve's hard rule.
 * Writes one product/line to ../staging/greenland.jsonl
 */
const fs = require('fs');
const path = require('path');

const BASE = 'https://api.greenlandwallcoverings.com/api';
const OUT = path.join(__dirname, '..', 'staging', 'greenland.jsonl');
const CATS_OUT = path.join(__dirname, '..', 'staging', 'greenland-categories.json');
const UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function getJson(url, tries = 4) {
  for (let i = 0; i < tries; i++) {
    try {
      const ctrl = new AbortController();
      const to = setTimeout(() => ctrl.abort(), 30000);
      const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json' }, signal: ctrl.signal });
      clearTimeout(to);
      if (!res.ok) throw new Error('HTTP ' + res.status);
      const j = await res.json();
      return j;
    } catch (e) {
      if (i === tries - 1) throw e;
      await sleep(800 * (i + 1));
    }
  }
}

const imgUrl = (fileName) => fileName ? `${BASE}/downloadFile?fileName=${encodeURIComponent(fileName)}` : null;
// downloadFile URLs come back already-formed in coverImg; raw filenames appear inside skus/rotograph
const asImage = (v) => {
  if (!v) return null;
  if (typeof v === 'string') return v.startsWith('http') ? v : imgUrl(v);
  return null;
};

async function main() {
  // 1. category tree (for labels + the "type" top-level buckets)
  const cats = await getJson(`${BASE}/category/0`);
  fs.writeFileSync(CATS_OUT, JSON.stringify(cats.data, null, 2));
  const topCats = (cats.data || []).map((c) => ({ id: c.id, label: c.label }));
  console.log('Top categories:', topCats.map((c) => `${c.id}:${c.label}`).join(', '));

  // 2. enumerate the FULL list (categoryId=0 / omitted returns all)
  const pageSize = 100;
  let page = 1;
  const listRows = [];
  while (true) {
    const j = await getJson(`${BASE}/product/page?page=${page}&pageSize=${pageSize}&categoryId=0`);
    const d = j.data || {};
    const list = d.list || [];
    listRows.push(...list);
    process.stdout.write(`\rlist page ${page}/${d.totalPage}  (${listRows.length}/${d.total})   `);
    if (page >= (d.totalPage || 1) || list.length === 0) break;
    page++;
    await sleep(150);
  }
  console.log(`\nlisted ${listRows.length} products. enriching with detail...`);

  // 3. enrich each with /product/{id}  (full specs + all images + colorways)
  const out = fs.createWriteStream(OUT);
  const catCount = {};
  let done = 0;
  for (const lr of listRows) {
    const id = lr.ID;
    let detail = null;
    try {
      const dj = await getJson(`${BASE}/product/${id}`);
      detail = dj.data;
    } catch (e) {
      console.error(`\n  detail fail id=${id}: ${e.message}`);
    }
    const d = detail || {};

    // collect ALL images: cover + rotograph banners + per-sku colorway images
    const images = [];
    const cover = lr.coverImg || d.coverImg || null;
    if (cover) images.push(cover);
    for (const r of (Array.isArray(d.rotograph) ? d.rotograph : [])) {
      const u = asImage(r.img || r.image || r.url || r.fileName || r);
      if (u) images.push(u);
    }
    // colorway objects use keys {color(int), colorDescription, model, thumbnail, imgs}
    const colorways = [];
    for (const s of (Array.isArray(d.skus) ? d.skus : [])) {
      const thumb = asImage(s.thumbnail || s.img || s.image || s.coverImg || s.fileName);
      if (thumb) images.push(thumb);
      for (const im of (Array.isArray(s.imgs) ? s.imgs : [])) { const u = asImage(im); if (u) images.push(u); }
      colorways.push({ id: s.id, model: s.model || null, color: s.colorDescription || s.colorName || null, image: thumb });
    }
    const uniqImages = [...new Set(images.filter(Boolean))];

    // subcategory breakdown by type (wallClass / wallType / newClass / news)
    const subCats = (Array.isArray(d.subCategories) ? d.subCategories : [])
      .filter((s) => s.category_name)
      .map((s) => ({ id: s.category_id, name: s.category_name, type: s.category_type }));
    const wallClass = subCats.filter((s) => s.type === 'wallClass').map((s) => s.name);
    const wallType = subCats.filter((s) => s.type === 'wallType').map((s) => s.name);

    const composition = (Array.isArray(d.compositions) ? d.compositions : [])
      .filter((c) => c.compositionName)
      .map((c) => `${c.compositionName}${c.percentage ? ' ' + c.percentage + '%' : ''}`)
      .join(', ') || null;

    // derive a real SKU: the vendor's `code` is sometimes a color ("Warm White") or a collection
    // name ("JUNGLE PULSE") instead of a style code, and occasionally the style code lands in `name`.
    // Prefer a code-looking `name`, else the first colorway model, else a code-looking `code`, else GL-<id>.
    const codeRe = /^[A-Z]{1,4}-?\d/i;
    const rawName = lr.name || d.name || null;
    const rawCode = lr.code || d.code || null;
    const firstModel = colorways.map((c) => c.model).find((m) => m && codeRe.test(m));
    let sku = null, pattern = null, collection = null, colorName = null;
    if (codeRe.test(rawName || '')) { sku = rawName; pattern = rawName; colorName = rawCode; }
    else if (firstModel) { sku = firstModel; pattern = rawName; collection = (rawCode && !codeRe.test(rawCode)) ? rawCode : null; }
    else if (codeRe.test(rawCode || '')) { sku = rawCode; pattern = rawName; }
    else { sku = 'GL-' + id; pattern = rawName; collection = rawCode; }

    const catName = lr.categoryName || d.categoryName || null;
    catCount[catName || 'Uncategorized'] = (catCount[catName || 'Uncategorized'] || 0) + 1;

    const row = {
      source: 'greenland',
      id,
      // NOTE: vendor stores pattern in `name`, color/code in `code` (sometimes swapped) — keep both raw
      name: rawName,
      code: rawCode,
      sku,                                                 // derived (see codeRe logic above)
      pattern,
      collection,                                          // vendor collection name, when `code` held one
      color: colorName,                                    // real color name, when `code` held one
      category_id: lr.categoryId || d.categoryId || null,
      category: catName,
      wall_class: wallClass.length ? wallClass : null,     // Cork / Natural / Textile etc.
      wall_type: wallType.length ? wallType : null,        // e.g. "Top 100 Cork& Wood"
      sub_categories: subCats.length ? subCats : null,
      use: d.use || null,
      width: d.width || null,
      repeat: d.repeat || null,
      minimum_order: d.minimumOrder || null,
      fire_rating: d.fireRating || null,
      backing: d.backing || null,
      weight: d.weight || null,
      composition,
      unit_price: d.unitPrice != null ? d.unitPrice : null,
      is_publish: (d.isPublish != null ? d.isPublish : lr.isPublish),
      cover_image: cover,
      images: uniqImages,
      image_count: uniqImages.length,
      colorways: colorways.length ? colorways : null,
      url: `http://m.greenlandwallcoverings.com/#/pages/detail/index?id=${id}`,
      detail_ok: !!detail,
    };
    out.write(JSON.stringify(row) + '\n');
    done++;
    if (done % 25 === 0) process.stdout.write(`\renriched ${done}/${listRows.length}   `);
    await sleep(120);
  }
  out.end();
  console.log(`\nDONE. wrote ${done} products -> ${OUT}`);
  console.log('Per-category counts:', JSON.stringify(catCount, null, 2));
}

main().catch((e) => { console.error('FATAL', e); process.exit(1); });