scaffold greenland-china: feed-first scraper + Sangetsu-pattern viewer

fc9618a477eeaf0d561811ceba2c207e17e77d4c · 2026-06-30 09:27:16 -0700 · Steve Abrams

API discovered via SPA JS bundle reverse-engineering:
base=https://api.greenlandwallcoverings.com/api
/product/page (paginated, total 880), /product/{id} (full specs),
/category/0 (tree), /downloadFile (images). No anti-bot on the API.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

A .gitignore
A scripts/scrape.js
A viewer/public/index.html
A viewer/server.js

Diff

commit fc9618a477eeaf0d561811ceba2c207e17e77d4c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jun 30 09:27:16 2026 -0700

    scaffold greenland-china: feed-first scraper + Sangetsu-pattern viewer
    
    API discovered via SPA JS bundle reverse-engineering:
    base=https://api.greenlandwallcoverings.com/api
    /product/page (paginated, total 880), /product/{id} (full specs),
    /category/0 (tree), /downloadFile (images). No anti-bot on the API.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .gitignore               |   8 +++
 scripts/scrape.js        | 160 +++++++++++++++++++++++++++++++++++++++++++++++
 viewer/public/index.html | 143 ++++++++++++++++++++++++++++++++++++++++++
 viewer/server.js         | 120 +++++++++++++++++++++++++++++++++++
 4 files changed, 431 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/scripts/scrape.js b/scripts/scrape.js
new file mode 100644
index 0000000..694fd51
--- /dev/null
+++ b/scripts/scrape.js
@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+/**
+ * Greenland Wallcoverings feed-first scraper (LOCAL ONLY, read-only).
+ *
+ * Discovered API (Vue/uni-app SPA backend, no anti-bot on the API itself):
+ *   BASE = https://api.greenlandwallcoverings.com/api
+ *   GET /category/0                          -> full category tree
+ *   GET /product/page?page&pageSize&categoryId&subCategoryIds&colors  -> paginated list {list,total,totalPage}
+ *   GET /product/{id}                        -> full detail (width, repeat, fireRating, backing, weight,
+ *                                               compositions, subCategories, skus[colorways], rotograph[banners])
+ *   images: /downloadFile?fileName=...       -> JPEG bytes
+ *
+ * Enumerates ALL products via /product/page, then enriches each with /product/{id}.
+ * Captures FULL page data + ALL images per Steve's hard rule.
+ * Writes one product/line to ../staging/greenland.jsonl
+ */
+const fs = require('fs');
+const path = require('path');
+
+const BASE = 'https://api.greenlandwallcoverings.com/api';
+const OUT = path.join(__dirname, '..', 'staging', 'greenland.jsonl');
+const CATS_OUT = path.join(__dirname, '..', 'staging', 'greenland-categories.json');
+const UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1';
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+async function getJson(url, tries = 4) {
+  for (let i = 0; i < tries; i++) {
+    try {
+      const ctrl = new AbortController();
+      const to = setTimeout(() => ctrl.abort(), 30000);
+      const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json' }, signal: ctrl.signal });
+      clearTimeout(to);
+      if (!res.ok) throw new Error('HTTP ' + res.status);
+      const j = await res.json();
+      return j;
+    } catch (e) {
+      if (i === tries - 1) throw e;
+      await sleep(800 * (i + 1));
+    }
+  }
+}
+
+const imgUrl = (fileName) => fileName ? `${BASE}/downloadFile?fileName=${encodeURIComponent(fileName)}` : null;
+// downloadFile URLs come back already-formed in coverImg; raw filenames appear inside skus/rotograph
+const asImage = (v) => {
+  if (!v) return null;
+  if (typeof v === 'string') return v.startsWith('http') ? v : imgUrl(v);
+  return null;
+};
+
+async function main() {
+  // 1. category tree (for labels + the "type" top-level buckets)
+  const cats = await getJson(`${BASE}/category/0`);
+  fs.writeFileSync(CATS_OUT, JSON.stringify(cats.data, null, 2));
+  const topCats = (cats.data || []).map((c) => ({ id: c.id, label: c.label }));
+  console.log('Top categories:', topCats.map((c) => `${c.id}:${c.label}`).join(', '));
+
+  // 2. enumerate the FULL list (categoryId=0 / omitted returns all)
+  const pageSize = 100;
+  let page = 1;
+  const listRows = [];
+  while (true) {
+    const j = await getJson(`${BASE}/product/page?page=${page}&pageSize=${pageSize}&categoryId=0`);
+    const d = j.data || {};
+    const list = d.list || [];
+    listRows.push(...list);
+    process.stdout.write(`\rlist page ${page}/${d.totalPage}  (${listRows.length}/${d.total})   `);
+    if (page >= (d.totalPage || 1) || list.length === 0) break;
+    page++;
+    await sleep(150);
+  }
+  console.log(`\nlisted ${listRows.length} products. enriching with detail...`);
+
+  // 3. enrich each with /product/{id}  (full specs + all images + colorways)
+  const out = fs.createWriteStream(OUT);
+  const catCount = {};
+  let done = 0;
+  for (const lr of listRows) {
+    const id = lr.ID;
+    let detail = null;
+    try {
+      const dj = await getJson(`${BASE}/product/${id}`);
+      detail = dj.data;
+    } catch (e) {
+      console.error(`\n  detail fail id=${id}: ${e.message}`);
+    }
+    const d = detail || {};
+
+    // collect ALL images: cover + rotograph banners + per-sku colorway images
+    const images = [];
+    const cover = lr.coverImg || d.coverImg || null;
+    if (cover) images.push(cover);
+    for (const r of (Array.isArray(d.rotograph) ? d.rotograph : [])) {
+      const u = asImage(r.img || r.image || r.url || r.fileName || r);
+      if (u) images.push(u);
+    }
+    const colorways = [];
+    for (const s of (Array.isArray(d.skus) ? d.skus : [])) {
+      const u = asImage(s.img || s.image || s.coverImg || s.fileName);
+      if (u) images.push(u);
+      colorways.push({ id: s.id, color: s.color || s.colorName || s.name || null, code: s.code || null, image: u });
+    }
+    const uniqImages = [...new Set(images.filter(Boolean))];
+
+    // subcategory breakdown by type (wallClass / wallType / newClass / news)
+    const subCats = (Array.isArray(d.subCategories) ? d.subCategories : [])
+      .filter((s) => s.category_name)
+      .map((s) => ({ id: s.category_id, name: s.category_name, type: s.category_type }));
+    const wallClass = subCats.filter((s) => s.type === 'wallClass').map((s) => s.name);
+    const wallType = subCats.filter((s) => s.type === 'wallType').map((s) => s.name);
+
+    const composition = (Array.isArray(d.compositions) ? d.compositions : [])
+      .filter((c) => c.compositionName)
+      .map((c) => `${c.compositionName}${c.percentage ? ' ' + c.percentage + '%' : ''}`)
+      .join(', ') || null;
+
+    const catName = lr.categoryName || d.categoryName || null;
+    catCount[catName || 'Uncategorized'] = (catCount[catName || 'Uncategorized'] || 0) + 1;
+
+    const row = {
+      source: 'greenland',
+      id,
+      // NOTE: vendor stores pattern in `name`, color/code in `code` (sometimes swapped) — keep both raw
+      name: lr.name || d.name || null,
+      code: lr.code || d.code || null,
+      sku: d.code || lr.code || lr.name || String(id),     // mfr code is the closest thing to a SKU
+      pattern: lr.name || d.name || null,
+      category_id: lr.categoryId || d.categoryId || null,
+      category: catName,
+      wall_class: wallClass.length ? wallClass : null,     // Cork / Natural / Textile etc.
+      wall_type: wallType.length ? wallType : null,        // e.g. "Top 100 Cork& Wood"
+      sub_categories: subCats.length ? subCats : null,
+      use: d.use || null,
+      width: d.width || null,
+      repeat: d.repeat || null,
+      minimum_order: d.minimumOrder || null,
+      fire_rating: d.fireRating || null,
+      backing: d.backing || null,
+      weight: d.weight || null,
+      composition,
+      unit_price: d.unitPrice != null ? d.unitPrice : null,
+      is_publish: (d.isPublish != null ? d.isPublish : lr.isPublish),
+      cover_image: cover,
+      images: uniqImages,
+      image_count: uniqImages.length,
+      colorways: colorways.length ? colorways : null,
+      url: `http://m.greenlandwallcoverings.com/#/pages/detail/index?id=${id}`,
+      detail_ok: !!detail,
+    };
+    out.write(JSON.stringify(row) + '\n');
+    done++;
+    if (done % 25 === 0) process.stdout.write(`\renriched ${done}/${listRows.length}   `);
+    await sleep(120);
+  }
+  out.end();
+  console.log(`\nDONE. wrote ${done} products -> ${OUT}`);
+  console.log('Per-category counts:', JSON.stringify(catCount, null, 2));
+}
+
+main().catch((e) => { console.error('FATAL', e); process.exit(1); });
diff --git a/viewer/public/index.html b/viewer/public/index.html
new file mode 100644
index 0000000..0bea6d0
--- /dev/null
+++ b/viewer/public/index.html
@@ -0,0 +1,143 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Greenland Wallcoverings — Staged SKUs (offline preview)</title>
+<style>
+  :root { --cols: 6; --bg:#0d0f0d; --card:#161b16; --line:#2a302a; --txt:#e8eae8; --mut:#9aa29a; --accent:#7fb069; }
+  * { box-sizing: border-box; }
+  body { margin:0; background:var(--bg); color:var(--txt); font:14px/1.4 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; }
+  header { position:sticky; top:0; z-index:10; background:rgba(13,15,13,.96); backdrop-filter:blur(8px); border-bottom:1px solid var(--line); padding:14px 20px; }
+  h1 { margin:0 0 10px; font-size:18px; font-weight:600; letter-spacing:.02em; }
+  h1 small { color:var(--mut); font-weight:400; font-size:13px; margin-left:8px; }
+  .controls { display:flex; flex-wrap:wrap; gap:10px 14px; align-items:center; }
+  select, input[type=search] { background:var(--card); color:var(--txt); border:1px solid var(--line); border-radius:8px; padding:7px 10px; font-size:13px; }
+  input[type=search] { min-width:220px; }
+  .dens { display:flex; align-items:center; gap:8px; color:var(--mut); }
+  .grid { display:grid; grid-template-columns:repeat(var(--cols),1fr); gap:12px; padding:18px 20px 60px; }
+  .card { background:var(--card); border:1px solid var(--line); border-radius:10px; overflow:hidden; display:flex; flex-direction:column; }
+  .thumb { aspect-ratio:1/1; background:#000 center/cover no-repeat; position:relative; }
+  .thumb.empty { display:flex; align-items:center; justify-content:center; color:#444; font-size:11px; }
+  .badge { position:absolute; top:6px; left:6px; font-size:10px; font-weight:700; letter-spacing:.04em; padding:2px 7px; border-radius:20px; text-transform:uppercase; background:#2f7d4f; color:#fff; }
+  .badge.imgs { position:static; background:#2a302a; color:#9aa29a; font-size:9px; padding:1px 5px; margin-left:5px; border-radius:10px; vertical-align:middle; text-transform:none; }
+  .meta { padding:8px 10px; }
+  .sku { font-weight:700; font-size:13px; color:var(--accent); word-break:break-all; }
+  .ttl { color:var(--txt); font-size:12px; margin-top:3px; max-height:2.8em; overflow:hidden; }
+  .color { color:var(--mut); font-size:11px; margin-top:2px; }
+  .cat { margin-top:4px; }
+  .cat span { display:inline-block; background:#1e241e; color:#9aa29a; font-size:9.5px; padding:1px 6px; border-radius:10px; margin-right:4px; margin-top:2px; }
+  .spec { margin-top:5px; font-size:11px; color:#777; display:flex; gap:8px; flex-wrap:wrap; }
+  #sentinel { height:1px; }
+  #status { text-align:center; color:var(--mut); padding:16px; font-size:13px; }
+</style>
+</head>
+<body>
+<header>
+  <h1>Greenland Wallcoverings <small id="sub">staged offline · read-only preview</small></h1>
+  <div class="controls">
+    <label class="dens">Category
+      <select id="cat"><option value="all">All categories</option></select>
+    </label>
+    <input type="search" id="q" placeholder="Search SKU, pattern, color, material…">
+    <label class="dens">Sort
+      <select id="sort">
+        <option value="newest">Newest</option>
+        <option value="sku">SKU A→Z</option>
+        <option value="title">Title A→Z</option>
+        <option value="width">Width ↑</option>
+        <option value="weight">Weight ↑</option>
+        <option value="images">Most images</option>
+      </select>
+    </label>
+    <label class="dens">Density
+      <input type="range" id="dens" min="3" max="10" value="6">
+    </label>
+  </div>
+</header>
+<div class="grid" id="grid"></div>
+<div id="sentinel"></div>
+<div id="status">loading…</div>
+
+<script>
+const grid = document.getElementById('grid');
+const statusEl = document.getElementById('status');
+const subEl = document.getElementById('sub');
+const catSel = document.getElementById('cat');
+const LIMIT = 120;
+let state = {
+  category: localStorage.getItem('gl_cat') || 'all',
+  sort: localStorage.getItem('gl_sort') || 'newest',
+  q: '',
+  offset: 0, total: 0, loading: false, done: false, catsBuilt: false,
+};
+
+// density persists
+const dens = document.getElementById('dens');
+dens.value = localStorage.getItem('gl_cols') || 6;
+document.documentElement.style.setProperty('--cols', dens.value);
+dens.oninput = () => { document.documentElement.style.setProperty('--cols', dens.value); localStorage.setItem('gl_cols', dens.value); };
+
+document.getElementById('sort').value = state.sort;
+
+function reset() { grid.innerHTML=''; state.offset=0; state.done=false; statusEl.textContent='loading…'; load(); }
+
+catSel.onchange = (e) => { state.category = e.target.value; localStorage.setItem('gl_cat', state.category); reset(); };
+document.getElementById('sort').onchange = (e) => { state.sort = e.target.value; localStorage.setItem('gl_sort', state.sort); reset(); };
+let qt; document.getElementById('q').oninput = (e) => { clearTimeout(qt); qt = setTimeout(() => { state.q = e.target.value.trim(); reset(); }, 250); };
+
+// same-origin proxy fallback so the grid renders even if CDN hot-link-blocks
+function imgSrc(u) { return u ? ('/img?u=' + encodeURIComponent(u)) : null; }
+
+function card(r) {
+  const el = document.createElement('div'); el.className='card';
+  const imgBadge = r.image_count>1 ? `<span class="badge imgs">📷 ${r.image_count}</span>` : '';
+  const src = imgSrc(r.image);
+  const thumb = src
+    ? `<div class="thumb" style="background-image:url('${src}')"><span class="badge">Greenland</span></div>`
+    : `<div class="thumb empty"><span class="badge">Greenland</span>no image</div>`;
+  const cats = [r.category, r.wall_class, r.wall_type].filter(Boolean).map(s=>`<span>${s}</span>`).join('');
+  const specs = r.specs.map(s=>`<span>${s}</span>`).join('');
+  el.innerHTML = `${thumb}<div class="meta">
+    <div class="sku">${r.sku||'—'}${imgBadge}</div>
+    <div class="ttl" title="${(r.title||'').replace(/"/g,'&quot;')}">${r.title||''}</div>
+    ${r.color && r.color!==r.title?`<div class="color">${r.color}</div>`:''}
+    ${cats?`<div class="cat">${cats}</div>`:''}
+    <div class="spec">${specs}</div>
+  </div>`;
+  if (r.url) { el.style.cursor='pointer'; el.onclick=()=>window.open(r.url,'_blank'); }
+  return el;
+}
+
+function buildCats(categories) {
+  if (state.catsBuilt) return;
+  const entries = Object.entries(categories).sort((a,b)=>b[1]-a[1]);
+  for (const [name,n] of entries) {
+    const o = document.createElement('option'); o.value=name; o.textContent=`${name} (${n})`; catSel.appendChild(o);
+  }
+  catSel.value = state.category;
+  state.catsBuilt = true;
+}
+
+async function load() {
+  if (state.loading || state.done) return;
+  state.loading = true;
+  const p = new URLSearchParams({ category:state.category, sort:state.sort, q:state.q, offset:state.offset, limit:LIMIT });
+  const res = await fetch('/api/skus?'+p); const data = await res.json();
+  buildCats(data.categories);
+  state.total = data.total;
+  const frag = document.createDocumentFragment();
+  data.rows.forEach(r => frag.appendChild(card(r)));
+  grid.appendChild(frag);
+  state.offset += data.rows.length;
+  state.done = state.offset >= data.total;
+  state.loading = false;
+  subEl.textContent = `staged offline · ${data.all.toLocaleString()} SKUs total · showing ${state.offset.toLocaleString()}/${state.total.toLocaleString()}`;
+  statusEl.textContent = state.done ? `— end · ${state.total.toLocaleString()} SKUs —` : 'scroll for more…';
+}
+
+new IntersectionObserver((es)=>{ if (es[0].isIntersecting) load(); }, { rootMargin:'600px' }).observe(document.getElementById('sentinel'));
+load();
+</script>
+</body>
+</html>
diff --git a/viewer/server.js b/viewer/server.js
new file mode 100644
index 0000000..64f6c35
--- /dev/null
+++ b/viewer/server.js
@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+/**
+ * Staging viewer for the Greenland Wallcoverings onboarding catalog (OFFLINE / read-only).
+ * Reads ../staging/greenland.jsonl, flattens to uniform SKU rows, and serves a paginated
+ * /api/skus endpoint behind a sort + density + infinite-scroll grid.  Mirrors the
+ * Sangetsu/Lilycolor viewer pattern.
+ *
+ * Images: Greenland's CDN (api.greenlandwallcoverings.com/api/downloadFile) is referenced
+ * directly; a /img proxy provides a same-origin fallback so the grid renders even if the
+ * remote CDN hot-link-blocks a browser.
+ *
+ * HARD: read-only. No Shopify, no dw_unified, no publish. Pure local preview of staged data.
+ *   node viewer/server.js [port]   (default 0 = OS-assigned free port)
+ */
+const http = require('http');
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+const PORT = parseInt(process.argv[2] || '0', 10);
+const DIR = path.join(__dirname, '..', 'staging');
+const GREEN = path.join(DIR, 'greenland.jsonl');
+const PUBLIC = path.join(__dirname, 'public');
+
+const readJsonl = (f) => (fs.existsSync(f)
+  ? fs.readFileSync(f, 'utf8').trim().split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean)
+  : []);
+
+function buildRows() {
+  const rows = [];
+  for (const r of readJsonl(GREEN)) {
+    const specs = [r.width, r.repeat, r.fire_rating, r.backing, r.weight ? r.weight + ' g' : null].filter(Boolean);
+    rows.push({
+      source: 'greenland',
+      id: r.id,
+      sku: r.sku || r.code || String(r.id),
+      title: r.pattern || r.name || r.sku || String(r.id),
+      color: r.code || null,
+      category: r.category || null,
+      wall_class: (r.wall_class && r.wall_class.length) ? r.wall_class.join(', ') : null,
+      wall_type: (r.wall_type && r.wall_type.length) ? r.wall_type.join(', ') : null,
+      width: r.width || null,
+      repeat: r.repeat || null,
+      fire: r.fire_rating || null,
+      backing: r.backing || null,
+      weight: r.weight || null,
+      composition: r.composition || null,
+      min_order: r.minimum_order || null,
+      specs,
+      image: r.cover_image || (r.images && r.images[0]) || null,
+      image_count: r.image_count || (r.images ? r.images.length : 0),
+      colorway_count: r.colorways ? r.colorways.length : 0,
+      url: r.url || null,
+    });
+  }
+  return rows;
+}
+
+let ROWS = buildRows();
+fs.watchFile(GREEN, { interval: 5000 }, () => { ROWS = buildRows(); });
+
+const sorters = {
+  newest: null, // server natural order (list order from vendor)
+  sku: (a, b) => String(a.sku).localeCompare(String(b.sku)),
+  title: (a, b) => String(a.title).localeCompare(String(b.title)),
+  width: (a, b) => (parseFloat(a.width) || 0) - (parseFloat(b.width) || 0),
+  weight: (a, b) => (a.weight || 0) - (b.weight || 0),
+  images: (a, b) => (b.image_count || 0) - (a.image_count || 0),
+};
+
+const categoriesOf = (rows) => {
+  const m = {};
+  for (const r of rows) { const c = r.category || 'Uncategorized'; m[c] = (m[c] || 0) + 1; }
+  return m;
+};
+
+const server = http.createServer((req, res) => {
+  const u = new URL(req.url, `http://localhost`);
+
+  if (u.pathname === '/api/skus') {
+    const cat = u.searchParams.get('category') || 'all';
+    const q = (u.searchParams.get('q') || '').toLowerCase();
+    const sort = u.searchParams.get('sort') || 'newest';
+    const offset = parseInt(u.searchParams.get('offset') || '0', 10);
+    const limit = Math.min(parseInt(u.searchParams.get('limit') || '120', 10), 500);
+    let rows = ROWS;
+    if (cat !== 'all') rows = rows.filter((r) => (r.category || 'Uncategorized') === cat);
+    if (q) rows = rows.filter((r) => (r.sku + ' ' + r.title + ' ' + (r.color || '') + ' ' + (r.wall_class || '')).toLowerCase().includes(q));
+    if (sorters[sort]) rows = [...rows].sort(sorters[sort]);
+    const page = rows.slice(offset, offset + limit);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ total: rows.length, all: ROWS.length, categories: categoriesOf(ROWS), offset, limit, rows: page }));
+    return;
+  }
+
+  // same-origin image proxy fallback (CDN hot-link safety)
+  if (u.pathname === '/img') {
+    const target = u.searchParams.get('u');
+    if (!target || !/^https:\/\/api\.greenlandwallcoverings\.com\//.test(target)) { res.writeHead(400); return res.end('bad'); }
+    https.get(target, { headers: { 'User-Agent': 'Mozilla/5.0', Referer: 'http://m.greenlandwallcoverings.com/' } }, (pr) => {
+      res.writeHead(pr.statusCode || 200, { 'Content-Type': pr.headers['content-type'] || 'image/jpeg', 'Cache-Control': 'max-age=86400' });
+      pr.pipe(res);
+    }).on('error', () => { res.writeHead(502); res.end('img err'); });
+    return;
+  }
+
+  // static
+  let p = u.pathname === '/' ? '/index.html' : u.pathname;
+  const file = path.join(PUBLIC, path.normalize(p).replace(/^(\.\.[/\\])+/, ''));
+  if (!file.startsWith(PUBLIC) || !fs.existsSync(file)) { res.writeHead(404); res.end('not found'); return; }
+  const ext = path.extname(file);
+  const mime = { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css' }[ext] || 'text/plain';
+  res.writeHead(200, { 'Content-Type': mime });
+  res.end(fs.readFileSync(file));
+});
+
+server.listen(PORT, () => {
+  const addr = server.address();
+  console.log(`Greenland staging viewer → http://localhost:${addr.port}  (${ROWS.length} SKUs)`);
+});

(oldest) · back to Greenland China · auto-save: 2026-06-30T09:39:21 (1 files) — staging/ 69b68a9 →