← back to Sanderson Onboard
TK-10877: generalized SDG feed harvester (full wp+fabric enumeration, SSP/2 trade, retail derive, checkpointed)
004cd214a80892a85fd36e01c1e1eb0df462f8c7 · 2026-08-26 09:34:05 -0700 · Steve Abrams
Files touched
A scripts/harvest_sdg_feed.mjs
Diff
commit 004cd214a80892a85fd36e01c1e1eb0df462f8c7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 26 09:34:05 2026 -0700
TK-10877: generalized SDG feed harvester (full wp+fabric enumeration, SSP/2 trade, retail derive, checkpointed)
---
scripts/harvest_sdg_feed.mjs | 126 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 126 insertions(+)
diff --git a/scripts/harvest_sdg_feed.mjs b/scripts/harvest_sdg_feed.mjs
new file mode 100644
index 0000000..02a410c
--- /dev/null
+++ b/scripts/harvest_sdg_feed.mjs
@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+// Generalized SDG .design feed harvester — FULL catalog (wallpaper AND fabric), $0, staging-only.
+//
+// USAGE: node harvest_sdg_feed.mjs --domain sanderson.design --brand sanderson [--limit N]
+//
+// Method (feed-first, verified):
+// 1. ENUMERATE the full SKU universe via /us/api/n/find?type=product&verbosity=1&limit=1000&skip=N
+// (paginates by `skip`; `result.catalog[].sku` = colorway SKUs, prefix xxW=wallpaper-code / xxF=fabric-code).
+// We collapse to DISTINCT priceable colorway base codes ^[A-Z]{2}[WF][0-9]{4}-[0-9]{2}.
+// 2. PRICE each base code at verbosity=3 via filter={"sku":"<base>"}: returns real US SSP in USD (`price`)
+// + type discriminator sdb_product_group_code_data[].label (Wallpaper|Fabric) + category_path + metadata.
+// INVARIANT (verified 1948/1948): SSP = 2 x TRADE => trade = SSP/2 ; retail = trade/0.65/0.85. Money -> 2dp.
+//
+// HARD RULES: NEVER fabricate a price — NULL on miss (price<=0 or no US price). Checkpoint continuously to
+// pilot/<brand>_feed_harvest.jsonl (resumes, loses nothing on rate-limit/crash). Staging only — writes NO DB here;
+// emits the JSONL checkpoint which a loader turns into <brand>_catalog rows.
+import https from 'https';
+import fs from 'fs';
+
+const args = Object.fromEntries(process.argv.slice(2).reduce((a,v,i,arr)=>{ if(v.startsWith('--')) a.push([v.slice(2), arr[i+1]&&!arr[i+1].startsWith('--')?arr[i+1]:true]); return a; }, []));
+const DOMAIN = args.domain; const BRAND = args.brand;
+if (!DOMAIN || !BRAND) { console.error('need --domain and --brand'); process.exit(1); }
+const PAGE = 1000;
+const DIR = new URL('..', import.meta.url).pathname;
+const CKPT = `${DIR}pilot/${BRAND}_feed_harvest.jsonl`;
+const ENUM_CACHE = `${DIR}pilot/${BRAND}_enum.json`;
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+function get(url) {
+ return new Promise(resolve => {
+ const req = https.get(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } }, res => {
+ let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch { resolve(null); } });
+ });
+ req.on('error', () => resolve(null));
+ req.setTimeout(60000, () => { req.destroy(); resolve(null); });
+ });
+}
+const enc = o => encodeURIComponent(JSON.stringify(o));
+const money = n => Math.round(n * 100) / 100;
+
+// group label -> normalized product_type. Robust to object|string|json-string shapes.
+function typeOf(it) {
+ let g = it.sdb_product_group_code_data;
+ if (typeof g === 'string') { try { g = JSON.parse(g); } catch {} }
+ let label = Array.isArray(g) && g[0] ? g[0].label : (g && g.label);
+ if (!label) {
+ const cp = String(it.category_path || '').toLowerCase();
+ if (cp.includes('wallpaper')) label = 'Wallpaper';
+ else if (cp.includes('fabric')) label = 'Fabric';
+ }
+ const l = String(label || '').toLowerCase();
+ if (l.includes('wallpaper') || l.includes('wallcovering')) return 'wallcovering';
+ if (l.includes('fabric')) return 'fabric';
+ // fall back to SKU letter: xxW = wallcovering, xxF = fabric
+ const m = String(it.sku || '').match(/^[A-Z]{2}([WF])/);
+ if (m) return m[1] === 'W' ? 'wallcovering' : 'fabric';
+ return null;
+}
+
+async function enumerate() {
+ if (fs.existsSync(ENUM_CACHE)) {
+ const cached = JSON.parse(fs.readFileSync(ENUM_CACHE, 'utf8'));
+ console.log(`[${BRAND}] using cached enumeration: ${cached.length} base codes`);
+ return cached;
+ }
+ const base = new Set(); let skip = 0, rows = 0;
+ while (true) {
+ const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=1&limit=${PAGE}&skip=${skip}`);
+ const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
+ if (!cat.length) break;
+ rows += cat.length;
+ for (const c of cat) { const m = String(c.sku||'').match(/^([A-Z]{2}[WF][0-9]{4}-[0-9]{2})/); if (m) base.add(m[1]); }
+ skip += PAGE;
+ process.stdout.write(`\r[${BRAND}] enumerated ${rows} rows -> ${base.size} distinct base codes`);
+ if (cat.length < PAGE) break;
+ await sleep(150);
+ }
+ console.log('');
+ const arr = [...base].sort();
+ fs.writeFileSync(ENUM_CACHE, JSON.stringify(arr));
+ return arr;
+}
+
+(async () => {
+ const bases = await enumerate();
+ const limit = args.limit ? Math.min(Number(args.limit), bases.length) : bases.length;
+ const done = new Set();
+ if (fs.existsSync(CKPT)) for (const l of fs.readFileSync(CKPT,'utf8').split('\n').filter(Boolean)) { try { done.add(JSON.parse(l).base_code); } catch {} }
+ const todo = bases.slice(0, limit).filter(b => !done.has(b));
+ console.log(`[${BRAND}] ${bases.length} base codes, ${done.size} checkpointed, ${todo.length} to harvest`);
+
+ const out = fs.createWriteStream(CKPT, { flags: 'a' });
+ let hit=0, miss=0, wp=0, fab=0, i=0;
+ for (const base of todo) {
+ i++;
+ const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=3&filter=${enc({sku:base})}&limit=8`);
+ const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
+ // pick the priced colorway row matching this base; else first priced row
+ let it = cat.find(x => String(x.sku||'').startsWith(base) && typeof x.price==='number' && x.price>0)
+ || cat.find(x => typeof x.price==='number' && x.price>0)
+ || cat.find(x => String(x.sku||'').startsWith(base))
+ || cat[0] || null;
+ let ssp=null, trade=null, retail=null;
+ if (it && typeof it.price==='number' && it.price>0) { ssp=money(it.price); trade=money(ssp/2); retail=money(trade/0.65/0.85); hit++; }
+ else miss++;
+ const ptype = it ? typeOf(it) : null;
+ if (ptype==='wallcovering') wp++; else if (ptype==='fabric') fab++;
+ out.write(JSON.stringify({
+ base_code: base,
+ mfr_sku: it ? (it.sku || base) : base,
+ product_type: ptype,
+ pattern: it ? (it.sdb_design_name || it.name || null) : null,
+ color: it ? (it.sdb_desc_colour || it.sdb_design_colour_description || it.sdb_colour_variant || null) : null,
+ collection: it ? (it.sdb_collection_name || it.sdb_product_collection || null) : null,
+ width: it ? (it.sdb_useable_width || it.sdb_usable_width_inches || null) : null,
+ length: it ? (it.sdb_standard_length_inches || it.sdb_standard_length || null) : null,
+ ssp_usd: ssp, trade_usd: trade, retail_usd: retail,
+ image: it ? (it.image || null) : null
+ }) + '\n');
+ if (i%25===0 || i===todo.length) console.log(`[${BRAND}] ${i}/${todo.length} hit=${hit} miss=${miss} wp=${wp} fab=${fab}`);
+ await sleep(120);
+ }
+ out.end();
+ console.log(`[${BRAND}] DONE hit=${hit} miss=${miss} wp=${wp} fab=${fab}. Checkpoint: ${CKPT}`);
+})();
← 3039681 Harlequin pricing pilot: feed-first trade harvest (732/842)
·
back to Sanderson Onboard
·
TK-10877: feed-pricing staging tables (wp+fabric, trade+reta 5516c54 →