← back to Sanderson Onboard
TK-10877: fix harvester — enumerate PATTERN codes, expand via CN_<pattern> to ALL priced colorways (recovers wallpaper universe the flat colorway-only enum missed)
6d6f3820ed9d07285c5036ba682da3308154d2cf · 2026-08-26 09:53:23 -0700 · Steve Abrams
Files touched
M scripts/harvest_sdg_feed.mjs
Diff
commit 6d6f3820ed9d07285c5036ba682da3308154d2cf
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 26 09:53:23 2026 -0700
TK-10877: fix harvester — enumerate PATTERN codes, expand via CN_<pattern> to ALL priced colorways (recovers wallpaper universe the flat colorway-only enum missed)
---
scripts/harvest_sdg_feed.mjs | 97 ++++++++++++++++++++++++++++----------------
1 file changed, 61 insertions(+), 36 deletions(-)
diff --git a/scripts/harvest_sdg_feed.mjs b/scripts/harvest_sdg_feed.mjs
index 02a410c..1524f2d 100644
--- a/scripts/harvest_sdg_feed.mjs
+++ b/scripts/harvest_sdg_feed.mjs
@@ -64,15 +64,22 @@ async function enumerate() {
console.log(`[${BRAND}] using cached enumeration: ${cached.length} base codes`);
return cached;
}
+ // Collect PATTERN-level base codes: xxW####/xxF#### (design patterns). The feed enumerates both
+ // CN_<PATTERN> pattern rows AND <PATTERN>-<NN>[UC] colorway rows; both collapse to the same pattern base.
+ // Each pattern base is then fetch-expanded (verbosity=3) into ALL its priced colorway SKUs downstream.
const base = new Set(); let skip = 0, rows = 0;
while (true) {
const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=1&limit=${PAGE}&skip=${skip}`);
const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
if (!cat.length) break;
rows += cat.length;
- for (const c of cat) { const m = String(c.sku||'').match(/^([A-Z]{2}[WF][0-9]{4}-[0-9]{2})/); if (m) base.add(m[1]); }
+ for (const c of cat) {
+ const s = String(c.sku||'').replace(/^CN_/, '');
+ const m = s.match(/^([A-Z]{2}[WF][0-9]{4})/); // pattern base, colorway/CN suffix stripped
+ if (m) base.add(m[1]);
+ }
skip += PAGE;
- process.stdout.write(`\r[${BRAND}] enumerated ${rows} rows -> ${base.size} distinct base codes`);
+ process.stdout.write(`\r[${BRAND}] enumerated ${rows} rows -> ${base.size} distinct pattern codes`);
if (cat.length < PAGE) break;
await sleep(150);
}
@@ -83,44 +90,62 @@ async function enumerate() {
}
(async () => {
- const bases = await enumerate();
- const limit = args.limit ? Math.min(Number(args.limit), bases.length) : bases.length;
- const done = new Set();
- if (fs.existsSync(CKPT)) for (const l of fs.readFileSync(CKPT,'utf8').split('\n').filter(Boolean)) { try { done.add(JSON.parse(l).base_code); } catch {} }
- const todo = bases.slice(0, limit).filter(b => !done.has(b));
- console.log(`[${BRAND}] ${bases.length} base codes, ${done.size} checkpointed, ${todo.length} to harvest`);
+ const patterns = await enumerate(); // pattern-level base codes (xxW####/xxF####)
+ const limit = args.limit ? Math.min(Number(args.limit), patterns.length) : patterns.length;
+ // resume by PATTERN: skip a pattern only if we already checkpointed at least one row for it
+ const donePat = new Set();
+ if (fs.existsSync(CKPT)) for (const l of fs.readFileSync(CKPT,'utf8').split('\n').filter(Boolean)) {
+ try { const r = JSON.parse(l); if (r.pattern_base) donePat.add(r.pattern_base); } catch {}
+ }
+ const todo = patterns.slice(0, limit).filter(p => !donePat.has(p));
+ console.log(`[${BRAND}] ${patterns.length} pattern codes, ${donePat.size} checkpointed, ${todo.length} to harvest`);
const out = fs.createWriteStream(CKPT, { flags: 'a' });
- let hit=0, miss=0, wp=0, fab=0, i=0;
- for (const base of todo) {
- i++;
- const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=3&filter=${enc({sku:base})}&limit=8`);
+ let patn=0, cwCount=0, hit=0, miss=0, wp=0, fab=0;
+ for (const pbase of todo) {
+ patn++;
+ // Expand the pattern into ALL its colorway rows. The feed's sku-filter matches the CN_<pattern>
+ // parent row, which returns every colorway child (verified: bare pattern base returns 0). limit=60
+ // covers the widest colorway sets.
+ const d = await get(`https://www.${DOMAIN}/us/api/n/find?type=product&verbosity=3&filter=${enc({sku:'CN_'+pbase})}&limit=60`);
const cat = d && Array.isArray(d.catalog) ? d.catalog : [];
- // pick the priced colorway row matching this base; else first priced row
- let it = cat.find(x => String(x.sku||'').startsWith(base) && typeof x.price==='number' && x.price>0)
- || cat.find(x => typeof x.price==='number' && x.price>0)
- || cat.find(x => String(x.sku||'').startsWith(base))
- || cat[0] || null;
- let ssp=null, trade=null, retail=null;
- if (it && typeof it.price==='number' && it.price>0) { ssp=money(it.price); trade=money(ssp/2); retail=money(trade/0.65/0.85); hit++; }
- else miss++;
- const ptype = it ? typeOf(it) : null;
- if (ptype==='wallcovering') wp++; else if (ptype==='fabric') fab++;
- out.write(JSON.stringify({
- base_code: base,
- mfr_sku: it ? (it.sku || base) : base,
- product_type: ptype,
- pattern: it ? (it.sdb_design_name || it.name || null) : null,
- color: it ? (it.sdb_desc_colour || it.sdb_design_colour_description || it.sdb_colour_variant || null) : null,
- collection: it ? (it.sdb_collection_name || it.sdb_product_collection || null) : null,
- width: it ? (it.sdb_useable_width || it.sdb_usable_width_inches || null) : null,
- length: it ? (it.sdb_standard_length_inches || it.sdb_standard_length || null) : null,
- ssp_usd: ssp, trade_usd: trade, retail_usd: retail,
- image: it ? (it.image || null) : null
- }) + '\n');
- if (i%25===0 || i===todo.length) console.log(`[${BRAND}] ${i}/${todo.length} hit=${hit} miss=${miss} wp=${wp} fab=${fab}`);
+ // real colorway rows = SKU form xxW####-## / xxF####-## (drop the CN_ pattern parent & any non-colorway).
+ // De-dup by colorway code, preferring the PRICED (non-UC) variant over a $0/discontinued duplicate.
+ const byCw = new Map();
+ for (const x of cat) {
+ const sku = String(x.sku||'');
+ const m = sku.match(/^([A-Z]{2}[WF][0-9]{4}-[0-9]{2})/);
+ if (!m) continue; // skip CN_ parent + malformed
+ const cw = m[1];
+ const priced = typeof x.price==='number' && x.price>0;
+ const prev = byCw.get(cw);
+ if (!prev || (priced && !(typeof prev.price==='number' && prev.price>0))) byCw.set(cw, x);
+ }
+ if (byCw.size === 0) miss++; // pattern yielded no colorway rows at all
+ for (const [cw, it] of byCw) {
+ cwCount++;
+ let ssp=null, trade=null, retail=null;
+ if (typeof it.price==='number' && it.price>0) { ssp=money(it.price); trade=money(ssp/2); retail=money(trade/0.65/0.85); hit++; }
+ else miss++;
+ const ptype = typeOf(it);
+ if (ptype==='wallcovering') wp++; else if (ptype==='fabric') fab++;
+ out.write(JSON.stringify({
+ base_code: cw, // colorway code = unique staging key
+ pattern_base: pbase,
+ mfr_sku: it.sku || cw,
+ product_type: ptype,
+ pattern: it.sdb_design_name || it.name || null,
+ color: it.sdb_desc_colour || it.sdb_design_colour_description || it.sdb_colour_variant || null,
+ collection: it.sdb_collection_name || it.sdb_product_collection || null,
+ width: it.sdb_useable_width || it.sdb_usable_width_inches || null,
+ length: it.sdb_standard_length_inches || it.sdb_standard_length || null,
+ ssp_usd: ssp, trade_usd: trade, retail_usd: retail,
+ image: it.image || null
+ }) + '\n');
+ }
+ if (patn%25===0 || patn===todo.length) console.log(`[${BRAND}] pat ${patn}/${todo.length} colorways=${cwCount} priced=${hit} miss=${miss} wp=${wp} fab=${fab}`);
await sleep(120);
}
out.end();
- console.log(`[${BRAND}] DONE hit=${hit} miss=${miss} wp=${wp} fab=${fab}. Checkpoint: ${CKPT}`);
+ console.log(`[${BRAND}] DONE patterns=${patn} colorways=${cwCount} priced=${hit} miss=${miss} wp=${wp} fab=${fab}. Checkpoint: ${CKPT}`);
})();
← e99ff7c auto-data-snapshot: 2026-08-26T09:38:32 (9 data files) — pil
·
back to Sanderson Onboard
·
TK-10877: full SDG feed harvest complete — Sanderson/Harlequ 3683ffb →