← back to Reidwitlin Landing
reidwitlin: wire source-image backfill into canonical build (shared lib) so a full rebuild preserves the 190 recovered photos instead of regressing to 197 no-photo; refactor standalone patcher onto same lib
e798590bd93eb4f579e9520d5fd1e67d45391c4f · 2026-07-07 09:56:31 -0700 · Steve Abrams
Files touched
A lib/backfill-source-images.jsM scripts/backfill-source-images.jsM scripts/build-rwltd-data.js
Diff
commit e798590bd93eb4f579e9520d5fd1e67d45391c4f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 7 09:56:31 2026 -0700
reidwitlin: wire source-image backfill into canonical build (shared lib) so a full rebuild preserves the 190 recovered photos instead of regressing to 197 no-photo; refactor standalone patcher onto same lib
---
lib/backfill-source-images.js | 73 +++++++++++++++++++++++++
scripts/backfill-source-images.js | 112 +++++++++-----------------------------
scripts/build-rwltd-data.js | 16 ++++++
3 files changed, 116 insertions(+), 85 deletions(-)
diff --git a/lib/backfill-source-images.js b/lib/backfill-source-images.js
new file mode 100644
index 0000000..6259860
--- /dev/null
+++ b/lib/backfill-source-images.js
@@ -0,0 +1,73 @@
+/**
+ * Authoritative per-colorway image backfill from the FREE rwltd.com Shopify feed.
+ *
+ * The mis-scraped rwltd_catalog flattened every pattern's images into every
+ * colorway, leaving ~197 colorways with no colorway-specific photo. The SOURCE
+ * store (rwltd.com/products.json, public, $0) carries a per-VARIANT
+ * featured_image — ground-truth colorway->photo. Key the feed by
+ * slug("<pattern title>-<variant option>") (handles numeric counters AND named
+ * colorways) and backfill each no-photo product's real image, clearing its
+ * image_note so the own-photo-or-draft partition auto-publishes it.
+ *
+ * Shared by scripts/build-rwltd-data.js (so a full rebuild PRESERVES the recovered
+ * photos instead of regressing to 197 no-photo) and scripts/backfill-source-images.js
+ * (the standalone DB-free patcher). $0 — public feed, no auth/Browserbase.
+ */
+const https = require('https');
+
+const slug = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
+
+function get(url) {
+ // Shopify serves an empty body to the default node UA — send a browser UA.
+ const opts = { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36' } };
+ return new Promise((res, rej) => {
+ https.get(url, opts, r => {
+ if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location) return res(get(r.headers.location));
+ let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
+ }).on('error', rej);
+ });
+}
+
+// Pull the full source feed and build slug("pattern-option") -> featured_image src.
+async function fetchSourceImageIndex(base = 'https://rwltd.com/products.json?limit=250') {
+ const feed = [];
+ for (let page = 1; page <= 6; page++) {
+ let ps = [];
+ try { ps = JSON.parse(await get(`${base}&page=${page}`)).products || []; } catch { break; }
+ if (!ps.length) break;
+ feed.push(...ps);
+ }
+ const idx = {};
+ for (const p of feed) for (const v of (p.variants || [])) {
+ const fi = v.featured_image && v.featured_image.src;
+ if (fi) idx[slug(`${p.title}-${v.option1}`)] = fi;
+ }
+ return { idx, patterns: feed.length };
+}
+
+/**
+ * Mutates `products` in place: for any colorway that lacks its own photo
+ * (image_note set or no images) and matches a source variant, set the real
+ * image and clear image_note. Returns { backfilled, conflicts } — conflicts are
+ * already-imaged products whose current swatch DIFFERS from source (reported,
+ * NOT auto-changed; e.g. Front Royal Fig/Java where source has a variant swap).
+ */
+async function backfillFromSourceFeed(products, opts = {}) {
+ const { idx, patterns } = await fetchSourceImageIndex(opts.base);
+ let backfilled = 0; const conflicts = [];
+ for (const p of products) {
+ const src = idx[slug(p.sku)];
+ if (!src) continue;
+ const noPhoto = p.image_note || !(p.images && p.images.length);
+ if (noPhoto) {
+ p.images = [src]; p.swatch = src; p.room = src; delete p.image_note;
+ p.image_source = 'rwltd-variant'; backfilled++;
+ } else {
+ const cur = (p.swatch || '').split('?')[0], s0 = src.split('?')[0];
+ if (cur !== s0) conflicts.push({ handle: p.handle, color: p.color, ours: cur.split('/').pop(), source: s0.split('/').pop() });
+ }
+ }
+ return { backfilled, conflicts, sourcePatterns: patterns, indexSize: Object.keys(idx).length };
+}
+
+module.exports = { backfillFromSourceFeed, fetchSourceImageIndex, slug };
diff --git a/scripts/backfill-source-images.js b/scripts/backfill-source-images.js
index abbe7ed..0a91f24 100644
--- a/scripts/backfill-source-images.js
+++ b/scripts/backfill-source-images.js
@@ -1,118 +1,60 @@
#!/usr/bin/env node
/**
- * Authoritative per-colorway image backfill from the FREE rwltd.com Shopify feed.
- *
- * The mis-scraped rwltd_catalog flattened every pattern's images into every
- * colorway, which forced the fragile filename-matching gallery pass and left
- * ~197 colorways with no colorway-specific photo (shown as a flagged
- * "representative" hero). But the SOURCE store (rwltd.com/products.json, public,
- * $0) carries a per-VARIANT featured_image — ground-truth colorway->photo. This
- * keys the feed by slug("<pattern title>-<variant option>") (handles numeric
- * counters AND named colorways) and backfills each no-photo product's real image.
- *
- * Also reports (does NOT auto-apply) any already-imaged product whose current
- * swatch DIFFERS from the source ground truth, for human review.
- *
- * $0 — public Shopify feed, no auth/Browserbase. Writes a .bak first.
+ * DB-free standalone patcher: apply the source-image backfill to the CURRENT
+ * data/products.json (the full rebuild reads the gated Kamatera DB; this runs the
+ * SAME lib against the on-disk snapshot). Handles the own-photo-or-draft model:
+ * applies to products[] AND drafts[], clears image_note on any that get a real
+ * photo, then re-partitions so newly-photo'd drafts auto-restore to published, and
+ * recomputes facets. build-rwltd-data.js runs the same lib, so a rebuild preserves
+ * this. Writes a .bak first. $0 — public feed. Idempotent (safe to re-run).
*/
const fs = require('fs');
const path = require('path');
-const https = require('https');
+const { backfillFromSourceFeed } = require('../lib/backfill-source-images');
const OUT = path.join(__dirname, '..', 'data', 'products.json');
-const FEED = 'https://rwltd.com/products.json?limit=250';
-
-const slug = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
-
-function get(url) {
- // Shopify serves an empty body to the default node UA — send a browser UA.
- const opts = { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36' } };
- return new Promise((res, rej) => {
- https.get(url, opts, r => {
- if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location) return res(get(r.headers.location));
- let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
- }).on('error', rej);
- });
-}
+const BUCKET_ORDER = ['white','grey','black','pink','red','orange','brown','gold','green','teal','blue','purple'];
(async () => {
- // pull all feed pages (source is ~150 patterns; page until empty)
- const feed = [];
- for (let page = 1; page <= 6; page++) {
- const body = await get(`${FEED}&page=${page}`);
- let ps = [];
- try { ps = JSON.parse(body).products || []; } catch { break; }
- if (!ps.length) break;
- feed.push(...ps);
- }
- // universal key: slug("<pattern title>-<variant option1>") -> featured_image src
- const idx = {};
- for (const p of feed) for (const v of (p.variants || [])) {
- const fi = v.featured_image && v.featured_image.src;
- if (fi) idx[slug(`${p.title}-${v.option1}`)] = fi;
- }
- console.log(`source patterns ${feed.length} | variant-image keys ${Object.keys(idx).length}`);
-
const snap = JSON.parse(fs.readFileSync(OUT, 'utf8'));
- // Steve's own-photo-or-draft model (commit 4535dde): snapshot.products[] = published
- // (has own photo), snapshot.drafts[] = no own photo, auto-restore "once a real
- // colorway photo lands". This backfill IS that photo landing — apply to BOTH arrays,
- // clear image_note on any that now have a real source photo, then re-partition so the
- // newly-photo'd drafts auto-restore to published. Fully honors the committed rule.
snap.drafts = snap.drafts || [];
- let backfilled = 0; const conflicts = [];
- const applyTo = arr => {
- for (const p of arr) {
- const src = idx[slug(p.sku)];
- if (!src) continue;
- const noPhoto = p.image_note || !(p.images && p.images.length);
- if (noPhoto) {
- p.images = [src]; p.swatch = src; p.room = src; delete p.image_note;
- p.image_source = 'rwltd-variant'; backfilled++;
- } else {
- const cur = (p.swatch || '').split('?')[0], s0 = src.split('?')[0];
- if (cur !== s0) conflicts.push({ handle: p.handle, color: p.color, ours: cur.split('/').pop(), source: s0.split('/').pop() });
- }
- }
- };
- applyTo(snap.products); applyTo(snap.drafts);
- // Re-partition: anything with a real photo (no image_note) is published; rest stays drafted.
+ // Apply to both arrays (a drafted colorway getting a photo must be able to restore).
+ const r1 = await backfillFromSourceFeed(snap.products);
+ const r2 = await backfillFromSourceFeed(snap.drafts);
+ const backfilled = r1.backfilled + r2.backfilled;
+ const conflicts = [...r1.conflicts, ...r2.conflicts];
+
+ // Re-partition: anything with a real photo (no image_note) is published; rest drafted.
const all = [...snap.products, ...snap.drafts];
snap.products = all.filter(p => !p.image_note);
snap.drafts = all.filter(p => p.image_note);
snap.count = snap.products.length;
snap.drafted_no_own_photo = snap.drafts.length;
- console.log(`backfilled ${backfilled} colorways with authoritative source images`);
- console.log(`re-partitioned -> published ${snap.products.length} | drafted (still no own photo) ${snap.drafts.length}`);
- if (conflicts.length) {
- console.log(`\n${conflicts.length} already-imaged products DIFFER from source (NOT auto-changed — review):`);
- for (const c of conflicts) console.log(` ${c.handle} (${c.color}): ours=${c.ours} source=${c.source}`);
- }
-
- // Recompute facets from the NEW published set — the re-partition changed which
- // products are live, so the filter chips/counts must reflect the 989, not the old 799.
- const BUCKET_ORDER = ['white','grey','black','pink','red','orange','brown','gold','green','teal','blue','purple'];
- const fb = {}, fs_ = {}, fc = {};
+ // Recompute facets from the NEW published set (the partition changed which are live).
+ const fb = {}, fsr = {}, fc = {};
for (const p of snap.products) {
if (p.book) fb[p.book] = (fb[p.book] || 0) + 1;
- if (p.series) fs_[p.series] = (fs_[p.series] || 0) + 1;
+ if (p.series) fsr[p.series] = (fsr[p.series] || 0) + 1;
if (p.color_bucket) fc[p.color_bucket] = (fc[p.color_bucket] || 0) + 1;
}
snap.facets = {
total: snap.products.length,
books: Object.entries(fb).sort((a, b) => b[1] - a[1]),
- series: Object.entries(fs_).sort((a, b) => b[1] - a[1]).slice(0, 300),
+ series: Object.entries(fsr).sort((a, b) => b[1] - a[1]).slice(0, 300),
colors: BUCKET_ORDER.filter(b => fc[b]).map(b => [b, fc[b]]),
};
- console.log(`facets recomputed: total ${snap.facets.total} | ${snap.facets.books.length} books | ${snap.facets.series.length} series`);
- // Always persist: even with 0 new backfills, the re-partition + facet recompute
- // may have corrected a stale snapshot (idempotent — safe to re-run).
+ console.log(`backfilled ${backfilled} colorways | published ${snap.products.length} | drafted ${snap.drafts.length} | facets.total ${snap.facets.total}`);
+ if (conflicts.length) {
+ console.log(`${conflicts.length} already-imaged differ from source (NOT auto-changed — review):`);
+ for (const c of conflicts) console.log(` ${c.handle} (${c.color}): ours=${c.ours} source=${c.source}`);
+ }
+
const bak = `${OUT}.${new Date().toISOString().replace(/[:.]/g, '-')}.bak`;
fs.copyFileSync(OUT, bak);
snap.image_backfilled_at = new Date().toISOString();
fs.writeFileSync(OUT, JSON.stringify(snap));
- console.log(`\nbackup -> ${path.basename(bak)} | products.json rewritten`);
+ console.log(`backup -> ${path.basename(bak)} | products.json rewritten`);
})().catch(e => { console.error(e); process.exit(1); });
diff --git a/scripts/build-rwltd-data.js b/scripts/build-rwltd-data.js
index e11b46f..6d9dd2f 100644
--- a/scripts/build-rwltd-data.js
+++ b/scripts/build-rwltd-data.js
@@ -330,6 +330,22 @@ async function main() {
const rl = relabelUnreliableSeries(products);
if (rl.productsFixed) console.log(` colorway labels: relabelled ${rl.productsFixed} in ${rl.seriesFixed} mis-stamped series (${rl.series.join(', ')})`);
+ // Source-image backfill — a FOURTH pass, BEFORE the own-photo partition. The
+ // scrape lost the per-colorway image mapping; the free rwltd.com Shopify feed
+ // still carries an authoritative per-variant featured_image. Recover it so a
+ // full rebuild PRESERVES the ~190 recovered photos (auto-restore) instead of
+ // regressing every no-own-photo colorway to DRAFT. $0, public feed. If the feed
+ // is unreachable, this no-ops (products keep their image_note → drafted) so a
+ // rebuild degrades safely rather than failing.
+ try {
+ const { backfillFromSourceFeed } = require('../lib/backfill-source-images');
+ const bf = await backfillFromSourceFeed(products);
+ console.log(` source-image backfill: ${bf.backfilled} colorways got an authoritative photo (feed ${bf.sourcePatterns} patterns / ${bf.indexSize} keys)`);
+ if (bf.conflicts.length) console.log(` ${bf.conflicts.length} already-imaged differ from source (kept ours): ${bf.conflicts.map(c => c.handle.split('--')[0]).join(', ')}`);
+ } catch (e) {
+ console.log(` source-image backfill SKIPPED (feed unreachable: ${e.message}) — no-photo colorways stay drafted`);
+ }
+
// HARD RULE (Steve, 2026-07-07): all SKUs must have their OWN photo or go to DRAFT.
// A shared/representative (pattern-level) stand-in does NOT count. `image_note` is
// set in PASS B exactly when a colorway lacks its own confirmed/unique swatch, so it
← 593987a reidwitlin: recompute facets after backfill re-partition (wa
·
back to Reidwitlin Landing
·
reidwitlin: contrarian FIX-THEN-SHIP — guard silent-zero-res d67eec9 →