← back to Wolfgordon Crawl
Add targeted room-image re-scrape ($0 HTTP, resumable): re-fetch live-cohort product pages, update all_images=[swatch,room shot] in catalog
55477d35b72d111bad7b3b65a250de12dce22902 · 2026-06-19 17:08:13 -0700 · Steve
Files touched
A rescrape-room-images.js
Diff
commit 55477d35b72d111bad7b3b65a250de12dce22902
Author: Steve <steve@designerwallcoverings.com>
Date: Fri Jun 19 17:08:13 2026 -0700
Add targeted room-image re-scrape ($0 HTTP, resumable): re-fetch live-cohort product pages, update all_images=[swatch,room shot] in catalog
---
rescrape-room-images.js | 138 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 138 insertions(+)
diff --git a/rescrape-room-images.js b/rescrape-room-images.js
new file mode 100644
index 0000000..c0412c7
--- /dev/null
+++ b/rescrape-room-images.js
@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+// ============================================================================
+// Wolf Gordon — TARGETED room-image re-scrape ($0, plain HTTP, resumable).
+//
+// The crawler fix (scrape-wolfgordon.js, commit 24bc927) now captures the
+// room/installation shot into all_images, but the 5,483 catalog rows still
+// hold swatch-only all_images. This re-fetches each product page over HTTP and
+// updates ONLY all_images (= [swatch, ...room/installation shots]) in
+// dw_unified.wolf_gordon_catalog. image_url (the primary swatch) is left
+// untouched. NEVER touches price, never creates rows, never deletes.
+//
+// Priority: the LIVE cohort (rows with a shopify_product_id) first — those are
+// the products customers see. --all widens to the whole catalog.
+//
+// Resumable: a row is SKIPPED if its all_images already contains an
+// installation shot (idempotent re-run safe). Polite: concurrency 3, jittered
+// pauses, honors the shared kill-switch ~/.dw-fixer-stop.
+//
+// Usage:
+// node rescrape-room-images.js # DRY-RUN, live cohort
+// node rescrape-room-images.js --apply # LIVE cohort, write
+// node rescrape-room-images.js --apply --all # whole catalog, write
+// node rescrape-room-images.js --apply --limit 50
+// ============================================================================
+const https = require('https');
+const os = require('os');
+const fs = require('fs');
+const { pool, wait } = require('./scraper-utils');
+const { parseProductPage } = require('./scrape-wolfgordon.js');
+
+const TABLE = 'wolf_gordon_catalog';
+const STOP = os.homedir() + '/.dw-fixer-stop';
+const argv = process.argv.slice(2);
+const APPLY = argv.includes('--apply');
+const ALL = argv.includes('--all');
+const LIMIT = (() => { const i = argv.indexOf('--limit'); return i !== -1 && argv[i + 1] ? parseInt(argv[i + 1], 10) : null; })();
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+function fetchPage(url, maxRedirects = 5) {
+ return new Promise((resolve, reject) => {
+ https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' } }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ let loc = res.headers.location;
+ if (loc.startsWith('/')) loc = `https://www.wolfgordon.com${loc}`;
+ return fetchPage(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', c => data += c);
+ res.on('end', () => resolve({ html: data, status: res.statusCode }));
+ res.on('error', reject);
+ }).on('error', reject).on('timeout', function () { this.destroy(); reject(new Error('timeout')); });
+ });
+}
+
+const nk = s => String(s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
+const hasRoom = (j) => { try { const a = typeof j === 'string' ? JSON.parse(j) : j; return Array.isArray(a) && a.some(u => /installation/.test(String(u))); } catch { return false; } };
+
+async function main() {
+ console.log('='.repeat(74));
+ console.log(` WG ROOM-IMAGE RE-SCRAPE mode=${APPLY ? 'APPLY (write)' : 'DRY-RUN'} scope=${ALL ? 'ALL' : 'LIVE-COHORT'}${LIMIT ? ` limit=${LIMIT}` : ''}`);
+ console.log('='.repeat(74));
+
+ // Source-of-truth rows. Live cohort = those with a shopify_product_id.
+ let where = `product_url IS NOT NULL`;
+ if (!ALL) where += ` AND shopify_product_id IS NOT NULL`;
+ const rows = (await pool.query(
+ `SELECT id, mfr_sku, product_url, all_images, shopify_product_id
+ FROM ${TABLE} WHERE ${where}
+ ORDER BY (shopify_product_id IS NOT NULL) DESC, updated_at DESC
+ ${LIMIT ? `LIMIT ${LIMIT}` : ''}`)).rows;
+ console.log(` candidate rows: ${rows.length}`);
+
+ // Group rows by product PAGE (one page yields many colorways). We fetch each
+ // distinct page once, then match parsed colorways back to rows by SKU.
+ const pages = new Map(); // url -> [row,...]
+ let alreadyHasRoom = 0;
+ for (const r of rows) {
+ if (hasRoom(r.all_images)) { alreadyHasRoom++; continue; } // resume skip
+ if (!pages.has(r.product_url)) pages.set(r.product_url, []);
+ pages.get(r.product_url).push(r);
+ }
+ // Also index every candidate row by SKU so a page that lists sibling colorways
+ // can update them too (the room shot is shared per pattern).
+ const bySku = new Map();
+ for (const r of rows) bySku.set(nk(r.mfr_sku), r);
+
+ const pageList = [...pages.keys()];
+ console.log(` rows already have a room shot (skipped): ${alreadyHasRoom}`);
+ console.log(` distinct product pages to fetch: ${pageList.length}`);
+
+ const summary = { pagesFetched: 0, rowsUpdated: 0, pagesNoRoom: 0, pages404: 0, errors: 0 };
+ const concurrency = 3;
+ let idx = 0;
+
+ async function worker() {
+ while (idx < pageList.length) {
+ if (fs.existsSync(STOP)) { console.log(' KILL-SWITCH present — stopping.'); return; }
+ const i = idx++;
+ const url = pageList[i];
+ try {
+ const { html, status } = await fetchPage(url);
+ if (status !== 200 || html.length < 2000) { summary.pages404++; await wait(400, 200); continue; }
+ summary.pagesFetched++;
+ const parsed = parseProductPage(html, url);
+ // build SKU -> all_images (with room shots) from this page
+ for (const p of parsed) {
+ let imgs; try { imgs = JSON.parse(p.all_images); } catch { continue; }
+ if (!Array.isArray(imgs) || !imgs.some(u => /installation/.test(String(u)))) continue;
+ const row = bySku.get(nk(p.mfr_sku));
+ if (!row) continue;
+ if (hasRoom(row.all_images)) continue; // already done in a prior page
+ row.all_images = JSON.stringify(imgs); // mark done in-memory (dedup siblings)
+ if (APPLY) {
+ await pool.query(`UPDATE ${TABLE} SET all_images=$1, updated_at=NOW(), last_scraped=NOW() WHERE id=$2`, [JSON.stringify(imgs), row.id]);
+ }
+ summary.rowsUpdated++;
+ }
+ // did THIS page yield any room shot at all?
+ if (!parsed.some(p => /installation/.test(String(p.all_images)))) summary.pagesNoRoom++;
+ if ((i + 1) % 25 === 0) console.log(` progress ${i + 1}/${pageList.length} pages | ${summary.rowsUpdated} rows updated`);
+ await wait(450, 250);
+ } catch (e) {
+ summary.errors++;
+ if (summary.errors <= 5) console.error(` err ${url}: ${e.message}`);
+ await wait(900, 400);
+ }
+ }
+ }
+
+ const ws = [];
+ for (let w = 0; w < concurrency; w++) ws.push(worker());
+ await Promise.all(ws);
+
+ console.log('-'.repeat(74));
+ console.log((APPLY ? ' APPLIED. ' : ' DRY-RUN. ') + JSON.stringify(summary));
+ await pool.end();
+}
+main().catch(e => { console.error('FATAL:', e); process.exit(1); });
← 24bc927 ROOT-CAUSE FIX: createProduct ships full desc+metafields fro
·
back to Wolfgordon Crawl
·
Wire net-new pusher to shared variant-budget ledger + kill-s 2fe399e →