← back to Wolfgordon Crawl
Add image-backfill (re-pull signed images via two-path extractor; run after push)
f850a62b149225e3dd12f16be010e774604273bc · 2026-06-19 10:42:01 -0700 · Steve
Files touched
Diff
commit f850a62b149225e3dd12f16be010e774604273bc
Author: Steve <steve@designerwallcoverings.com>
Date: Fri Jun 19 10:42:01 2026 -0700
Add image-backfill (re-pull signed images via two-path extractor; run after push)
---
image-backfill.js | 43 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/image-backfill.js b/image-backfill.js
new file mode 100644
index 0000000..dc92311
--- /dev/null
+++ b/image-backfill.js
@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+// Re-populate image_url + all_images with SIGNED (200) URLs using the fixed
+// two-path extractor. One fetch per pattern (images are per-pattern); updates
+// every colorway of that pattern by mfr_sku. Local dw_unified only. $0.
+// RUN ONLY AFTER the Shopify push run finishes (avoids image_url read/write race).
+const https = require('https');
+const { parseProductPage } = require('./scrape-wolfgordon.js');
+const { pool, wait, closePool } = require('./scraper-utils');
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+function fetchPage(url, n = 5) { return new Promise((res, rej) => {
+ https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html' } }, r => {
+ if ((r.statusCode === 301 || r.statusCode === 302) && r.headers.location && n > 0) {
+ let l = r.headers.location; if (l.startsWith('/')) l = 'https://www.wolfgordon.com' + l;
+ return fetchPage(l, n - 1).then(res).catch(rej); }
+ let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
+ }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); });
+});}
+const nk = s => (s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
+(async () => {
+ const { rows } = await pool.query(`SELECT pattern_name, MIN(product_url) url FROM wolf_gordon_catalog
+ WHERE pattern_name IS NOT NULL AND product_url IS NOT NULL GROUP BY pattern_name ORDER BY pattern_name`);
+ console.log(`Patterns: ${rows.length}`);
+ let updated = 0, i = 0, errors = 0;
+ async function worker() { while (i < rows.length) { const r = rows[i++];
+ try {
+ const html = await fetchPage(r.url);
+ const prods = parseProductPage(html, r.url);
+ const map = {}; for (const p of prods) if (p.image_url && p.image_url.includes('s=')) map[nk(p.mfr_sku)] = p.image_url;
+ for (const k in map) {
+ const u = await pool.query(`UPDATE wolf_gordon_catalog SET image_url=$1, all_images=$2, updated_at=NOW()
+ WHERE upper(regexp_replace(mfr_sku,'[^A-Za-z0-9]','','g'))=$3 AND (image_url IS NULL OR image_url NOT LIKE '%s=%')`,
+ [map[k], JSON.stringify([map[k]]), k]);
+ updated += u.rowCount;
+ }
+ if (i % 50 === 0) console.log(` ${i}/${rows.length} patterns, ${updated} rows fixed`);
+ await wait(250, 150);
+ } catch (e) { errors++; if (errors <= 5) console.error(` err ${r.pattern_name}: ${e.message}`); await wait(500, 300); }
+ }}
+ await Promise.all(Array.from({ length: 4 }, worker));
+ const c = await pool.query(`SELECT count(*) FILTER(WHERE image_url LIKE '%s=%') signed, count(*) total FROM wolf_gordon_catalog`);
+ console.log(`\nDONE: signed images ${c.rows[0].signed}/${c.rows[0].total} (errors ${errors})`);
+ await closePool();
+})().catch(e => { console.error('FATAL', e); closePool().then(() => process.exit(1)); });
← 79044e4 Fix image extraction: capture both CDN paths (/production/im
·
back to Wolfgordon Crawl
·
Add per-row image gap-fill (fetch each unsigned row's own pa 71763fa →