Add per-row image gap-fill (fetch each unsigned row's own page for its signed swatch)

71763faf3986d688da5ff2ceae7b3e8451ebed55 · 2026-06-19 11:19:07 -0700 · Steve

Files touched

A gapfill-images.js

Diff

commit 71763faf3986d688da5ff2ceae7b3e8451ebed55
Author: Steve <steve@designerwallcoverings.com>
Date:   Fri Jun 19 11:19:07 2026 -0700

    Add per-row image gap-fill (fetch each unsigned row's own page for its signed swatch)
---
 gapfill-images.js | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/gapfill-images.js b/gapfill-images.js
new file mode 100644
index 0000000..3cb1e4c
--- /dev/null
+++ b/gapfill-images.js
@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+// Per-row image gap-fill: for each row still lacking a signed image, fetch ITS
+// OWN product page (where its own swatch is guaranteed) and grab the signed URL.
+const https = require('https');
+const { parseProductPage } = require('./scrape-wolfgordon.js');
+const { pool, wait, closePool } = require('./scraper-utils');
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+function fetchPage(url, n = 5) { return new Promise((res, rej) => {
+  https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html' } }, r => {
+    if ((r.statusCode === 301 || r.statusCode === 302) && r.headers.location && n > 0) {
+      let l = r.headers.location; if (l.startsWith('/')) l = 'https://www.wolfgordon.com' + l;
+      return fetchPage(l, n - 1).then(res).catch(rej); }
+    let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
+  }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); });
+});}
+const nk = s => (s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
+(async () => {
+  const { rows } = await pool.query(`SELECT mfr_sku, product_url FROM wolf_gordon_catalog
+    WHERE (image_url IS NULL OR image_url NOT LIKE '%s=%') AND product_url IS NOT NULL`);
+  console.log(`Unsigned rows to gap-fill: ${rows.length}`);
+  let fixed = 0, miss = 0, err = 0, i = 0;
+  async function worker() { while (i < rows.length) { const r = rows[i++];
+    try {
+      const prods = parseProductPage(await fetchPage(r.product_url), r.product_url);
+      const want = nk(r.mfr_sku);
+      const hit = prods.find(p => nk(p.mfr_sku) === want && p.image_url && p.image_url.includes('s='));
+      if (hit) { await pool.query(`UPDATE wolf_gordon_catalog SET image_url=$1, all_images=$2, updated_at=NOW() WHERE mfr_sku=$3`,
+        [hit.image_url, JSON.stringify([hit.image_url]), r.mfr_sku]); fixed++; }
+      else miss++;
+      if (i % 200 === 0) console.log(`  ${i}/${rows.length}  fixed ${fixed}  miss ${miss}`);
+      await wait(200, 120);
+    } catch (e) { err++; await wait(400, 200); }
+  }}
+  await Promise.all(Array.from({ length: 5 }, worker));
+  const c = await pool.query(`SELECT count(*) FILTER(WHERE image_url LIKE '%s=%') signed, count(*) total FROM wolf_gordon_catalog`);
+  console.log(`\nDONE: fixed ${fixed}, still-no-image ${miss}, errors ${err} | signed now ${c.rows[0].signed}/${c.rows[0].total}`);
+  await closePool();
+})().catch(e => { console.error('FATAL', e); closePool().then(() => process.exit(1)); });

← f850a62 Add image-backfill (re-pull signed images via two-path extra · back to Wolfgordon Crawl · Add Wolf Gordon image-repair script (idempotent, image-add o a437445 →