`image-backfill.js`

44 lines
#!/usr/bin/env node
// Re-populate image_url + all_images with SIGNED (200) URLs using the fixed
// two-path extractor. One fetch per pattern (images are per-pattern); updates
// every colorway of that pattern by mfr_sku. Local dw_unified only. $0.
// RUN ONLY AFTER the Shopify push run finishes (avoids image_url read/write race).
const https = require('https');
const { parseProductPage } = require('./scrape-wolfgordon.js');
const { pool, wait, closePool } = require('./scraper-utils');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
function fetchPage(url, n = 5) { return new Promise((res, rej) => {
  https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html' } }, r => {
    if ((r.statusCode === 301 || r.statusCode === 302) && r.headers.location && n > 0) {
      let l = r.headers.location; if (l.startsWith('/')) l = 'https://www.wolfgordon.com' + l;
      return fetchPage(l, n - 1).then(res).catch(rej); }
    let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
  }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); });
});}
const nk = s => (s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
(async () => {
  const { rows } = await pool.query(`SELECT pattern_name, MIN(product_url) url FROM wolf_gordon_catalog
    WHERE pattern_name IS NOT NULL AND product_url IS NOT NULL GROUP BY pattern_name ORDER BY pattern_name`);
  console.log(`Patterns: ${rows.length}`);
  let updated = 0, i = 0, errors = 0;
  async function worker() { while (i < rows.length) { const r = rows[i++];
    try {
      const html = await fetchPage(r.url);
      const prods = parseProductPage(html, r.url);
      const map = {}; for (const p of prods) if (p.image_url && p.image_url.includes('s=')) map[nk(p.mfr_sku)] = p.image_url;
      for (const k in map) {
        const u = await pool.query(`UPDATE wolf_gordon_catalog SET image_url=$1, all_images=$2, updated_at=NOW()
          WHERE upper(regexp_replace(mfr_sku,'[^A-Za-z0-9]','','g'))=$3 AND (image_url IS NULL OR image_url NOT LIKE '%s=%')`,
          [map[k], JSON.stringify([map[k]]), k]);
        updated += u.rowCount;
      }
      if (i % 50 === 0) console.log(`  ${i}/${rows.length} patterns, ${updated} rows fixed`);
      await wait(250, 150);
    } catch (e) { errors++; if (errors <= 5) console.error(`  err ${r.pattern_name}: ${e.message}`); await wait(500, 300); }
  }}
  await Promise.all(Array.from({ length: 4 }, worker));
  const c = await pool.query(`SELECT count(*) FILTER(WHERE image_url LIKE '%s=%') signed, count(*) total FROM wolf_gordon_catalog`);
  console.log(`\nDONE: signed images ${c.rows[0].signed}/${c.rows[0].total} (errors ${errors})`);
  await closePool();
})().catch(e => { console.error('FATAL', e); closePool().then(() => process.exit(1)); });