`gapfill-images.js`

39 lines
#!/usr/bin/env node
// Per-row image gap-fill: for each row still lacking a signed image, fetch ITS
// OWN product page (where its own swatch is guaranteed) and grab the signed URL.
const https = require('https');
const { parseProductPage } = require('./scrape-wolfgordon.js');
const { pool, wait, closePool } = require('./scraper-utils');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
function fetchPage(url, n = 5) { return new Promise((res, rej) => {
  https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html' } }, r => {
    if ((r.statusCode === 301 || r.statusCode === 302) && r.headers.location && n > 0) {
      let l = r.headers.location; if (l.startsWith('/')) l = 'https://www.wolfgordon.com' + l;
      return fetchPage(l, n - 1).then(res).catch(rej); }
    let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
  }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); });
});}
const nk = s => (s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
(async () => {
  const { rows } = await pool.query(`SELECT mfr_sku, product_url FROM wolf_gordon_catalog
    WHERE (image_url IS NULL OR image_url NOT LIKE '%s=%') AND product_url IS NOT NULL`);
  console.log(`Unsigned rows to gap-fill: ${rows.length}`);
  let fixed = 0, miss = 0, err = 0, i = 0;
  async function worker() { while (i < rows.length) { const r = rows[i++];
    try {
      const prods = parseProductPage(await fetchPage(r.product_url), r.product_url);
      const want = nk(r.mfr_sku);
      const hit = prods.find(p => nk(p.mfr_sku) === want && p.image_url && p.image_url.includes('s='));
      if (hit) { await pool.query(`UPDATE wolf_gordon_catalog SET image_url=$1, all_images=$2, updated_at=NOW() WHERE mfr_sku=$3`,
        [hit.image_url, JSON.stringify([hit.image_url]), r.mfr_sku]); fixed++; }
      else miss++;
      if (i % 200 === 0) console.log(`  ${i}/${rows.length}  fixed ${fixed}  miss ${miss}`);
      await wait(200, 120);
    } catch (e) { err++; await wait(400, 200); }
  }}
  await Promise.all(Array.from({ length: 5 }, worker));
  const c = await pool.query(`SELECT count(*) FILTER(WHERE image_url LIKE '%s=%') signed, count(*) total FROM wolf_gordon_catalog`);
  console.log(`\nDONE: fixed ${fixed}, still-no-image ${miss}, errors ${err} | signed now ${c.rows[0].signed}/${c.rows[0].total}`);
  await closePool();
})().catch(e => { console.error('FATAL', e); closePool().then(() => process.exit(1)); });