`rescrape-room-images.js`

139 lines
#!/usr/bin/env node
// ============================================================================
// Wolf Gordon — TARGETED room-image re-scrape ($0, plain HTTP, resumable).
//
// The crawler fix (scrape-wolfgordon.js, commit 24bc927) now captures the
// room/installation shot into all_images, but the 5,483 catalog rows still
// hold swatch-only all_images. This re-fetches each product page over HTTP and
// updates ONLY all_images (= [swatch, ...room/installation shots]) in
// dw_unified.wolf_gordon_catalog. image_url (the primary swatch) is left
// untouched. NEVER touches price, never creates rows, never deletes.
//
// Priority: the LIVE cohort (rows with a shopify_product_id) first — those are
// the products customers see. --all widens to the whole catalog.
//
// Resumable: a row is SKIPPED if its all_images already contains an
// installation shot (idempotent re-run safe). Polite: concurrency 3, jittered
// pauses, honors the shared kill-switch ~/.dw-fixer-stop.
//
// Usage:
//   node rescrape-room-images.js                 # DRY-RUN, live cohort
//   node rescrape-room-images.js --apply         # LIVE cohort, write
//   node rescrape-room-images.js --apply --all   # whole catalog, write
//   node rescrape-room-images.js --apply --limit 50
// ============================================================================
const https = require('https');
const os = require('os');
const fs = require('fs');
const { pool, wait } = require('./scraper-utils');
const { parseProductPage } = require('./scrape-wolfgordon.js');

const TABLE = 'wolf_gordon_catalog';
const STOP = os.homedir() + '/.dw-fixer-stop';
const argv = process.argv.slice(2);
const APPLY = argv.includes('--apply');
const ALL = argv.includes('--all');
const LIMIT = (() => { const i = argv.indexOf('--limit'); return i !== -1 && argv[i + 1] ? parseInt(argv[i + 1], 10) : null; })();

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
function fetchPage(url, maxRedirects = 5) {
  return new Promise((resolve, reject) => {
    https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' } }, (res) => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        let loc = res.headers.location;
        if (loc.startsWith('/')) loc = `https://www.wolfgordon.com${loc}`;
        return fetchPage(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      let data = '';
      res.on('data', c => data += c);
      res.on('end', () => resolve({ html: data, status: res.statusCode }));
      res.on('error', reject);
    }).on('error', reject).on('timeout', function () { this.destroy(); reject(new Error('timeout')); });
  });
}

const nk = s => String(s || '').toUpperCase().replace(/[^A-Z0-9]/g, '');
const hasRoom = (j) => { try { const a = typeof j === 'string' ? JSON.parse(j) : j; return Array.isArray(a) && a.some(u => /installation/.test(String(u))); } catch { return false; } };

async function main() {
  console.log('='.repeat(74));
  console.log(`  WG ROOM-IMAGE RE-SCRAPE  mode=${APPLY ? 'APPLY (write)' : 'DRY-RUN'}  scope=${ALL ? 'ALL' : 'LIVE-COHORT'}${LIMIT ? `  limit=${LIMIT}` : ''}`);
  console.log('='.repeat(74));

  // Source-of-truth rows. Live cohort = those with a shopify_product_id.
  let where = `product_url IS NOT NULL`;
  if (!ALL) where += ` AND shopify_product_id IS NOT NULL`;
  const rows = (await pool.query(
    `SELECT id, mfr_sku, product_url, all_images, shopify_product_id
       FROM ${TABLE} WHERE ${where}
       ORDER BY (shopify_product_id IS NOT NULL) DESC, updated_at DESC
       ${LIMIT ? `LIMIT ${LIMIT}` : ''}`)).rows;
  console.log(`  candidate rows: ${rows.length}`);

  // Group rows by product PAGE (one page yields many colorways). We fetch each
  // distinct page once, then match parsed colorways back to rows by SKU.
  const pages = new Map(); // url -> [row,...]
  let alreadyHasRoom = 0;
  for (const r of rows) {
    if (hasRoom(r.all_images)) { alreadyHasRoom++; continue; } // resume skip
    if (!pages.has(r.product_url)) pages.set(r.product_url, []);
    pages.get(r.product_url).push(r);
  }
  // Also index every candidate row by SKU so a page that lists sibling colorways
  // can update them too (the room shot is shared per pattern).
  const bySku = new Map();
  for (const r of rows) bySku.set(nk(r.mfr_sku), r);

  const pageList = [...pages.keys()];
  console.log(`  rows already have a room shot (skipped): ${alreadyHasRoom}`);
  console.log(`  distinct product pages to fetch: ${pageList.length}`);

  const summary = { pagesFetched: 0, rowsUpdated: 0, pagesNoRoom: 0, pages404: 0, errors: 0 };
  const concurrency = 3;
  let idx = 0;

  async function worker() {
    while (idx < pageList.length) {
      if (fs.existsSync(STOP)) { console.log('  KILL-SWITCH present — stopping.'); return; }
      const i = idx++;
      const url = pageList[i];
      try {
        const { html, status } = await fetchPage(url);
        if (status !== 200 || html.length < 2000) { summary.pages404++; await wait(400, 200); continue; }
        summary.pagesFetched++;
        const parsed = parseProductPage(html, url);
        // build SKU -> all_images (with room shots) from this page
        for (const p of parsed) {
          let imgs; try { imgs = JSON.parse(p.all_images); } catch { continue; }
          if (!Array.isArray(imgs) || !imgs.some(u => /installation/.test(String(u)))) continue;
          const row = bySku.get(nk(p.mfr_sku));
          if (!row) continue;
          if (hasRoom(row.all_images)) continue; // already done in a prior page
          row.all_images = JSON.stringify(imgs); // mark done in-memory (dedup siblings)
          if (APPLY) {
            await pool.query(`UPDATE ${TABLE} SET all_images=$1, updated_at=NOW(), last_scraped=NOW() WHERE id=$2`, [JSON.stringify(imgs), row.id]);
          }
          summary.rowsUpdated++;
        }
        // did THIS page yield any room shot at all?
        if (!parsed.some(p => /installation/.test(String(p.all_images)))) summary.pagesNoRoom++;
        if ((i + 1) % 25 === 0) console.log(`  progress ${i + 1}/${pageList.length} pages | ${summary.rowsUpdated} rows updated`);
        await wait(450, 250);
      } catch (e) {
        summary.errors++;
        if (summary.errors <= 5) console.error(`  err ${url}: ${e.message}`);
        await wait(900, 400);
      }
    }
  }

  const ws = [];
  for (let w = 0; w < concurrency; w++) ws.push(worker());
  await Promise.all(ws);

  console.log('-'.repeat(74));
  console.log((APPLY ? '  APPLIED. ' : '  DRY-RUN. ') + JSON.stringify(summary));
  await pool.end();
}
main().catch(e => { console.error('FATAL:', e); process.exit(1); });