`momentum-scrape/scrape.js`

102 lines
// Parallel Playwright scraper: visit each /products/<sku> on Momentum, extract L-code, save to PG.
// 8 contexts × 1 page each, queue-driven, resume-safe (skips rows where lcode IS NOT NULL).
const { chromium } = require('playwright');
const { Pool } = require('pg');

const CONCURRENCY = parseInt(process.env.CONC || '8', 10);
const DELAY_MS = parseInt(process.env.DELAY || '300', 10);
const pool = new Pool({ database: 'dw_unified', user: 'stevestudio2', host: '/tmp' });

let processed = 0, ok = 0, fail = 0, notFound = 0;
const start = Date.now();

async function fetchTodo() {
  const r = await pool.query(`
    SELECT momentum_sku FROM momentum_colorways
    WHERE momentum_sku IS NOT NULL AND momentum_sku <> '' AND lcode IS NULL
    ORDER BY momentum_sku
  `);
  return r.rows.map(r => r.momentum_sku);
}

async function worker(workerId, browser, queue) {
  const ctx = await browser.newContext({
    viewport: { width: 1280, height: 800 },
    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36',
  });
  const page = await ctx.newPage();
  // Block heavy assets — we only need HTML for the SKU
  await page.route('**/*', route => {
    const t = route.request().resourceType();
    if (['image','font','media','stylesheet'].includes(t)) return route.abort();
    return route.continue();
  });

  while (queue.length) {
    const sku = queue.shift();
    if (!sku) break;
    const url = `https://www.momentumtextilesandwalls.com/products/${sku}`;
    try {
      const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 25000 });
      const status = resp?.status() || 0;
      if (status === 404) {
        await pool.query(`UPDATE momentum_colorways SET lcode = '__NOT_FOUND__', lcode_scraped_at = now() WHERE momentum_sku = $1`, [sku]);
        notFound++;
      } else if (status >= 200 && status < 400) {
        // Wait for "SKU:" text + alphanumeric code to be hydrated
        let lcode = null;
        for (let attempt = 0; attempt < 2; attempt++) {
          try { await page.waitForFunction(() => /SKU:\s*[A-Z0-9][A-Z0-9-]+/i.test(document.body.innerText), { timeout: 12000 }); } catch {}
          lcode = await page.evaluate(() => {
            const matches = [...document.body.innerText.matchAll(/SKU:\s*([A-Z0-9][A-Z0-9-]+)/gi)].map(m => m[1].trim().toUpperCase());
            // Prefer hyphenated short code (the L-code) if present; otherwise return whatever printed SKU exists
            const shortCodes = matches.filter(c => c.includes('-') && !/^\d+$/.test(c));
            return shortCodes[0] || matches[0] || null;
          }).catch(() => null);
          if (lcode) break;
          if (attempt === 0) await page.waitForTimeout(2000);
        }
        if (lcode) {
          await pool.query(`UPDATE momentum_colorways SET lcode = $1, lcode_scraped_at = now() WHERE momentum_sku = $2`, [lcode, sku]);
          ok++;
        } else {
          await pool.query(`UPDATE momentum_colorways SET lcode = '__NO_SKU__', lcode_scraped_at = now() WHERE momentum_sku = $1`, [sku]);
          fail++;
        }
      } else {
        fail++;
      }
    } catch (e) {
      fail++;
    }
    processed++;
    if (processed % 25 === 0) {
      const elapsed = (Date.now() - start) / 1000;
      const rate = (processed / elapsed).toFixed(1);
      const remaining = queue.length;
      const eta = remaining > 0 ? Math.round(remaining / parseFloat(rate)) : 0;
      console.log(`  [w${workerId}] processed=${processed} ok=${ok} miss=${notFound} fail=${fail} | rate=${rate}/s | remaining=${remaining} | ETA=${Math.round(eta/60)}m`);
    }
    if (DELAY_MS) await page.waitForTimeout(DELAY_MS + Math.random() * 200);
  }

  await page.close();
  await ctx.close();
}

(async () => {
  console.log(`Loading work queue from PG...`);
  const queue = await fetchTodo();
  console.log(`Queue: ${queue.length} momentum SKUs to scrape`);
  console.log(`Concurrency: ${CONCURRENCY} workers, delay=${DELAY_MS}ms\n`);

  const browser = await chromium.launch({ headless: true });
  const workers = Array.from({ length: CONCURRENCY }, (_, i) => worker(i + 1, browser, queue));
  await Promise.all(workers);
  await browser.close();
  await pool.end();

  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
  console.log(`\nDONE. processed=${processed} ok=${ok} not_found=${notFound} fail=${fail} | ${elapsed}s`);
})();