`recrawl-jffabrics.js`

139 lines
#!/usr/bin/env node
// ============================================================================
// JF Fabrics re-crawl driver  (mirror of recrawl-osborne.js)
// ----------------------------------------------------------------------------
// Pulls distinct jf_fabrics product_urls from the DB and re-runs the FIXED
// extractor (extract-jffabrics.js: all_images + full specs), then ADDITIVELY
// upserts via scraper-utils.upsertProduct (ON CONFLICT (mfr_sku) DO UPDATE — no
// new rows, no duplicates; only backfills the missing image/spec columns).
//
// DEPLOY TARGET (Kamatera): copy this file + extract-jffabrics.js into
//   /root/DW-Agents/vendor-scrapers/   then run there (writes prod dw_unified).
//
// Flags:
//   --net-new      only rows not yet on Shopify (on_shopify=false / no shopify id)
//   --all-images   only rows whose all_images is empty (the completeness gap)
//   --limit N      cap the worklist (smoke test)
//   --dry-run      crawl + extract + log, but DO NOT write to the DB
// ============================================================================
require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const https = require('https');
const path = require('path');

// scraper-utils + the fixed extractor resolve from the vendor-scrapers dir when
// deployed there; fall back to local sibling for off-box dry tests.
function req(localName, deployedPath) {
  try { return require(deployedPath); } catch (e) { return require(path.join(__dirname, localName)); }
}
const { upsertProduct, pool } = req('./_stub_scraper_utils.js', '/root/DW-Agents/vendor-scrapers/scraper-utils');
const { extractFromHtml } = req('./extract-jffabrics.js', '/root/DW-Agents/vendor-scrapers/extract-jffabrics.js');

const TABLE = 'jf_fabrics_catalog';
const SITE_BASE = 'https://www.jffabrics.com';
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const MIN_BYTES = 30000; // a real JF product page is ~60KB; shorter = throttled/challenge

const args = process.argv.slice(2);
const NET_NEW = args.includes('--net-new');
const ALL_IMAGES_ONLY = args.includes('--all-images');
const DRY = args.includes('--dry-run');
const limIdx = args.indexOf('--limit');
const LIMIT = limIdx >= 0 ? parseInt(args[limIdx + 1], 10) : 0;

function httpGet(url, maxRedirects = 4) {
  return new Promise((resolve, reject) => {
    const req2 = https.get(url, { timeout: 25000, headers: {
      'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9',
    }}, (res) => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
        res.resume();
        return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      if (res.statusCode !== 200) {
        res.resume();
        const err = new Error('HTTP ' + res.statusCode);
        err.statusCode = res.statusCode;
        const ra = res.headers['retry-after'];
        if (ra) { const s = parseInt(ra, 10); if (!isNaN(s)) err.retryAfterMs = Math.min(120000, s * 1000); }
        return reject(err);
      }
      let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
    });
    req2.on('error', reject);
    req2.on('timeout', () => { req2.destroy(); reject(new Error('timeout')); });
  });
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));

// Fetch with retry on transient short pages AND on HTTP 429 rate-limiting.
// JF Fabrics enforces an IP burst quota that depletes after ~25 quick hits, then
// returns 429 until it refills — so on 429 we back off (Retry-After if given,
// else escalating cooldown) and retry, letting the single worker self-pace to a
// sustainable rate rather than burning the whole worklist into errors.
async function fetchPage(url) {
  for (let attempt = 1; attempt <= 6; attempt++) {
    try {
      const html = await httpGet(url);
      if (html && html.length >= MIN_BYTES) { await sleep(0); return html; }
      await sleep(1500 * attempt); // short page (throttle/challenge) — back off + retry
    } catch (e) {
      if (e.statusCode === 429) {
        const wait = e.retryAfterMs || Math.min(90000, 4000 * attempt * attempt); // 4s,16s,36s,64s,90s,90s
        await sleep(wait);
        continue;
      }
      if (attempt >= 6) throw e;
      await sleep(1000 * attempt);
    }
  }
  throw new Error('exhausted retries (throttled)');
}

(async () => {
  let where = "product_url IS NOT NULL AND product_url <> ''";
  if (NET_NEW) where += " AND COALESCE(on_shopify,false)=false AND (shopify_product_id IS NULL OR shopify_product_id='')";
  if (ALL_IMAGES_ONLY) where += " AND (all_images IS NULL OR btrim(all_images)='' OR all_images IN ('[]','null','{}'))";
  let sql = `SELECT DISTINCT split_part(product_url,'?',1) AS url FROM ${TABLE} WHERE ${where} ORDER BY 1`;
  if (LIMIT > 0) sql += ` LIMIT ${LIMIT}`;

  const { rows } = await pool.query(sql);
  console.log(`[recrawl-jffabrics] ${rows.length} pages | net_new=${NET_NEW} all_images_only=${ALL_IMAGES_ONLY} dry=${DRY}`);

  let pages = 0, skus = 0, withImg = 0, withSpecs = 0, errs = 0;
  let idx = 0;
  // Single worker — JF rate-limits an IP burst; serial + backoff is the only
  // way the full worklist completes instead of mostly 429-erroring.
  const concurrency = 1;
  async function worker() {
    while (idx < rows.length) {
      const i = idx++;
      const url = rows[i].url;
      try {
        const html = await fetchPage(url);
        const p = extractFromHtml(html, url);
        if (p && p.mfr_sku) {
          const hasImgs = p.all_images && p.all_images !== '[]';
          const hasSpecs = p.fire_rating || p.material || p.width || p.match_type || p.features;
          if (DRY) {
            skus++; if (hasImgs) withImg++; if (hasSpecs) withSpecs++;
          } else {
            const id = await upsertProduct(TABLE, p);
            if (id) { skus++; if (hasImgs) withImg++; if (hasSpecs) withSpecs++; }
          }
        }
        pages++;
        if (pages % 25 === 0) console.log(`  ${pages}/${rows.length} | ${skus} skus | ${withImg} w/img | ${withSpecs} w/specs | ${errs} errs`);
        await sleep(1100); // polite spacing to stay under the burst quota
      } catch (e) {
        errs++;
        if (errs % 10 === 0) console.error(`  err ${url}: ${e.message}`);
        await sleep(900);
      }
    }
  }
  await Promise.all(Array.from({ length: concurrency }, worker));
  console.log(`\nDONE. pages=${pages} upserted=${skus} with_img=${withImg} with_specs=${withSpecs} errors=${errs}${DRY ? ' (DRY-RUN, no writes)' : ''}`);
  await pool.end();
})().catch(e => { console.error('FATAL', e); try { pool.end(); } catch (_) {} process.exit(1); });