`recrawl-osborne.js`

78 lines
#!/usr/bin/env node
// Re-crawl driver: pulls every distinct osborne product_url from the DB and
// re-runs the FIXED extraction (images + specs) via the deployed scraper's
// exported functions, then upserts into osborne_catalog. Avoids the flaky
// browser collection-listing crawl by reusing known product URLs.
require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const fs = require('fs');
const https = require('https');
const { upsertProduct, pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');

// Load extractFromHtml from the deployed scraper without running its main().
// Write the temp module INSIDE vendor-scrapers so node_modules (dotenv/pg) resolve.
const EXTRACT_TMP = '/root/DW-Agents/vendor-scrapers/_osb_extract.js';
let src = fs.readFileSync('/root/DW-Agents/vendor-scrapers/scrape-osborne.js', 'utf8')
  .replace(/main\(\)\.catch\([\s\S]*$/, 'module.exports={extractFromHtml};\n');
fs.writeFileSync(EXTRACT_TMP, src);
const { extractFromHtml } = require(EXTRACT_TMP);

const SITE_BASE = 'https://www.osborneandlittle.com';
const TABLE = 'osborne_catalog';
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

function httpGet(url, maxRedirects = 4) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
        res.resume();
        return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
  });
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));

(async () => {
  const { rows } = await pool.query(
    `SELECT DISTINCT split_part(product_url,'?',1) AS url FROM ${TABLE} WHERE product_url IS NOT NULL AND product_url <> '' ORDER BY 1`
  );
  console.log(`Re-crawling ${rows.length} distinct osborne product pages...`);
  let pages = 0, skus = 0, withImg = 0, withSpecs = 0, errs = 0;
  const concurrency = 3;
  let idx = 0;
  async function worker() {
    while (idx < rows.length) {
      const i = idx++;
      const url = rows[i].url;
      try {
        const html = await httpGet(url);
        const products = extractFromHtml(html, url);
        for (const p of products) {
          if (!p.mfr_sku) continue;
          const id = await upsertProduct(TABLE, p);
          if (id) {
            skus++;
            if (p.image_url) withImg++;
            if (p.specs) withSpecs++;
          }
        }
        pages++;
        if (pages % 25 === 0) console.log(`  ${pages}/${rows.length} pages | ${skus} skus | ${withImg} w/img | ${withSpecs} w/specs | ${errs} errs`);
        await sleep(350);
      } catch (e) {
        errs++;
        if (errs % 10 === 0) console.error(`  err at ${url}: ${e.message}`);
        await sleep(800);
      }
    }
  }
  await Promise.all(Array.from({ length: concurrency }, worker));
  console.log(`\nDONE. pages=${pages} skus_upserted=${skus} with_img=${withImg} with_specs=${withSpecs} errors=${errs}`);
  await pool.end();
})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });