`recrawl-hollyhunt.js`

206 lines
#!/usr/bin/env node
// ============================================================================
// Holly Hunt FULL-PAGE re-crawl — all_images completeness pass
// ============================================================================
// WHY: scrape-hollyhunt.js is Algolia-feed-based and captures only the single
// grid thumbnail per SKU (image_url). Steve's standing rule "always pull all
// data and images" requires every product carry ALL its images (all angles +
// per-colorway swatches) in all_images before it is import-ready. Holly Hunt
// detail pages (https://www.hollyhunt.com/sku/<pattern>/<color>) carry the
// full set (HERO / Install / DET / colorway shots) as Drupal /dam/files assets,
// served in many /styles/NNN_w/ sizes plus the un-styled full-res original.
//
// This driver crawls each net-new holly_hunt product_url, harvests the distinct
// FULL-RES originals (strips the /styles/NNN_w/public/ resize segment + ?itok
// query), dedupes, and ADDITIVELY writes all_images (JSON array) onto BOTH
// holly_hunt_catalog AND the unified vendor_catalog (the import-readiness gate
// table), keyed by mfr_sku. Nothing is wiped — image_url is only backfilled if
// null. No Shopify publish. Idempotent: re-running only fills rows still empty.
//
// Deploy target: /root/DW-Agents/vendor-scrapers/recrawl-hollyhunt.js
// Run:  node recrawl-hollyhunt.js            # net-new rows missing all_images
//       node recrawl-hollyhunt.js --all      # every holly_hunt row missing all_images
// ============================================================================

require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const https = require('https');
const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');

const SITE_BASE = 'https://www.hollyhunt.com';
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const ALL = process.argv.includes('--all');
const CONCURRENCY = 3;

function httpGet(url, maxRedirects = 4) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, {
      timeout: 25000,
      headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' },
    }, (res) => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
        res.resume();
        return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
  });
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));

// Canonicalize a Drupal image ref to the full-res original:
//   /sites/default/files/styles/1200_w/public/dam/files/2021-06/LX1218_..._HERO.jpg?itok=X
//   -> https://www.hollyhunt.com/sites/default/files/dam/files/2021-06/LX1218_..._HERO.jpg
function canonicalImage(ref) {
  if (!ref) return null;
  let u = ref.split('?')[0].replace(/\\\//g, '/');
  u = u.replace(/\/styles\/[^/]+\/public\//, '/');         // drop the resize derivative segment
  if (!/\.(jpe?g|png|webp)$/i.test(u)) return null;
  if (u.startsWith('http')) return u;
  if (u.startsWith('/')) return SITE_BASE + u;
  return null;
}

const norm = (s) => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '');

// Extract the distinct full-res product images RELEVANT TO THIS product.
// Holly Hunt PDPs also render "you may also like" / cross-pattern recommendation
// thumbnails under the same /dam/files/ path, so a raw harvest pulls in foreign
// products (e.g. a Pampa sectional, sibling patterns). We keep only images whose
// filename references THIS product's pattern name or mfr_sku — which retains the
// HERO / Install / DET shots AND every per-colorway swatch of the pattern (the
// "all angles + per-colorway" the rule wants) while dropping recommendations.
function extractAllImages(html, patternName, mfrSku) {
  const all = new Set();
  const re = /\/sites\/default\/files\/[^"'\s)]*\/dam\/files\/[^"'\s)]+\.(?:jpe?g|png|webp)/gi;
  let m;
  while ((m = re.exec(html)) !== null) {
    const c = canonicalImage(m[0]);
    if (c) all.add(c);
  }
  const pn = norm(patternName);
  const sk = norm(mfrSku);
  const fileOf = (u) => norm(u.split('/').pop());
  let kept = [...all].filter((u) => {
    const f = fileOf(u);
    return (pn.length >= 3 && f.includes(pn)) || (sk.length >= 3 && f.includes(sk));
  });
  // Fallbacks: never strand a row imageless. Pattern-name miss → sku-only; still
  // empty → accept the raw set (better a slightly noisy gallery than no images).
  if (!kept.length && sk.length >= 3) kept = [...all].filter((u) => fileOf(u).includes(sk));
  if (!kept.length) kept = [...all];
  // Stable order: HERO → Install → colorway WM → DET; deterministic by sort.
  return kept.sort((a, b) => {
    const score = (s) => (/_HERO\./i.test(s) ? 0 : /_Install/i.test(s) ? 1 : /_DET\./i.test(s) ? 3 : 2);
    const sa = score(a), sb = score(b);
    return sa !== sb ? sa - sb : a.localeCompare(b);
  });
}

// Light spec harvest (best-effort, additive). Holly Hunt PDPs expose a spec
// table; we grab a few labelled rows if present. Never overwrites non-null.
function extractSpecs(html) {
  const specs = {};
  const grab = (label) => {
    const re = new RegExp(label + '\\s*<\\/[^>]+>\\s*<[^>]+>\\s*([^<]{1,80})', 'i');
    const m = html.match(re);
    return m ? m[1].trim() : null;
  };
  for (const [key, label] of [['width', 'Width'], ['repeat_v', 'Repeat'], ['material', 'Content'], ['composition', 'Composition']]) {
    const v = grab(label);
    if (v) specs[key] = v;
  }
  return specs;
}

async function loadWorklist() {
  // Source from the gate table (vendor_catalog). product_url drives the crawl.
  const where = ALL
    ? `vendor_code='holly_hunt'`
    : `vendor_code='holly_hunt' AND on_shopify IS NOT TRUE`;
  const { rows } = await pool.query(
    `SELECT mfr_sku, pattern_name, split_part(product_url,'?',1) AS url
       FROM vendor_catalog
      WHERE ${where}
        AND (all_images IS NULL OR all_images IN ('','[]','null'))
        AND product_url IS NOT NULL AND product_url <> ''
      ORDER BY mfr_sku`
  );
  return rows;
}

async function writeImages(mfrSku, allImages, specs) {
  const json = JSON.stringify(allImages);
  const first = allImages[0] || null;
  // Gate table — additive: set all_images, backfill image_url only if empty.
  await pool.query(
    `UPDATE vendor_catalog
        SET all_images = $1,
            image_url  = COALESCE(NULLIF(image_url,''), $2),
            last_scraped_at = NOW()
      WHERE vendor_code='holly_hunt' AND mfr_sku=$3`,
    [json, first, mfrSku]
  );
  // Per-vendor catalog — keep consistent if the row exists.
  await pool.query(
    `UPDATE holly_hunt_catalog
        SET all_images = $1,
            image_url  = COALESCE(NULLIF(image_url,''), $2),
            updated_at = NOW()
      WHERE mfr_sku=$3`,
    [json, first, mfrSku]
  ).catch(() => {});
  // Best-effort specs backfill (only columns that exist & are currently null).
  if (specs && Object.keys(specs).length) {
    const sets = [], vals = []; let i = 1;
    for (const [k, v] of Object.entries(specs)) {
      const col = k === 'repeat_v' ? 'pattern_repeat' : k;
      sets.push(`${col} = COALESCE(NULLIF(${col},''), $${i++})`);
      vals.push(v);
    }
    vals.push(mfrSku);
    await pool.query(
      `UPDATE vendor_catalog SET ${sets.join(', ')} WHERE vendor_code='holly_hunt' AND mfr_sku=$${i}`,
      vals
    ).catch(() => {});
  }
}

(async () => {
  const rows = await loadWorklist();
  console.log(`Holly Hunt full-page re-crawl: ${rows.length} ${ALL ? '' : 'net-new '}rows missing all_images`);
  let pages = 0, updated = 0, multiImg = 0, noImg = 0, errs = 0;
  let idx = 0;
  async function worker() {
    while (idx < rows.length) {
      const i = idx++;
      const { mfr_sku, pattern_name, url } = rows[i];
      try {
        const html = await httpGet(url);
        const imgs = extractAllImages(html, pattern_name, mfr_sku);
        if (imgs.length) {
          const specs = extractSpecs(html);
          await writeImages(mfr_sku, imgs, specs);
          updated++;
          if (imgs.length > 1) multiImg++;
        } else {
          noImg++;
        }
        pages++;
        if (pages % 50 === 0) console.log(`  ${pages}/${rows.length} pages | ${updated} updated | ${multiImg} multi-img | ${noImg} no-img | ${errs} errs`);
        await sleep(350);
      } catch (e) {
        errs++;
        if (errs % 20 === 0) console.error(`  err at ${url}: ${e.message}`);
        await sleep(900);
      }
    }
  }
  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
  console.log(`\nDONE. pages=${pages} updated=${updated} multi_img=${multiImg} no_img=${noImg} errors=${errs}`);
  await pool.end();
})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });