← back to Dw Vendor Scrapers Local
Add recrawl-hollyhunt.js: full-page all_images completeness pass (HERO/Install/DET + colorway swatches, relevance-filtered)
50a4c55df72cd3f364e83667a5313e9cc318a741 · 2026-06-10 12:21:23 -0700 · Steve
Files touched
Diff
commit 50a4c55df72cd3f364e83667a5313e9cc318a741
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jun 10 12:21:23 2026 -0700
Add recrawl-hollyhunt.js: full-page all_images completeness pass (HERO/Install/DET + colorway swatches, relevance-filtered)
---
recrawl-hollyhunt.js | 205 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 205 insertions(+)
diff --git a/recrawl-hollyhunt.js b/recrawl-hollyhunt.js
new file mode 100644
index 0000000..821b415
--- /dev/null
+++ b/recrawl-hollyhunt.js
@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+// ============================================================================
+// Holly Hunt FULL-PAGE re-crawl — all_images completeness pass
+// ============================================================================
+// WHY: scrape-hollyhunt.js is Algolia-feed-based and captures only the single
+// grid thumbnail per SKU (image_url). Steve's standing rule "always pull all
+// data and images" requires every product carry ALL its images (all angles +
+// per-colorway swatches) in all_images before it is import-ready. Holly Hunt
+// detail pages (https://www.hollyhunt.com/sku/<pattern>/<color>) carry the
+// full set (HERO / Install / DET / colorway shots) as Drupal /dam/files assets,
+// served in many /styles/NNN_w/ sizes plus the un-styled full-res original.
+//
+// This driver crawls each net-new holly_hunt product_url, harvests the distinct
+// FULL-RES originals (strips the /styles/NNN_w/public/ resize segment + ?itok
+// query), dedupes, and ADDITIVELY writes all_images (JSON array) onto BOTH
+// holly_hunt_catalog AND the unified vendor_catalog (the import-readiness gate
+// table), keyed by mfr_sku. Nothing is wiped — image_url is only backfilled if
+// null. No Shopify publish. Idempotent: re-running only fills rows still empty.
+//
+// Deploy target: /root/DW-Agents/vendor-scrapers/recrawl-hollyhunt.js
+// Run: node recrawl-hollyhunt.js # net-new rows missing all_images
+// node recrawl-hollyhunt.js --all # every holly_hunt row missing all_images
+// ============================================================================
+
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const https = require('https');
+const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+const SITE_BASE = 'https://www.hollyhunt.com';
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const ALL = process.argv.includes('--all');
+const CONCURRENCY = 3;
+
+function httpGet(url, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, {
+ timeout: 25000,
+ headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' },
+ }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+// Canonicalize a Drupal image ref to the full-res original:
+// /sites/default/files/styles/1200_w/public/dam/files/2021-06/LX1218_..._HERO.jpg?itok=X
+// -> https://www.hollyhunt.com/sites/default/files/dam/files/2021-06/LX1218_..._HERO.jpg
+function canonicalImage(ref) {
+ if (!ref) return null;
+ let u = ref.split('?')[0].replace(/\\\//g, '/');
+ u = u.replace(/\/styles\/[^/]+\/public\//, '/'); // drop the resize derivative segment
+ if (!/\.(jpe?g|png|webp)$/i.test(u)) return null;
+ if (u.startsWith('http')) return u;
+ if (u.startsWith('/')) return SITE_BASE + u;
+ return null;
+}
+
+const norm = (s) => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
+
+// Extract the distinct full-res product images RELEVANT TO THIS product.
+// Holly Hunt PDPs also render "you may also like" / cross-pattern recommendation
+// thumbnails under the same /dam/files/ path, so a raw harvest pulls in foreign
+// products (e.g. a Pampa sectional, sibling patterns). We keep only images whose
+// filename references THIS product's pattern name or mfr_sku — which retains the
+// HERO / Install / DET shots AND every per-colorway swatch of the pattern (the
+// "all angles + per-colorway" the rule wants) while dropping recommendations.
+function extractAllImages(html, patternName, mfrSku) {
+ const all = new Set();
+ const re = /\/sites\/default\/files\/[^"'\s)]*\/dam\/files\/[^"'\s)]+\.(?:jpe?g|png|webp)/gi;
+ let m;
+ while ((m = re.exec(html)) !== null) {
+ const c = canonicalImage(m[0]);
+ if (c) all.add(c);
+ }
+ const pn = norm(patternName);
+ const sk = norm(mfrSku);
+ const fileOf = (u) => norm(u.split('/').pop());
+ let kept = [...all].filter((u) => {
+ const f = fileOf(u);
+ return (pn.length >= 3 && f.includes(pn)) || (sk.length >= 3 && f.includes(sk));
+ });
+ // Fallbacks: never strand a row imageless. Pattern-name miss → sku-only; still
+ // empty → accept the raw set (better a slightly noisy gallery than no images).
+ if (!kept.length && sk.length >= 3) kept = [...all].filter((u) => fileOf(u).includes(sk));
+ if (!kept.length) kept = [...all];
+ // Stable order: HERO → Install → colorway WM → DET; deterministic by sort.
+ return kept.sort((a, b) => {
+ const score = (s) => (/_HERO\./i.test(s) ? 0 : /_Install/i.test(s) ? 1 : /_DET\./i.test(s) ? 3 : 2);
+ const sa = score(a), sb = score(b);
+ return sa !== sb ? sa - sb : a.localeCompare(b);
+ });
+}
+
+// Light spec harvest (best-effort, additive). Holly Hunt PDPs expose a spec
+// table; we grab a few labelled rows if present. Never overwrites non-null.
+function extractSpecs(html) {
+ const specs = {};
+ const grab = (label) => {
+ const re = new RegExp(label + '\\s*<\\/[^>]+>\\s*<[^>]+>\\s*([^<]{1,80})', 'i');
+ const m = html.match(re);
+ return m ? m[1].trim() : null;
+ };
+ for (const [key, label] of [['width', 'Width'], ['repeat_v', 'Repeat'], ['material', 'Content'], ['composition', 'Composition']]) {
+ const v = grab(label);
+ if (v) specs[key] = v;
+ }
+ return specs;
+}
+
+async function loadWorklist() {
+ // Source from the gate table (vendor_catalog). product_url drives the crawl.
+ const where = ALL
+ ? `vendor_code='holly_hunt'`
+ : `vendor_code='holly_hunt' AND on_shopify IS NOT TRUE`;
+ const { rows } = await pool.query(
+ `SELECT mfr_sku, pattern_name, split_part(product_url,'?',1) AS url
+ FROM vendor_catalog
+ WHERE ${where}
+ AND (all_images IS NULL OR all_images IN ('','[]','null'))
+ AND product_url IS NOT NULL AND product_url <> ''
+ ORDER BY mfr_sku`
+ );
+ return rows;
+}
+
+async function writeImages(mfrSku, allImages, specs) {
+ const json = JSON.stringify(allImages);
+ const first = allImages[0] || null;
+ // Gate table — additive: set all_images, backfill image_url only if empty.
+ await pool.query(
+ `UPDATE vendor_catalog
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url,''), $2),
+ last_scraped_at = NOW()
+ WHERE vendor_code='holly_hunt' AND mfr_sku=$3`,
+ [json, first, mfrSku]
+ );
+ // Per-vendor catalog — keep consistent if the row exists.
+ await pool.query(
+ `UPDATE holly_hunt_catalog
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url,''), $2),
+ updated_at = NOW()
+ WHERE mfr_sku=$3`,
+ [json, first, mfrSku]
+ ).catch(() => {});
+ // Best-effort specs backfill (only columns that exist & are currently null).
+ if (specs && Object.keys(specs).length) {
+ const sets = [], vals = []; let i = 1;
+ for (const [k, v] of Object.entries(specs)) {
+ const col = k === 'repeat_v' ? 'pattern_repeat' : k;
+ sets.push(`${col} = COALESCE(NULLIF(${col},''), $${i++})`);
+ vals.push(v);
+ }
+ vals.push(mfrSku);
+ await pool.query(
+ `UPDATE vendor_catalog SET ${sets.join(', ')} WHERE vendor_code='holly_hunt' AND mfr_sku=$${i}`,
+ vals
+ ).catch(() => {});
+ }
+}
+
+(async () => {
+ const rows = await loadWorklist();
+ console.log(`Holly Hunt full-page re-crawl: ${rows.length} ${ALL ? '' : 'net-new '}rows missing all_images`);
+ let pages = 0, updated = 0, multiImg = 0, noImg = 0, errs = 0;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const { mfr_sku, pattern_name, url } = rows[i];
+ try {
+ const html = await httpGet(url);
+ const imgs = extractAllImages(html, pattern_name, mfr_sku);
+ if (imgs.length) {
+ const specs = extractSpecs(html);
+ await writeImages(mfr_sku, imgs, specs);
+ updated++;
+ if (imgs.length > 1) multiImg++;
+ } else {
+ noImg++;
+ }
+ pages++;
+ if (pages % 50 === 0) console.log(` ${pages}/${rows.length} pages | ${updated} updated | ${multiImg} multi-img | ${noImg} no-img | ${errs} errs`);
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 20 === 0) console.error(` err at ${url}: ${e.message}`);
+ await sleep(900);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+ console.log(`\nDONE. pages=${pages} updated=${updated} multi_img=${multiImg} no_img=${noImg} errors=${errs}`);
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
← aada824 Add recrawl-versa_ds.js: additive all_images re-crawl + hex→
·
back to Dw Vendor Scrapers Local
·
Add recrawl-schumacher.js: additive all_images completeness 5c43b67 →