← back to Dw Vendor Scrapers Local
Add recrawl-schumacher.js: additive all_images completeness pass (CDN gallery + full product-page scrape, reuses deployed enrich extractors, all_images-empty row-selection)
5c43b67c65a5accb4f2f7e9412adfd54b1ed461d · 2026-06-10 12:41:35 -0700 · SteveStudio2
Files touched
Diff
commit 5c43b67c65a5accb4f2f7e9412adfd54b1ed461d
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed Jun 10 12:41:35 2026 -0700
Add recrawl-schumacher.js: additive all_images completeness pass (CDN gallery + full product-page scrape, reuses deployed enrich extractors, all_images-empty row-selection)
---
recrawl-schumacher.js | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 187 insertions(+)
diff --git a/recrawl-schumacher.js b/recrawl-schumacher.js
new file mode 100644
index 0000000..ea28a17
--- /dev/null
+++ b/recrawl-schumacher.js
@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+// ============================================================================
+// Schumacher ADDITIVE all_images re-crawl — full-page completeness pass
+// ============================================================================
+// WHY: schumacher_catalog (→ vendor_catalog vendor_code='schumacher', 6,474
+// rows) was image-enriched ONLY for rows missing the PRIMARY image. The
+// deployed enrich-schumacher-images.js selects `WHERE image_url IS NULL OR
+// image_url = ''`, so the 5,766 rows that already have a primary hero but an
+// EMPTY all_images array are never re-touched. Steve's standing rule "always
+// pull all data and images" requires every product to carry its FULL image set
+// (all angles + per-colorway swatches) + specs before it is import-ready, so a
+// populated image_url is NOT sufficient — all_images must be non-empty.
+//
+// 2026-06-10 gap (local dw_unified):
+// * vendor_catalog vendor_code='schumacher' missing all_images: 5,766
+// * net-new (shopify_product_id IS NULL) missing all_images: 3,438
+// — and 100% of those already have a primary image_url, i.e. the existing
+// enrich job's row-selection skips every single one of them.
+// * schumacher_catalog (source table this script writes) missing
+// all_images: 4,474 (all have product_url + mfr_sku).
+//
+// FIX vs the existing enrich script: same proven extractors, different
+// row-selection. We re-use the deployed enrich-schumacher-images.js functions
+// (discoverCdnImages = CDN pattern + {SKU}-1..8 alt-image probe,
+// extractImagesFromHtml = full product-page gallery scrape,
+// fetchProductSpecs = page specs) so the all_images CAPTURE logic is
+// identical to the reviewed prod path — we only widen WHICH rows get processed
+// (all_images-empty, regardless of whether image_url is already set). Mirrors
+// the load-deployed-scraper harness of recrawl-osborne.js and the additive
+// safety contract of recrawl-versa_ds.js.
+//
+// SAFETY / ADDITIVE-ONLY:
+// * NEVER inserts new rows. Only UPDATEs all_images / specs / image_url on
+// rows that already exist (matched by mfr_sku).
+// * Only touches rows whose all_images is empty.
+// * Does NOT push to Shopify, does NOT publish, does NOT change SKUs/prices.
+// * After this populates schumacher_catalog.all_images, the existing
+// consolidate-vendor-catalogs.py sync propagates it to vendor_catalog
+// (both tables track all_images; run the consolidate step to surface it).
+//
+// MODES:
+// --canary N process only the first N rows (default 5) — run FIRST on
+// Kamatera to confirm the deployed extractors still match live
+// HTML / CDN before the full pass.
+// --all process every eligible row (full ~4,474-row pass).
+// --net-new restrict to rows whose mfr_sku is NOT already in
+// shopify_products (skip the already-live cohort).
+//
+// Deploy target: Kamatera /root/DW-Agents/vendor-scrapers/. Authored from the
+// local mirror; FIRST run on Kamatera MUST be --canary to validate extraction.
+// This file is a PARKED artifact — see
+// ~/.claude/yolo-queue/pending-approval/schumacher-recrawl.md for the exact
+// deploy + run + consolidate commands. Do NOT run against prod without Steve.
+// ============================================================================
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const fs = require('fs');
+const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+const TABLE = 'schumacher_catalog';
+const ENRICH_SRC = '/root/DW-Agents/vendor-scrapers/enrich-schumacher-images.js';
+const EXTRACT_TMP = '/root/DW-Agents/vendor-scrapers/_sch_extract.js';
+
+// Load the deployed enrich script's extractors WITHOUT running its main().
+// Strip the trailing `main().catch(...)` invocation, append a module.exports of
+// the functions we need (they are top-level declarations, so in scope here).
+let src = fs.readFileSync(ENRICH_SRC, 'utf8')
+ .replace(/main\(\)\.catch\([\s\S]*$/,
+ 'module.exports = { discoverCdnImages, fetchProductSpecs, extractImagesFromHtml, httpGet, checkUrlExists };\n');
+fs.writeFileSync(EXTRACT_TMP, src);
+const { discoverCdnImages, fetchProductSpecs, extractImagesFromHtml, httpGet } = require(EXTRACT_TMP);
+
+const argv = process.argv.slice(2);
+const FULL = argv.includes('--all');
+const NET_NEW_ONLY = argv.includes('--net-new');
+const canaryIdx = argv.indexOf('--canary');
+const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+// Junk filter — never store logos, icons, sprites, payment badges as images.
+const JUNK = /(logo|icon|favicon|placeholder|sprite|payment|visa|mastercard|paypal|flag|avatar|loader|spinner)/i;
+function isImg(u) {
+ if (!u) return false;
+ if (!/\.(jpe?g|png|webp)(\?.*)?$/i.test(u)) return false;
+ if (JUNK.test(u)) return false;
+ return true;
+}
+
+(async () => {
+ let sql = `
+ SELECT mfr_sku,
+ split_part(product_url,'?',1) AS url,
+ image_url
+ FROM ${TABLE}
+ WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = '""')
+ AND mfr_sku IS NOT NULL AND mfr_sku <> ''
+ `;
+ if (NET_NEW_ONLY) {
+ sql += ` AND NOT EXISTS (SELECT 1 FROM shopify_products s
+ WHERE upper(trim(s.mfr_sku)) = upper(trim(${TABLE}.mfr_sku)))`;
+ }
+ sql += ` ORDER BY mfr_sku`;
+ if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;
+
+ const { rows } = await pool.query(sql);
+ console.log(`Schumacher additive all_images re-crawl — ${rows.length} rows`
+ + `${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}`
+ + `${NET_NEW_ONLY ? ' [net-new only]' : ''}`);
+
+ let n = 0, updated = 0, withImgs = 0, withSpecs = 0, noImg = 0, errs = 0;
+ for (const r of rows) {
+ n++;
+ const sku = String(r.mfr_sku).trim();
+ try {
+ let images = [];
+
+ // (1) CDN pattern + {SKU}-1..8 alt-image probe (deployed extractor).
+ try {
+ const cdn = await discoverCdnImages(sku);
+ if (cdn && Array.isArray(cdn.allImages)) images.push(...cdn.allImages);
+ } catch (e) {
+ if (CANARY_N > 0) console.error(` cdn-miss ${sku}: ${e.message}`);
+ }
+
+ // (2) Full product-page gallery scrape (deployed extractor) — the real
+ // "all data and images" source: every gallery angle + colorway swatch.
+ const pageUrl = (r.url && /^https?:\/\//i.test(r.url)) ? r.url : null;
+ let specs = {};
+ if (pageUrl) {
+ try {
+ const res = await httpGet(pageUrl, 4);
+ const body = (res && (res.body || res)) || '';
+ if (typeof body === 'string' && body.length) {
+ for (const u of (extractImagesFromHtml(body, sku) || [])) {
+ if (isImg(u) && !images.includes(u)) images.push(u);
+ }
+ }
+ } catch (e) {
+ if (CANARY_N > 0) console.error(` page-miss ${sku}: ${e.message}`);
+ }
+ // Specs from the same page (additive, never overwrites).
+ try {
+ const sp = await fetchProductSpecs(sku);
+ if (sp && sp.specs && Object.keys(sp.specs).length) specs = sp.specs;
+ } catch (e) { /* specs are best-effort */ }
+ await sleep(600);
+ }
+
+ // Keep the existing primary hero in the set too, then dedupe + filter.
+ if (r.image_url) images.unshift(r.image_url);
+ images = [...new Set(images.filter(isImg))];
+
+ if (!images.length) {
+ noImg++;
+ if (CANARY_N > 0) console.log(` ${sku}: no images`);
+ continue;
+ }
+
+ await pool.query(
+ `UPDATE ${TABLE}
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url, ''), $2),
+ specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
+ updated_at = NOW()
+ WHERE mfr_sku = $4`,
+ [JSON.stringify(images), images[0], JSON.stringify(specs), r.mfr_sku]
+ );
+ updated++; withImgs++;
+ if (Object.keys(specs).length) withSpecs++;
+
+ if (CANARY_N > 0) {
+ console.log(` ${sku}: ${images.length} imgs, ${Object.keys(specs).length} specs`);
+ console.log(` ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
+ } else if (n % 50 === 0) {
+ console.log(` ${n}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | w/specs ${withSpecs} | noImg ${noImg} | errs ${errs}`);
+ }
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0 || CANARY_N > 0) console.error(` err ${sku}: ${e.message}`);
+ await sleep(500);
+ }
+ }
+
+ console.log(`\nDONE. rows=${n} updated=${updated} with_imgs=${withImgs} with_specs=${withSpecs} no_img=${noImg} errors=${errs}`);
+ try { fs.unlinkSync(EXTRACT_TMP); } catch (_) {}
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
← 50a4c55 Add recrawl-hollyhunt.js: full-page all_images completeness
·
back to Dw Vendor Scrapers Local
·
coordonne recrawl: reject 3rd-party share-link images (pinte 0851c0c →