← back to Dw Vendor Scrapers Local
recrawl-schumacher.js
204 lines
#!/usr/bin/env node
// ============================================================================
// Schumacher ADDITIVE all_images re-crawl — full-page completeness pass
// ============================================================================
// WHY: schumacher_catalog (→ vendor_catalog vendor_code='schumacher', 6,474
// rows) was image-enriched ONLY for rows missing the PRIMARY image. The
// deployed enrich-schumacher-images.js selects `WHERE image_url IS NULL OR
// image_url = ''`, so the 5,766 rows that already have a primary hero but an
// EMPTY all_images array are never re-touched. Steve's standing rule "always
// pull all data and images" requires every product to carry its FULL image set
// (all angles + per-colorway swatches) + specs before it is import-ready, so a
// populated image_url is NOT sufficient — all_images must be non-empty.
//
// 2026-06-10 gap (local dw_unified):
// * vendor_catalog vendor_code='schumacher' missing all_images: 5,766
// * net-new (shopify_product_id IS NULL) missing all_images: 3,438
// — and 100% of those already have a primary image_url, i.e. the existing
// enrich job's row-selection skips every single one of them.
// * schumacher_catalog (source table this script writes) missing
// all_images: 4,474 (all have product_url + mfr_sku).
//
// FIX vs the existing enrich script: same proven extractors, different
// row-selection. We re-use the deployed enrich-schumacher-images.js functions
// (discoverCdnImages = CDN pattern + {SKU}-1..8 alt-image probe,
// extractImagesFromHtml = full product-page gallery scrape,
// fetchProductSpecs = page specs) so the all_images CAPTURE logic is
// identical to the reviewed prod path — we only widen WHICH rows get processed
// (all_images-empty, regardless of whether image_url is already set). Mirrors
// the load-deployed-scraper harness of recrawl-osborne.js and the additive
// safety contract of recrawl-versa_ds.js.
//
// SAFETY / ADDITIVE-ONLY:
// * NEVER inserts new rows. Only UPDATEs all_images / specs / image_url on
// rows that already exist (matched by mfr_sku).
// * Only touches rows whose all_images is empty.
// * Does NOT push to Shopify, does NOT publish, does NOT change SKUs/prices.
// * After this populates schumacher_catalog.all_images, the existing
// consolidate-vendor-catalogs.py sync propagates it to vendor_catalog
// (both tables track all_images; run the consolidate step to surface it).
//
// MODES:
// --canary N process only the first N rows (default 5) — run FIRST on
// Kamatera to confirm the deployed extractors still match live
// HTML / CDN before the full pass.
// --dry-run READ-ONLY: extract + print images/specs but SKIP the DB
// UPDATE entirely. Mandatory for the validation canary so no
// bad write can touch prod before the extraction is eyeballed.
// --all process every eligible row (full ~4,474-row pass).
// --net-new restrict to rows whose mfr_sku is NOT already in
// shopify_products (skip the already-live cohort).
//
// Deploy target: Kamatera /root/DW-Agents/vendor-scrapers/. Authored from the
// local mirror; FIRST run on Kamatera MUST be --canary to validate extraction.
// This file is a PARKED artifact — see
// ~/.claude/yolo-queue/pending-approval/schumacher-recrawl.md for the exact
// deploy + run + consolidate commands. Do NOT run against prod without Steve.
// ============================================================================
require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const fs = require('fs');
const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
const TABLE = 'schumacher_catalog';
const ENRICH_SRC = '/root/DW-Agents/vendor-scrapers/enrich-schumacher-images.js';
const EXTRACT_TMP = '/root/DW-Agents/vendor-scrapers/_sch_extract.js';
// Load the deployed enrich script's extractors WITHOUT running its main().
// Strip the trailing `main().catch(...)` invocation, append a module.exports of
// the functions we need (they are top-level declarations, so in scope here).
let src = fs.readFileSync(ENRICH_SRC, 'utf8')
.replace(/main\(\)\.catch\([\s\S]*$/,
'module.exports = { discoverCdnImages, fetchProductSpecs, extractImagesFromHtml, httpGet, checkUrlExists };\n');
fs.writeFileSync(EXTRACT_TMP, src);
const { discoverCdnImages, fetchProductSpecs, extractImagesFromHtml, httpGet } = require(EXTRACT_TMP);
const argv = process.argv.slice(2);
const FULL = argv.includes('--all');
const NET_NEW_ONLY = argv.includes('--net-new');
const DRY_RUN = argv.includes('--dry-run');
const canaryIdx = argv.indexOf('--canary');
const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
// Junk filter — never store logos, icons, sprites, payment badges as images.
const JUNK = /(logo|icon|favicon|placeholder|sprite|payment|visa|mastercard|paypal|flag|avatar|loader|spinner)/i;
function isImg(u) {
if (!u) return false;
if (!/\.(jpe?g|png|webp)(\?.*)?$/i.test(u)) return false;
if (JUNK.test(u)) return false;
return true;
}
(async () => {
let sql = `
SELECT mfr_sku,
split_part(product_url,'?',1) AS url,
image_url
FROM ${TABLE}
WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = '""'
OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%')
AND mfr_sku IS NOT NULL AND mfr_sku <> ''
`;
if (NET_NEW_ONLY) {
sql += ` AND NOT EXISTS (SELECT 1 FROM shopify_products s
WHERE upper(trim(s.mfr_sku)) = upper(trim(${TABLE}.mfr_sku)))`;
}
sql += ` ORDER BY mfr_sku`;
if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;
const { rows } = await pool.query(sql);
console.log(`Schumacher additive all_images re-crawl — ${rows.length} rows`
+ `${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}`
+ `${NET_NEW_ONLY ? ' [net-new only]' : ''}`
+ `${DRY_RUN ? ' [DRY-RUN — NO DB WRITES]' : ''}`);
let n = 0, updated = 0, withImgs = 0, withSpecs = 0, noImg = 0, errs = 0;
for (const r of rows) {
n++;
const sku = String(r.mfr_sku).trim();
try {
let images = [];
// (1) CDN pattern + {SKU}-1..8 alt-image probe (deployed extractor).
try {
const cdn = await discoverCdnImages(sku);
if (cdn && Array.isArray(cdn.allImages)) images.push(...cdn.allImages);
} catch (e) {
if (CANARY_N > 0) console.error(` cdn-miss ${sku}: ${e.message}`);
}
// (2) Full product-page gallery scrape (deployed extractor) — the real
// "all data and images" source: every gallery angle + colorway swatch.
const pageUrl = (r.url && /^https?:\/\//i.test(r.url)) ? r.url : null;
let specs = {};
if (pageUrl) {
try {
const res = await httpGet(pageUrl, 4);
const body = (res && (res.body || res)) || '';
if (typeof body === 'string' && body.length) {
for (const u of (extractImagesFromHtml(body, sku) || [])) {
if (isImg(u) && !images.includes(u)) images.push(u);
}
}
} catch (e) {
if (CANARY_N > 0) console.error(` page-miss ${sku}: ${e.message}`);
}
// Specs from the same page (additive, never overwrites).
try {
const sp = await fetchProductSpecs(sku);
if (sp && sp.specs && Object.keys(sp.specs).length) specs = sp.specs;
} catch (e) { /* specs are best-effort */ }
await sleep(600);
}
// Keep the existing primary hero in the set too, then dedupe + filter.
if (r.image_url) images.unshift(r.image_url);
images = [...new Set(images.filter(isImg))];
if (!images.length) {
noImg++;
if (CANARY_N > 0) console.log(` ${sku}: no images`);
continue;
}
if (DRY_RUN) {
// READ-ONLY canary: extract + print, but NEVER touch the DB so a bad
// extraction can't write to prod before it is eyeballed.
withImgs++;
if (Object.keys(specs).length) withSpecs++;
console.log(` [dry-run] ${sku}: ${images.length} imgs, ${Object.keys(specs).length} specs (NOT written)`);
console.log(` ${images.slice(0, 5).join('\n ')}${images.length > 5 ? '\n …' : ''}`);
continue;
}
await pool.query(
`UPDATE ${TABLE}
SET all_images = $1,
image_url = COALESCE(NULLIF(image_url, ''), $2),
specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
updated_at = NOW()
WHERE mfr_sku = $4`,
[JSON.stringify(images), images[0], JSON.stringify(specs), r.mfr_sku]
);
updated++; withImgs++;
if (Object.keys(specs).length) withSpecs++;
if (CANARY_N > 0) {
console.log(` ${sku}: ${images.length} imgs, ${Object.keys(specs).length} specs`);
console.log(` ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
} else if (n % 50 === 0) {
console.log(` ${n}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | w/specs ${withSpecs} | noImg ${noImg} | errs ${errs}`);
}
} catch (e) {
errs++;
if (errs % 10 === 0 || CANARY_N > 0) console.error(` err ${sku}: ${e.message}`);
await sleep(500);
}
}
console.log(`\nDONE. rows=${n} updated=${updated} with_imgs=${withImgs} with_specs=${withSpecs} no_img=${noImg} errors=${errs}`);
try { fs.unlinkSync(EXTRACT_TMP); } catch (_) {}
await pool.end();
})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });