← back to Dw Vendor Scrapers Local
Add recrawl-designers-guild.js: additive all_images re-crawl (1->5 imgs, /1024/ gallery scoping)
ec029c3ccda243892e69ce114d762e93cd46fdba · 2026-06-10 11:46:46 -0700 · SteveStudio2
Files touched
A recrawl-designers-guild.js
Diff
commit ec029c3ccda243892e69ce114d762e93cd46fdba
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed Jun 10 11:46:46 2026 -0700
Add recrawl-designers-guild.js: additive all_images re-crawl (1->5 imgs, /1024/ gallery scoping)
---
recrawl-designers-guild.js | 174 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 174 insertions(+)
diff --git a/recrawl-designers-guild.js b/recrawl-designers-guild.js
new file mode 100644
index 0000000..d2d3d7a
--- /dev/null
+++ b/recrawl-designers-guild.js
@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+// ============================================================================
+// Designers Guild ADDITIVE all_images re-crawl — Codename: Diana (full-page pass)
+// ============================================================================
+// WHY: the deployed scrape-designers-guild.js captures only ONE image per
+// product — the first `imageId` (→ /image-nonwebp/600/<id>), with og:image as a
+// fallback. Steve's standing rule "always pull all data and images" requires
+// every product to carry its FULL image set (cutting + roomsets + per-colourway
+// swatches) before it is import-ready. As of 2026-06-10 ALL 2,163 designers_guild
+// rows in vendor_catalog have an empty all_images. This driver re-crawls the
+// actual product PAGE for every eligible row and fills all_images additively.
+//
+// IMAGE SCOPING (confirmed against live HTML 2026-06-10):
+// A Designers Guild product page is a ~900KB SSR JSON page that also embeds
+// thumbnails of *related* products. The product's OWN gallery is exactly the
+// set of imageIds that the page references at the full /image-nonwebp/1024/<id>
+// size — related-product cards only ever appear at the 156/239 thumb sizes.
+// So: collect every distinct id in `/image-nonwebp/1024/<id>`; if none are
+// present, fall back to the ids inside the product's "images":[{imageId}] JSON
+// arrays; final fallback = the existing hero image_url. Each id is stored as a
+// canonical https://www.designersguild.com/image-nonwebp/1024/<id> URL.
+//
+// SAFETY / ADDITIVE-ONLY:
+// * NEVER inserts rows. UPDATE only, matched by (vendor_code,mfr_sku).
+// * Only touches rows whose all_images is empty AND whose product_url is a real
+// designersguild.com page. The 869 URL-less rows (all in the already-on-Shopify
+// / dup cohort) are skipped — they are dedup'd out at publish anyway.
+// * Does NOT push to Shopify, does NOT publish, does NOT change SKUs.
+// * Targets vendor_catalog (vendor_code='designers_guild') — the unified table
+// where the all_images-completeness + never-duplicate gates actually read.
+//
+// MODES:
+// --canary N re-crawl only the first N rows (default 5) — RUN THIS FIRST to
+// confirm the /1024/ gallery heuristic against live HTML.
+// --all re-crawl every eligible row (the full pass).
+// --net-new restrict to rows whose mfr_sku is NOT already in shopify_products
+// (the genuine net-net-new push-prep cohort: 985 rows on 2026-06-10).
+//
+// Mirrors the harness of the FIXED recrawl-osborne.js / recrawl-coordonne.js
+// (httpGet + worker pool). Designed to run ON Kamatera (/root paths, scraper-utils).
+// ============================================================================
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const https = require('https');
+const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+const VENDOR_CODE = 'designers_guild';
+const SITE_BASE = 'https://www.designersguild.com';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+const argv = process.argv.slice(2);
+const FULL = argv.includes('--all');
+const NET_NEW_ONLY = argv.includes('--net-new');
+const canaryIdx = argv.indexOf('--canary');
+const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
+
+function httpGet(url, maxRedirects = 5) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, {
+ timeout: 30000,
+ headers: {
+ 'User-Agent': UA,
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ 'Cookie': 'country=US; currency=USD',
+ },
+ }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+const decode = (s) => s
+ .replace(/"/g, '"').replace(/"/g, '"')
+ .replace(/&/g, '&').replace(/&/g, '&');
+
+// Extract the product's OWN gallery imageIds, scoped via the /1024/ full-size
+// heuristic, with the embedded "images":[{imageId}] JSON arrays as a fallback.
+function extractImageIds(html) {
+ const ids = [];
+ const seen = new Set();
+ const add = (id) => { if (id && !seen.has(id)) { seen.add(id); ids.push(id); } };
+
+ // 1) Canonical: every full-size /image-nonwebp/1024/<id> the product references.
+ let m;
+ const reFull = /\/image-nonwebp\/1024\/(\d+)/g;
+ while ((m = reFull.exec(html))) add(m[1]);
+
+ // 2) Fallback: ids inside the product's "images":[{"imageId":N}] JSON arrays
+ // (HTML-entity-encoded in the SSR payload).
+ if (!ids.length) {
+ const d = decode(html);
+ const reArr = /"images":\[(.*?)\]/g;
+ let a;
+ while ((a = reArr.exec(d))) {
+ let im; const reId = /"imageId":(\d+)/g;
+ while ((im = reId.exec(a[1]))) add(im[1]);
+ }
+ }
+ return ids;
+}
+
+const idToUrl = (id) => `${SITE_BASE}/image-nonwebp/1024/${id}`;
+
+(async () => {
+ let sql = `
+ SELECT mfr_sku, split_part(product_url,'?',1) AS url, image_url
+ FROM vendor_catalog
+ WHERE vendor_code = '${VENDOR_CODE}'
+ AND (all_images IS NULL OR all_images = '' OR all_images = '[]')
+ AND product_url ~ 'designersguild\\.com'
+ `;
+ if (NET_NEW_ONLY) {
+ sql += ` AND upper(btrim(mfr_sku)) NOT IN
+ (SELECT upper(btrim(mfr_sku)) FROM shopify_products WHERE mfr_sku IS NOT NULL AND mfr_sku <> '')`;
+ }
+ sql += ` ORDER BY mfr_sku`;
+ if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;
+
+ const { rows } = await pool.query(sql);
+ console.log(`Designers Guild additive all_images re-crawl — ${rows.length} pages${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}${NET_NEW_ONLY ? ' [net-new only]' : ''}`);
+
+ let pages = 0, updated = 0, withImgs = 0, multiImg = 0, noImgs = 0, errs = 0;
+ const concurrency = 3;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const { mfr_sku, url, image_url } = rows[i];
+ try {
+ const html = await httpGet(url);
+ let ids = extractImageIds(html);
+ let images = ids.map(idToUrl);
+ // final fallback: keep the existing hero if the page yielded nothing
+ if (!images.length && image_url) images = [image_url];
+ pages++;
+ if (!images.length) { noImgs++; await sleep(300); continue; }
+ await pool.query(
+ `UPDATE vendor_catalog
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url,''), $2),
+ last_scraped_at = NOW()
+ WHERE vendor_code = $3 AND mfr_sku = $4`,
+ [JSON.stringify(images), images[0], VENDOR_CODE, mfr_sku]
+ );
+ updated++; withImgs++;
+ if (images.length > 1) multiImg++;
+ if (CANARY_N > 0) {
+ console.log(` ${mfr_sku}: ${images.length} imgs`);
+ console.log(` ${images.slice(0, 4).join('\n ')}${images.length > 4 ? '\n …' : ''}`);
+ } else if (pages % 50 === 0) {
+ console.log(` ${pages}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | multi ${multiImg} | noImgs ${noImgs} | errs ${errs}`);
+ }
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0 || CANARY_N > 0) console.error(` err ${mfr_sku} ${url}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: concurrency }, worker));
+ console.log(`\nDONE. pages=${pages} updated=${updated} with_imgs=${withImgs} multi_image=${multiImg} no_imgs=${noImgs} errors=${errs}`);
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
← 2933b61 Add recrawl-coordonne.js: additive all_images re-crawl + loc
·
back to Dw Vendor Scrapers Local
·
README: document recrawl-designers-guild.js 652003b →