← back to Designer Wallcoverings
Add Designtex BigCommerce plain-HTTP enrichment scraper (no puppeteer, no login)
a509ba7ac274158e3315060889dc3d16deb5da1e · 2026-06-23 15:03:56 -0700 · Steve
DTD 3/3: replaces the broken puppeteer headless scraper. Parses the public
Stencil pattern pages by product_url -> JSON-LD + dt/dd spec table (37 fields:
Content/Width/Weight/Repeat/Flammability/Cleaning Code/Origin...) + per-colorway
swatch+gallery images. Writes sidecars to enrich-cache/designtex/. Smoke-tested
on /gilded/ (16 colorways, 37 specs). Connects via unix socket (peer auth).
Files touched
M .gitignoreA scripts/designtex-bigcommerce-enrich.js
Diff
commit a509ba7ac274158e3315060889dc3d16deb5da1e
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jun 23 15:03:56 2026 -0700
Add Designtex BigCommerce plain-HTTP enrichment scraper (no puppeteer, no login)
DTD 3/3: replaces the broken puppeteer headless scraper. Parses the public
Stencil pattern pages by product_url -> JSON-LD + dt/dd spec table (37 fields:
Content/Width/Weight/Repeat/Flammability/Cleaning Code/Origin...) + per-colorway
swatch+gallery images. Writes sidecars to enrich-cache/designtex/. Smoke-tested
on /gilded/ (16 colorways, 37 specs). Connects via unix socket (peer auth).
---
.gitignore | 1 +
scripts/designtex-bigcommerce-enrich.js | 188 ++++++++++++++++++++++++++++++++
2 files changed, 189 insertions(+)
diff --git a/.gitignore b/.gitignore
index f5b36f4c..b4d5ba07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,3 +72,4 @@ DW-Programming/data/color-clean-ledger.jsonl
shopify/scripts/gmc-race-clean/out/
shopify/scripts/cadence/stock-2026-guard.latest.json
shopify/theme-backups/
+enrich-cache/
diff --git a/scripts/designtex-bigcommerce-enrich.js b/scripts/designtex-bigcommerce-enrich.js
new file mode 100644
index 00000000..44972270
--- /dev/null
+++ b/scripts/designtex-bigcommerce-enrich.js
@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+/**
+ * Designtex BigCommerce (Stencil) full-page enrichment scraper.
+ *
+ * WHY: shop.designtex.com is BigCommerce Stencil (store s-uq1t2yn8xz). Each PATTERN
+ * is ONE product page (e.g. /gilded/) served as static HTML — no JS, NO LOGIN needed.
+ * The page contains, per pattern:
+ * - JSON-LD Product block (name, primary image)
+ * - a dt.productView-info-name / dd.productView-info-value spec table
+ * (Content, Finish, Backing, Width, Weight, Origin, UOM, Repeat (V x H),
+ * Flammability, Lightfastness, Cleaning Code, Average Bolt Size, ...)
+ * - per-colorway swatch images labelled "<ColorName> <full-sku>" (e.g. "Plaster 8380-051")
+ * - per-colorway gallery images named "<sku>__*.jpg" (flat) + "<sku>_D__*.jpg" (detail)
+ *
+ * DTD verdict 2026-06-23 (3/3): use a lightweight plain-HTTP fetch + cheerio parser
+ * (NOT puppeteer) keyed off the existing designtex_catalog.product_url values, feeding
+ * enrichment SIDECARS. This script writes ONLY sidecars + a manifest — it does NOT
+ * touch dw_unified or Shopify (that is the gated write step, drafted separately).
+ *
+ * Standing rules honored:
+ * - No hardcoded credentials (none needed — public pages).
+ * - FULL-PAGE-SCRAPE: captures ALL images + all specs + fire/flammability + repeat.
+ * - $0 (local network fetch only).
+ *
+ * Usage:
+ * node designtex-bigcommerce-enrich.js # crawl all distinct pattern pages
+ * node designtex-bigcommerce-enrich.js --limit 5 # crawl first 5 (smoke test)
+ * node designtex-bigcommerce-enrich.js --pattern gilded # one pattern by url-slug
+ */
+
+const fs = require('fs');
+const path = require('path');
+const cheerio = require('cheerio');
+const pg = require('pg');
+
+const ARGV = process.argv.slice(2);
+const argVal = (flag) => { const i = ARGV.indexOf(flag); return i >= 0 ? ARGV[i + 1] : null; };
+const LIMIT = argVal('--limit') ? parseInt(argVal('--limit'), 10) : null;
+const ONE_PATTERN = argVal('--pattern');
+
+const OUT_DIR = path.join(__dirname, '..', 'enrich-cache', 'designtex');
+fs.mkdirSync(OUT_DIR, { recursive: true });
+
+// Connect over the unix socket (host=/tmp) so peer auth applies — matches `psql -d dw_unified`
+// for the stevestudio2 OS user (no password). TCP/localhost would force SCRAM with no password.
+const pgClient = new pg.Client({ user: 'stevestudio2', host: '/tmp', database: 'dw_unified', port: 5432 });
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// "8380-051" -> "8380051" (DB stores mfr_sku without the dash)
+const normSku = (s) => (s || '').replace(/[^0-9A-Za-z]/g, '').toUpperCase();
+
+function parseSpecTable($) {
+ const specs = {};
+ // Pattern Type can repeat; collect into an array.
+ const patternTypes = [];
+ $('dt.productView-info-name').each((_, dt) => {
+ const label = $(dt).text().replace(/:\s*$/, '').replace(/\s+/g, ' ').trim();
+ const val = $(dt).next('dd.productView-info-value').text().replace(/\s+/g, ' ').trim();
+ if (!label || !val) return;
+ if (/^Pattern Type$/i.test(label)) { patternTypes.push(val); return; }
+ specs[label] = val;
+ });
+ if (patternTypes.length) specs['Pattern Type'] = patternTypes.join(', ');
+ return specs;
+}
+
+function parseColorways($) {
+ // Each swatch <img src=...attribute_rule_images...> sits inside a label/li whose
+ // text is "<ColorName> <full-sku>" e.g. "Plaster 8380-051".
+ const out = [];
+ $('img[src*="attribute_rule_images"], img[data-src*="attribute_rule_images"]').each((_, el) => {
+ const ctx = $(el).closest('label,li,div').text().replace(/\s+/g, ' ').trim();
+ const m = ctx.match(/^(.*?)\s+([0-9]{2,}-[0-9]{2,}|[0-9]{5,})\s*$/);
+ let name = ctx, skuRaw = '';
+ if (m) { name = m[1].trim(); skuRaw = m[2].trim(); }
+ const swatch = ($(el).attr('src') || $(el).attr('data-src') || '').replace(/^\/\//, 'https://');
+ out.push({ color_name: name, mfr_sku: normSku(skuRaw), sku_raw: skuRaw, swatch_image: swatch });
+ });
+ return out;
+}
+
+function parseGalleryImages($) {
+ // Gallery product photos (not swatches). Map by leading numeric sku in filename.
+ const bySku = {};
+ $('img[src*="images/stencil"], img[data-src*="images/stencil"]').each((_, el) => {
+ let s = ($(el).attr('src') || $(el).attr('data-src') || '');
+ if (!s.includes('/products/')) return;
+ s = s.replace(/^\/\//, 'https://').replace(/\?.*$/, '');
+ const fn = s.split('/').pop(); // e.g. 8380051_D__14704.1713201930.jpg
+ const skuMatch = fn.match(/^([0-9]{5,})/);
+ if (!skuMatch) return;
+ const sku = normSku(skuMatch[1]);
+ (bySku[sku] = bySku[sku] || []);
+ if (!bySku[sku].includes(s)) bySku[sku].push(s);
+ });
+ return bySku;
+}
+
+function parseJsonLd($) {
+ let prod = null;
+ $('script[type="application/ld+json"]').each((_, el) => {
+ try {
+ const j = JSON.parse($(el).contents().text());
+ if (j && (j['@type'] === 'Product' || (Array.isArray(j['@graph']) && j['@graph'].some((g) => g['@type'] === 'Product')))) {
+ prod = j['@type'] === 'Product' ? j : j['@graph'].find((g) => g['@type'] === 'Product');
+ }
+ } catch (_) { /* ignore */ }
+ });
+ return prod;
+}
+
+async function fetchPattern(url) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'text/html' }, redirect: 'follow' });
+ const status = res.status;
+ const finalUrl = res.url;
+ const html = await res.text();
+ // Discontinued detection: redirect away from the product slug or tiny body.
+ const discontinued = status >= 300 && status < 400 ? true
+ : (!finalUrl.includes(new URL(url).pathname.replace(/\/$/, '')) && new URL(url).pathname !== '/');
+ return { status, finalUrl, html, discontinued };
+}
+
+async function main() {
+ await pgClient.connect();
+ let q = `SELECT DISTINCT product_url, pattern_name FROM designtex_catalog
+ WHERE product_url IS NOT NULL AND product_url <> '' `;
+ const params = [];
+ if (ONE_PATTERN) { params.push(`%/${ONE_PATTERN}%`); q += ` AND product_url ILIKE $1`; }
+ q += ` ORDER BY product_url`;
+ let { rows } = await pgClient.query(q, params);
+ if (LIMIT) rows = rows.slice(0, LIMIT);
+
+ console.log(`Designtex BigCommerce enrich — ${rows.length} pattern pages | $0 (local fetch)`);
+ const manifest = [];
+ let ok = 0, fail = 0, disc = 0;
+
+ for (let i = 0; i < rows.length; i++) {
+ const { product_url, pattern_name } = rows[i];
+ const slug = (new URL(product_url).pathname.replace(/\//g, '') || 'root');
+ try {
+ const { status, finalUrl, html, discontinued } = await fetchPattern(product_url);
+ if (status >= 400 || !html || html.length < 2000) {
+ fail++; manifest.push({ product_url, slug, status, ok: false, reason: 'bad-response' });
+ console.log(` [${i + 1}/${rows.length}] ${slug} FAIL status=${status} len=${html ? html.length : 0}`);
+ await sleep(500); continue;
+ }
+ const $ = cheerio.load(html);
+ const specs = parseSpecTable($);
+ const colorways = parseColorways($);
+ const galleryBySku = parseGalleryImages($);
+ const jsonld = parseJsonLd($);
+
+ // attach gallery images to each colorway by sku
+ colorways.forEach((c) => { c.images = galleryBySku[c.mfr_sku] || []; });
+
+ const sidecar = {
+ product_url, final_url: finalUrl, slug, pattern_name,
+ scraped_at: new Date().toISOString(), status,
+ discontinued: discontinued || false,
+ specs, // raw label->value map
+ colorways, // [{color_name, mfr_sku, swatch_image, images[]}]
+ json_ld: jsonld || null,
+ all_gallery_skus: Object.keys(galleryBySku),
+ };
+ fs.writeFileSync(path.join(OUT_DIR, `${slug}.json`), JSON.stringify(sidecar, null, 2));
+ ok++;
+ if (discontinued) disc++;
+ manifest.push({ product_url, slug, status, ok: true, discontinued: sidecar.discontinued,
+ n_colorways: colorways.length, n_specs: Object.keys(specs).length });
+ if ((i + 1) % 20 === 0 || i === rows.length - 1)
+ console.log(` [${i + 1}/${rows.length}] ${slug} ok colorways=${colorways.length} specs=${Object.keys(specs).length}`);
+ } catch (e) {
+ fail++; manifest.push({ product_url, slug, ok: false, reason: e.message });
+ console.log(` [${i + 1}/${rows.length}] ${slug} ERROR ${e.message}`);
+ }
+ await sleep(600); // polite gap
+ }
+
+ fs.writeFileSync(path.join(OUT_DIR, '_manifest.json'), JSON.stringify({
+ generated_at: new Date().toISOString(), total: rows.length, ok, fail, discontinued: disc, items: manifest,
+ }, null, 2));
+ console.log(`\nDONE. ok=${ok} fail=${fail} discontinued=${disc}. Sidecars in ${OUT_DIR}`);
+ await pgClient.end();
+}
+
+main().catch((e) => { console.error('FATAL', e); process.exit(1); });
← dfed2390 RESUME: add mid-flight resume instructions + CS live-write/r
·
back to Designer Wallcoverings
·
RESUME: record CS step 3 no-op finding + steps 3d N/A 5df32f5e →