← back to Designer Wallcoverings
auto-save: 2026-07-10T11:38:04 (7 files) — pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price vendor-scrapers/china-seas-refresh DW-Programming/schu-refresh-harvest.mjs DW-Programming/schu_pillow_apisniff.mjs
4bfcd4e0b34eadd9c54820bae0332a7c844ab69b · 2026-07-10 11:38:09 -0700 · Steve Abrams
Files touched
A DW-Programming/schu-refresh-harvest.mjsA DW-Programming/schu_pillow_apisniff.mjsA DW-Programming/scrapers/designers-guild/README.mdA DW-Programming/scrapers/designers-guild/auth-price.jsA DW-Programming/scrapers/designers-guild/enrich.jsA DW-Programming/scrapers/designers-guild/parse.jsA DW-Programming/scrapers/designers-guild/scrape.jsA _dw-login-record.mjs
Diff
commit 4bfcd4e0b34eadd9c54820bae0332a7c844ab69b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Jul 10 11:38:09 2026 -0700
auto-save: 2026-07-10T11:38:04 (7 files) — pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price vendor-scrapers/china-seas-refresh DW-Programming/schu-refresh-harvest.mjs DW-Programming/schu_pillow_apisniff.mjs
---
DW-Programming/schu-refresh-harvest.mjs | 122 +++++++++
DW-Programming/schu_pillow_apisniff.mjs | 32 +++
DW-Programming/scrapers/designers-guild/README.md | 69 +++++
.../scrapers/designers-guild/auth-price.js | 299 +++++++++++++++++++++
DW-Programming/scrapers/designers-guild/enrich.js | 111 ++++++++
DW-Programming/scrapers/designers-guild/parse.js | 163 +++++++++++
DW-Programming/scrapers/designers-guild/scrape.js | 175 ++++++++++++
_dw-login-record.mjs | 86 ++++++
8 files changed, 1057 insertions(+)
diff --git a/DW-Programming/schu-refresh-harvest.mjs b/DW-Programming/schu-refresh-harvest.mjs
new file mode 100644
index 00000000..a443f83d
--- /dev/null
+++ b/DW-Programming/schu-refresh-harvest.mjs
@@ -0,0 +1,122 @@
+// Schumacher STAGING refresh harvester — writes ONLY to dw_unified.schumacher_catalog.
+// Anonymous api.schumacher.com (no login, $0). Prices are gated (login-only) -> left NULL.
+// Usage: node schu-refresh-harvest.mjs <categoryId> <product_type> [--pilot N]
+import pg from 'pg';
+
+const API = 'https://api.schumacher.com';
+const H = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36', 'Referer': 'https://schumacher.com/', 'Origin': 'https://schumacher.com' };
+
+const categoryId = process.argv[2];
+const productType = process.argv[3];
+const pilotIdx = process.argv.indexOf('--pilot');
+const pilotN = pilotIdx > -1 ? parseInt(process.argv[pilotIdx + 1]) : 0;
+if (!categoryId || !productType) { console.error('usage: <categoryId> <product_type> [--pilot N]'); process.exit(1); }
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+async function getJSON(url, opt = {}, tries = 3) {
+ for (let i = 0; i < tries; i++) {
+ try {
+ const r = await fetch(url, { headers: H, ...opt });
+ if (r.ok) return await r.json();
+ if (r.status === 500 && i < tries - 1) { await sleep(800); continue; }
+ return null;
+ } catch (e) { if (i === tries - 1) return null; await sleep(800); }
+ }
+}
+const attr = (v, slug) => {
+ const a = (v.attributes || []).find(x => x.slug === slug);
+ if (!a) return null;
+ const val = a.value;
+ if (Array.isArray(val) && val.length) return val[0].value;
+ return typeof val === 'string' ? val : null;
+};
+const img = (v) => {
+ if (!v.images || !v.images.length) return null;
+ return v.images[0].largeUrl || v.images[0].thumbnailUrl || null;
+};
+
+// 1) page through the category listing
+const SIZE = 24;
+let page = 0, totalPages = 1, all = [];
+while (page < totalPages) {
+ const d = await getJSON(`${API}/catalog/entries?categoryId=${categoryId}&size=${SIZE}&page=${page}`);
+ if (!d) { console.error('page', page, 'failed'); break; }
+ totalPages = d.totalPages;
+ for (const grp of (d.content || [])) for (const v of (grp.variations || [])) all.push(v);
+ if (page === 0) console.log(`total=${d.totalElements} pages=${totalPages}`);
+ page++;
+ if (pilotN && all.length >= pilotN) break;
+ await sleep(120);
+}
+// dedupe by itemNumber
+const byItem = new Map();
+for (const v of all) if (v.itemNumber && !byItem.has(v.itemNumber)) byItem.set(v.itemNumber, v);
+let rows = [...byItem.values()];
+if (pilotN) rows = rows.slice(0, pilotN);
+console.log(`harvested ${rows.length} unique variations`);
+
+// 2) enrich collection+description from /products per SKU (bounded concurrency)
+async function enrich(v) {
+ const d = await getJSON(`${API}/products?itemNumber=${encodeURIComponent(v.itemNumber)}`);
+ const c = d && d.content && d.content[0];
+ return {
+ collection: c && c.collection ? c.collection.name : null,
+ description: c ? c.description : null,
+ };
+}
+const enriched = {};
+const CONC = 8;
+for (let i = 0; i < rows.length; i += CONC) {
+ const batch = rows.slice(i, i + CONC);
+ const res = await Promise.all(batch.map(enrich));
+ batch.forEach((v, j) => enriched[v.itemNumber] = res[j]);
+ if (i % 96 === 0) process.stdout.write(` enriched ${Math.min(i + CONC, rows.length)}/${rows.length}\r`);
+ await sleep(80);
+}
+console.log(`\nenrichment done`);
+
+// 3) upsert
+const client = new pg.Client({ database: 'dw_unified' });
+await client.connect();
+let ins = 0, upd = 0;
+for (const v of rows) {
+ const e = enriched[v.itemNumber] || {};
+ const size = attr(v, 'size');
+ const soldBy = attr(v, 'sold-by');
+ const pricedBy = attr(v, 'priced-by');
+ const typ = attr(v, 'type');
+ const priceUsd = v.priceUsd != null ? v.priceUsd : null; // gated -> null
+ const q = `
+ INSERT INTO schumacher_catalog
+ (mfr_sku, pattern_name, color_name, collection, product_type, width,
+ image_url, product_url, description, material, sold_by, priced_by,
+ brand, retail_price, price_unit, crawled_at, updated_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,'Schumacher',$13,$14, now(), now())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name = COALESCE(EXCLUDED.pattern_name, schumacher_catalog.pattern_name),
+ color_name = COALESCE(EXCLUDED.color_name, schumacher_catalog.color_name),
+ collection = COALESCE(EXCLUDED.collection, schumacher_catalog.collection),
+ product_type = EXCLUDED.product_type,
+ width = COALESCE(EXCLUDED.width, schumacher_catalog.width),
+ image_url = COALESCE(EXCLUDED.image_url, schumacher_catalog.image_url),
+ product_url = COALESCE(EXCLUDED.product_url, schumacher_catalog.product_url),
+ description = COALESCE(EXCLUDED.description, schumacher_catalog.description),
+ sold_by = COALESCE(EXCLUDED.sold_by, schumacher_catalog.sold_by),
+ priced_by = COALESCE(EXCLUDED.priced_by, schumacher_catalog.priced_by),
+ retail_price = COALESCE(EXCLUDED.retail_price, schumacher_catalog.retail_price),
+ price_unit = COALESCE(EXCLUDED.price_unit, schumacher_catalog.price_unit),
+ crawled_at = now(),
+ updated_at = now()
+ RETURNING (xmax = 0) AS inserted`;
+ const vals = [
+ v.itemNumber, v.name || null, v.colorName || null, e.collection || null, productType, size || null,
+ img(v), `https://schumacher.com/catalog/products/${v.itemNumber}`, e.description || null,
+ typ || null, soldBy || null, pricedBy || null, priceUsd, pricedBy ? pricedBy.toLowerCase() : null,
+ ];
+ try {
+ const r = await client.query(q, vals);
+ if (r.rows[0].inserted) ins++; else upd++;
+ } catch (err) { console.error('upsert fail', v.itemNumber, err.message); }
+}
+await client.end();
+console.log(`DONE product_type=${productType}: inserted=${ins} updated=${upd}`);
diff --git a/DW-Programming/schu_pillow_apisniff.mjs b/DW-Programming/schu_pillow_apisniff.mjs
new file mode 100644
index 00000000..57491f60
--- /dev/null
+++ b/DW-Programming/schu_pillow_apisniff.mjs
@@ -0,0 +1,32 @@
+// ANONYMOUS (no login) — load pillows category, capture the client JSON API that returns variations,
+// then replay it page-by-page to harvest all 782 pillow variations into a JSONL.
+import { chromium } from 'playwright';
+import fs from 'fs';
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
+const browser = await chromium.launch({ headless: true });
+const ctx = await browser.newContext({ userAgent: UA });
+const page = await ctx.newPage();
+
+let apiUrl = null;
+let apiHeaders = null;
+const seen = [];
+page.on('request', (r) => {
+ const u = r.url();
+ if (/variation|products\/search|catalog.*product|api/i.test(u) && r.resourceType() === 'fetch') {
+ seen.push(r.method() + ' ' + u.slice(0, 160));
+ if (/variation|search/i.test(u) && !apiUrl) { apiUrl = u; apiHeaders = r.headers(); }
+ }
+});
+
+await page.goto('https://schumacher.com/catalog/7', { waitUntil: 'domcontentloaded', timeout: 40000 });
+await page.waitForTimeout(6000);
+// try to trigger pagination / more loads by scrolling
+for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 4000); await page.waitForTimeout(1500); }
+
+console.log('=== fetch requests seen ===');
+console.log(seen.slice(0, 20).join('\n'));
+console.log('=== candidate apiUrl ===', apiUrl);
+if (apiHeaders) console.log('=== apiHeaders ===', JSON.stringify({ authorization: apiHeaders.authorization, 'x-api-key': apiHeaders['x-api-key'], origin: apiHeaders.origin, referer: apiHeaders.referer }));
+
+await browser.close();
diff --git a/DW-Programming/scrapers/designers-guild/README.md b/DW-Programming/scrapers/designers-guild/README.md
new file mode 100644
index 00000000..cc084825
--- /dev/null
+++ b/DW-Programming/scrapers/designers-guild/README.md
@@ -0,0 +1,69 @@
+# Designers Guild — dedicated scraper
+
+Vendor-scoped scraper for **Designers Guild** (designersguild.com). Writes
+**STAGING ONLY** → `dw_unified.designers_guild_catalog`. Never touches
+`shopify_products`; never pushes to Shopify. Go-live is separate + Steve-gated.
+
+Created per Steve's hard rule: *every DW vendor gets its own dedicated scraper*
+(previously DG only had the shared `product-import-from-url` engine + an ad-hoc
+`Letsbegin/scrape_dg_specs.js` enrichment pass).
+
+## Platform
+
+- **ASP.NET MVC 5 + Knockout.js** SPA behind **Cloudflare** (`x-powered-by:
+ ASP.NET`, `x-aspnetmvc-version: 5.2`). Not Shopify / Woo / Magento.
+- **Feed-first WORKS anonymously, $0** — no login, no Browserbase, no proxy,
+ no captcha. Plain `https` GET on any product page (`/en-us/.../pNNNN`)
+ returns HTTP 200 (`x-bot: false`) with a rich, HTML-entity-encoded product
+ model embedded in the SSR HTML, including a `TechInfo` block:
+ - width, weight, horizontal/vertical pattern repeat, composition,
+ patternMatch, rollLength, additionalInfo (paste/washability), aftercare,
+ - `attributes` (Brand / Pattern / Type / Design), parent `collection`,
+ GTIN-13, product code, and gallery `imageId`s.
+- **Enumeration** is via the anonymous server-rendered search page:
+ `/en-us/search-results/l76?search-term=<term>&page=<N>` (48 products/page,
+ server-rendered `/pNNNN` links). Existing rows already carry `product_url`s,
+ so the normal refresh just re-fetches those.
+
+## PRICE — the one gated field ⚠️
+
+`priceMin` / `priceMax` / `displayPrice` are **null on the anonymous SSR page
+for every region** (verified en-us AND en-gb). Retail pricing for
+wallpaper/fabric **rolls** is fetched by a **post-login AJAX call** the SPA
+makes only after trade authentication. The login is a Knockout SPA POST (no
+plain form action), so a blind curl login risks the one-attempt/lockout rule.
+
+**What IS priced anonymously:** accessories — cushions (`CCDG`), rugs
+(`RUGDG`), trims (`TRDG`), gift cards (`VOUCHER`). ~8/19 in the pilot.
+**What is NOT:** the actual wallpaper rolls (`PDG`/`PCL`/`PRL`/`PQ`/`P*`) and
+fabric rolls (`FDG`) — i.e. the ~842 active target SKUs.
+
+To fill roll prices you need an **authed pass** (Browserbase headless login
+with `DESIGNERSGUILD_PORTAL_USER/PASS`, then read the price AJAX per product).
+That is metered spend and is **gated to Steve** — not run by this module.
+
+## Usage
+
+```bash
+cd scrapers/designers-guild
+
+node scrape.js --url <productUrl> # single page (debug)
+node scrape.js --pilot 20 # re-fetch 20 existing rows → upsert
+node scrape.js --full # re-fetch ALL existing rows → upsert
+node scrape.js --discover # find NEW /pNNNN via search paging → upsert
+ # --discover-term wallpaper|fabric|...
+# flags: --limit N --sleep MS(500) --db <conn>
+
+node enrich.js # LOCAL $0 enrichment (qwen3 desc + qwen2.5vl color)
+ # --desc | --color | --limit N | --sleep MS
+```
+
+- Upsert key: `mfr_sku` (UNIQUE). `price_retail` is only written when
+ non-null, so an authed price is never nulled-out by an anonymous re-run.
+- `last_scraped` stamped via `now()` on every upsert.
+
+## Files
+
+- `parse.js` — HTML → normalized product object (entity-decode + field regex).
+- `scrape.js` — fetch + parse + upsert + enumerate/discover.
+- `enrich.js` — local Ollama enrichment (descriptions + dominant color hex).
diff --git a/DW-Programming/scrapers/designers-guild/auth-price.js b/DW-Programming/scrapers/designers-guild/auth-price.js
new file mode 100644
index 00000000..6cf78410
--- /dev/null
+++ b/DW-Programming/scrapers/designers-guild/auth-price.js
@@ -0,0 +1,299 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Designers Guild — AUTHENTICATED price pass (STAGING ONLY).
+ *
+ * Second-pass pricing run. The dedicated $0 anonymous scraper (scrape.js) fills
+ * every field EXCEPT price for wallpaper/fabric ROLLS — retail/trade pricing is
+ * gated behind the trade-portal login. Steve approved ONE careful login attempt.
+ *
+ * Mechanism (reverse-engineered from /bundles/js/login-page):
+ * POST https://www.designersguild.com/api/member/signIn
+ * headers: {accept: application/json, content-type: application/json}
+ * body: {email, password, countryId:238, languageId:2, accountType:0} (en-us)
+ * Success -> response body === true (or {redirect}), sets a session cookie.
+ * Failure -> {error:true, message:"InvalidLogin"|"InvalidCaptcha"|...}.
+ *
+ * HARD RULES enforced here:
+ * - AT MOST ONE login attempt. Any failure -> print reason + exit(2). No retry.
+ * - Writes ONLY price_retail / price_trade (+ updated_at) into
+ * designers_guild_catalog, matched by mfr_sku (fallback product_url).
+ * - Never touches shopify_products; never pushes to Shopify.
+ *
+ * Usage:
+ * node auth-price.js --login-check # login once, print marker, exit
+ * node auth-price.js --pilot [N] # login once + price N unpriced rows
+ * node auth-price.js --full # login once + price ALL unpriced rows
+ * flags: --limit N --sleep MS(1200) --db <conn> --dry (no DB writes)
+ */
+
+const https = require('https');
+const { Pool } = require('pg');
+
+const argv = process.argv.slice(2);
+const has = (f) => argv.includes(f);
+const val = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+
+const DB = val('--db', process.env.DATABASE_URL || 'postgresql://macstudio3@127.0.0.1:5432/dw_unified');
+const SLEEP = parseInt(val('--sleep', '1200'), 10);
+const DRY = has('--dry');
+const pool = new Pool({ connectionString: DB });
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+const HOST = 'www.designersguild.com';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+
+// ---- read creds from secrets-manager/.env (quoted values) ----
+function readSecret(key) {
+ const fs = require('fs');
+ const path = require('os').homedir() + '/Projects/secrets-manager/.env';
+ const txt = fs.readFileSync(path, 'utf8');
+ const m = txt.match(new RegExp('^' + key + '=(.*)$', 'm'));
+ if (!m) return null;
+ let v = m[1].trim();
+ if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
+ return v;
+}
+
+// ---- cookie jar (Set-Cookie -> Cookie) ----
+const jar = {};
+function storeCookies(setCookieArr) {
+ if (!setCookieArr) return;
+ for (const sc of setCookieArr) {
+ const kv = sc.split(';')[0];
+ const eq = kv.indexOf('=');
+ if (eq > 0) jar[kv.slice(0, eq).trim()] = kv.slice(eq + 1).trim();
+ }
+}
+function cookieHeader() { return Object.entries(jar).map(([k, v]) => `${k}=${v}`).join('; '); }
+
+function request(method, path, { json, headers } = {}) {
+ return new Promise((resolve, reject) => {
+ const body = json != null ? JSON.stringify(json) : null;
+ const h = {
+ 'User-Agent': UA,
+ 'Accept': json != null ? 'application/json' : 'text/html,application/xhtml+xml',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ ...(headers || {}),
+ };
+ if (body) { h['Content-Type'] = 'application/json'; h['Content-Length'] = Buffer.byteLength(body); }
+ const ck = cookieHeader();
+ if (ck) h['Cookie'] = ck;
+ const req = https.request({ hostname: HOST, path, method, headers: h }, res => {
+ storeCookies(res.headers['set-cookie']);
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+ const loc = res.headers.location;
+ const next = loc.startsWith('http') ? new URL(loc).pathname + new URL(loc).search : loc;
+ res.resume();
+ return request('GET', next, { headers }).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', c => (data += c));
+ res.on('end', () => resolve({ status: res.statusCode, body: data, headers: res.headers }));
+ });
+ req.on('error', reject);
+ req.setTimeout(30000, () => { req.destroy(); reject(new Error('timeout')); });
+ if (body) req.write(body);
+ req.end();
+ });
+}
+
+// ---- ONE login attempt ----
+let LOGGED_IN = false;
+async function loginOnce() {
+ const email = readSecret('DESIGNERSGUILD_PORTAL_USER');
+ const password = readSecret('DESIGNERSGUILD_PORTAL_PASS');
+ if (!email || !password) throw new Error('missing creds in secrets .env');
+
+ // Prime a session (get anti-forgery/session cookies) from the login page.
+ await request('GET', '/en-us/login/l102?t=1');
+
+ const res = await request('POST', '/api/member/signIn', {
+ json: { email, password, countryId: 238, languageId: 2, accountType: 0 },
+ });
+
+ let parsed = null;
+ try { parsed = JSON.parse(res.body); } catch (_) {}
+
+ // Success shapes: body === true, or {redirect:...}
+ if (res.status === 200 && (parsed === true || (parsed && parsed.redirect && !parsed.error))) {
+ LOGGED_IN = true;
+ return { ok: true, marker: `status200 body=${JSON.stringify(parsed)} cookies=[${Object.keys(jar).join(',')}]` };
+ }
+ // Failure shapes
+ const reason =
+ parsed && parsed.error ? (parsed.message || 'error')
+ : res.status !== 200 ? `HTTP ${res.status}`
+ : `unexpected body: ${String(res.body).slice(0, 160)}`;
+ return { ok: false, reason };
+}
+
+// ---- price parse from an AUTHENTICATED product page ----
+function money(s) {
+ if (s == null) return null;
+ if (typeof s === 'number') return s;
+ const m = String(s).replace(/,/g, '').match(/([\d.]+)/);
+ return m ? parseFloat(m[1]) : null;
+}
+/**
+ * Extract the MAIN product's retail + trade price from an authed page.
+ * The main product's order-form block carries rLRRP (retail) + rLTradePrice
+ * (trade) once authenticated; displayPrice/priceMin are numeric fallbacks.
+ * We anchor on the product's own orderFormName to avoid grabbing the
+ * related-accessory (cushion) price that also renders on roll pages.
+ */
+function parsePrices(html, orderFormName) {
+ const dec = html
+ .replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'")
+ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
+
+ const pick = (blob) => {
+ const rrp = blob.match(/"rLRRP":"?\$?([\d.,]+)"?/);
+ const trade = blob.match(/"rLTradePrice":"?\$?([\d.,]+)"?/);
+ const disp = blob.match(/"displayPrice":([\d.]+)/);
+ const sale = blob.match(/"displaySalePrice":([\d.]+)/);
+ const pmin = blob.match(/"priceMin":([\d.]+)/);
+ let retail = money(rrp && rrp[1]) || money(disp && disp[1]) || money(pmin && pmin[1]);
+ let tradeP = money(trade && trade[1]) || money(sale && sale[1]);
+ return { retail, tradeP };
+ };
+
+ // 1) Prefer a block anchored on the product's own name.
+ if (orderFormName) {
+ const esc = orderFormName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ const re = new RegExp('"orderFormName":"' + esc + '"[\\s\\S]{0,1500}');
+ const m = dec.match(re);
+ if (m) {
+ const r = pick(m[0]);
+ if (r.retail || r.tradeP) return r;
+ }
+ }
+ // 2) Fallback: top-level product summary price fields (priceMin/displayPrice
+ // at the page root become populated once authed).
+ return pick(dec);
+}
+
+// The anonymous parse already stored orderFormName as pattern_name (lowercased).
+// The authed page renders it in its original case; match case-insensitively.
+function ciAnchor(html, patternName) {
+ if (!patternName) return null;
+ const m = html.match(/"orderFormName":"([^"]+)"/gi) || [];
+ const want = patternName.toLowerCase().trim();
+ for (const raw of m) {
+ const name = raw.replace(/^"orderFormName":"/i, '').replace(/"$/, '');
+ if (name.toLowerCase().trim() === want || name.toLowerCase().startsWith(want)) return name;
+ }
+ return null;
+}
+
+async function priceOne(row) {
+ const res = await request('GET', new URL(row.product_url).pathname);
+ if (res.status !== 200) return { ...row, err: `HTTP ${res.status}` };
+ const anchor = ciAnchor(res.body, row.pattern_name) || row.pattern_name;
+ const { retail, tradeP } = parsePrices(res.body, anchor);
+ if (retail == null && tradeP == null) return { ...row, err: 'no-price' };
+ if (!DRY) {
+ await pool.query(
+ `UPDATE designers_guild_catalog
+ SET price_retail = COALESCE($1, price_retail),
+ price_trade = COALESCE($2, price_trade),
+ updated_at = now()
+ WHERE mfr_sku = $3`,
+ [retail, tradeP, row.mfr_sku]
+ );
+ }
+ return { ...row, retail, tradeP };
+}
+
+async function getUnpriced(limit) {
+ const q = `SELECT mfr_sku, pattern_name, product_url
+ FROM designers_guild_catalog
+ WHERE product_url IS NOT NULL AND product_url <> ''
+ AND (price_retail IS NULL OR price_retail = 0)
+ ORDER BY id
+ ${limit ? 'LIMIT ' + parseInt(limit, 10) : ''}`;
+ const { rows } = await pool.query(q);
+ return rows;
+}
+
+(async () => {
+ try {
+ console.log('=== Designers Guild AUTHED price pass (STAGING ONLY) ===');
+ const login = await loginOnce();
+ if (!login.ok) {
+ console.error(`DG login failed: ${login.reason}`);
+ await pool.end();
+ process.exit(2);
+ }
+ console.log(`LOGIN OK — ${login.marker}`);
+ if (has('--login-check')) { return; }
+
+ // One-time audit dump: fetch a known roll page authed and print its raw
+ // price block so we can confirm retail vs trade are distinct fields.
+ if (has('--audit')) {
+ const auditUrl = val('--audit-url', '/en-us/wallpaper/designers-guild/coquelicot-delft-wallpaper/p46484');
+ const a = await request('GET', auditUrl.startsWith('http') ? new URL(auditUrl).pathname : auditUrl);
+ const dec = a.body.replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&');
+ const idx = dec.indexOf('"orderFormName":"Coquelicot Delft"');
+ const blob = dec.slice(Math.max(0, idx - 100), idx + 900);
+ const keys = ['rLRRP', 'rLTradePrice', 'displayPrice', 'displaySalePrice', 'priceMin', 'priceMax', 'rRPMessage', 'priceMessage'];
+ console.log('--- AUDIT price block (main product anchor) ---');
+ for (const k of keys) { const m = blob.match(new RegExp('"' + k + '":([^,}]+)')); console.log(` ${k} = ${m ? m[1] : 'ABSENT'}`); }
+ console.log('--- AUDIT all distinct values page-wide ---');
+ for (const k of keys) {
+ const vals = [...new Set((dec.match(new RegExp('"' + k + '":"?\\$?([\\d.,]+)"?', 'g')) || []))];
+ console.log(` ${k}: ${vals.slice(0, 8).join(' | ') || 'none'}`);
+ }
+ // Show what my parser actually extracts for this page
+ const anchor = ciAnchor(a.body, 'coquelicot delft') || 'Coquelicot Delft';
+ console.log('--- parser output for p46484 ---', JSON.stringify(parsePrices(a.body, anchor)));
+
+ // Deep probe: find the ROLL product's OWN order-form block + its SKU code,
+ // and enumerate every price-ish key inside it. Also list all /api/ calls
+ // referenced on the authed page (a per-variant price AJAX would show here).
+ console.log('--- ROLL own block probe ---');
+ const codeIdx = dec.indexOf('"code":"PDG');
+ if (codeIdx >= 0) {
+ const blk = dec.slice(Math.max(0, codeIdx - 600), codeIdx + 600);
+ const kv = [...blk.matchAll(/"([a-zA-Z]*[Pp]rice[a-zA-Z]*|rL[A-Za-z]+|rRP[A-Za-z]*)":([^,}]+)/g)];
+ console.log(' code ctx price keys:', kv.length ? kv.map(x => x[1] + '=' + x[2]).join(' | ') : 'NONE');
+ console.log(' raw code block snippet:', blk.slice(400, 800).replace(/\s+/g, ' '));
+ } else console.log(' no "code":"PDG..." found in authed page');
+ console.log('--- /api/ endpoints on authed page ---');
+ console.log(' ' + [...new Set((a.body.match(/\/api\/[A-Za-z0-9/_-]+/g) || []))].join('\n '));
+ // Look for the roll's per-roll price near its orderFormName (Coquelicot Delft appears; find blocks WITH price)
+ console.log('--- blocks with BOTH orderFormName + a price ---');
+ let cnt = 0;
+ const re2 = /"orderFormName":"([^"]+)"[\s\S]{0,600}?("rLTradePrice"|"displayPrice")/g;
+ let mm;
+ while ((mm = re2.exec(dec)) && cnt < 10) { console.log(` "${mm[1]}" has ${mm[2]}`); cnt++; }
+ return;
+ }
+
+ const n = has('--pilot') ? (parseInt(val('--pilot', '15'), 10) || 15) : null;
+ let rows = await getUnpriced(has('--full') ? null : n);
+ const lim = val('--limit', null);
+ if (lim) rows = rows.slice(0, parseInt(lim, 10));
+ console.log(`Pricing ${rows.length} unpriced rows${DRY ? ' (DRY, no writes)' : ''}...\n`);
+
+ let priced = 0, noPrice = 0, err = 0;
+ const samples = [];
+ for (let i = 0; i < rows.length; i++) {
+ try {
+ const r = await priceOne(rows[i]);
+ if (r.err === 'no-price') { noPrice++; }
+ else if (r.err) { err++; if (err <= 8) console.log(` ERR ${r.err} ${r.mfr_sku}`); }
+ else { priced++; if (samples.length < 12) samples.push(`${r.mfr_sku}: retail=${r.retail} trade=${r.tradeP}`); }
+ } catch (e) { err++; if (err <= 8) console.log(` EXC ${e.message} ${rows[i].mfr_sku}`); }
+ if ((i + 1) % 50 === 0) console.log(` [${i + 1}/${rows.length}] priced:${priced} noPrice:${noPrice} err:${err}`);
+ await sleep(SLEEP);
+ }
+ console.log(`\n=== DONE — priced:${priced} noPrice:${noPrice} err:${err} of ${rows.length} ===`);
+ if (samples.length) console.log('samples:\n ' + samples.join('\n '));
+ } catch (e) {
+ console.error('Fatal:', e.message);
+ process.exit(1);
+ } finally {
+ await pool.end();
+ }
+})();
diff --git a/DW-Programming/scrapers/designers-guild/enrich.js b/DW-Programming/scrapers/designers-guild/enrich.js
new file mode 100644
index 00000000..e56b09bd
--- /dev/null
+++ b/DW-Programming/scrapers/designers-guild/enrich.js
@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Designers Guild — LOCAL enrichment ($0, Ollama). STAGING ONLY.
+ * - ai_description : qwen3:14b (from pattern/color/collection/specs)
+ * - dominant_color : qwen2.5vl:7b vision on the product image
+ *
+ * Usage: node enrich.js [--limit N] [--desc] [--color] [--sleep MS]
+ * default runs BOTH desc + color for rows missing them.
+ */
+
+const https = require('https');
+const { Pool } = require('pg');
+
+const argv = process.argv.slice(2);
+const has = (f) => argv.includes(f);
+const val = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+const OLLAMA = process.env.OLLAMA_URL || 'http://localhost:11434';
+const DB = val('--db', process.env.DATABASE_URL || 'postgresql://macstudio3@127.0.0.1:5432/dw_unified');
+const LIMIT = parseInt(val('--limit', '1000000'), 10);
+const SLEEP = parseInt(val('--sleep', '150'), 10);
+const doDesc = has('--desc') || (!has('--desc') && !has('--color'));
+const doColor = has('--color') || (!has('--desc') && !has('--color'));
+const pool = new Pool({ connectionString: DB });
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+function ollama(path, payload) {
+ return new Promise((resolve, reject) => {
+ const body = JSON.stringify(payload);
+ const u = new URL(OLLAMA + path);
+ const req = require('http').request({
+ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
+ headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
+ }, res => { let d = ''; res.on('data', c => (d += c)); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } }); });
+ req.on('error', reject);
+ req.setTimeout(120000, () => { req.destroy(); reject(new Error('ollama timeout')); });
+ req.write(body); req.end();
+ });
+}
+
+function fetchImage(url) {
+ return new Promise((resolve, reject) => {
+ https.get(url, res => {
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('img HTTP ' + res.statusCode)); }
+ const chunks = [];
+ res.on('data', c => chunks.push(c));
+ res.on('end', () => resolve(Buffer.concat(chunks).toString('base64')));
+ }).on('error', reject);
+ });
+}
+
+async function genDescription(p) {
+ const prompt = `Write a concise 2-sentence luxury product description for this Designers Guild ${p.product_type}. ` +
+ `Pattern: ${p.pattern_name || 'n/a'}. Color: ${p.color_name || 'n/a'}. Collection: ${p.collection || 'n/a'}. ` +
+ `Material: ${p.material || 'n/a'}. Design/motif: ${p.design || 'n/a'}. ` +
+ `Do not invent prices, dimensions, or specs. British luxury tone. Output only the description text.`;
+ const r = await ollama('/api/generate', { model: 'qwen3:14b', prompt, stream: false, options: { temperature: 0.6 } });
+ let t = (r.response || '').replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+ t = t.replace(/^["']|["']$/g, '').trim();
+ return t.slice(0, 800) || null;
+}
+
+async function genColor(imageUrl) {
+ const b64 = await fetchImage(imageUrl);
+ const r = await ollama('/api/generate', {
+ model: 'qwen2.5vl:7b',
+ prompt: 'Look at this wallcovering/fabric swatch. Reply with ONLY the single dominant color as a hex code like #A1B2C3. No other text.',
+ images: [b64], stream: false, options: { temperature: 0 },
+ });
+ const m = (r.response || '').match(/#[0-9a-fA-F]{6}/);
+ return m ? m[0].toUpperCase() : null;
+}
+
+(async () => {
+ let descN = 0, colorN = 0, fail = 0;
+ try {
+ if (doDesc) {
+ const { rows } = await pool.query(
+ `SELECT id, pattern_name, color_name, collection, product_type, material, design
+ FROM designers_guild_catalog
+ WHERE (ai_description IS NULL OR ai_description='') AND pattern_name IS NOT NULL
+ ORDER BY id LIMIT $1`, [LIMIT]);
+ console.log(`[desc] ${rows.length} rows`);
+ for (let i = 0; i < rows.length; i++) {
+ try {
+ const d = await genDescription(rows[i]);
+ if (d) { await pool.query('UPDATE designers_guild_catalog SET ai_description=$1, updated_at=now() WHERE id=$2', [d, rows[i].id]); descN++; }
+ } catch (e) { fail++; if (fail <= 5) console.log(' desc err', e.message); }
+ if ((i + 1) % 25 === 0) console.log(` [desc ${i + 1}/${rows.length}] done:${descN}`);
+ await sleep(SLEEP);
+ }
+ }
+ if (doColor) {
+ const { rows } = await pool.query(
+ `SELECT id, image_url FROM designers_guild_catalog
+ WHERE (dominant_color_hex IS NULL OR dominant_color_hex='') AND image_url IS NOT NULL
+ ORDER BY id LIMIT $1`, [LIMIT]);
+ console.log(`[color] ${rows.length} rows`);
+ for (let i = 0; i < rows.length; i++) {
+ try {
+ const hex = await genColor(rows[i].image_url);
+ if (hex) { await pool.query('UPDATE designers_guild_catalog SET dominant_color_hex=$1, updated_at=now() WHERE id=$2', [hex, rows[i].id]); colorN++; }
+ } catch (e) { fail++; if (fail <= 5) console.log(' color err', e.message); }
+ if ((i + 1) % 25 === 0) console.log(` [color ${i + 1}/${rows.length}] done:${colorN}`);
+ await sleep(SLEEP);
+ }
+ }
+ console.log(`\n=== enrich DONE — desc:${descN} color:${colorN} fail:${fail} ($0 local) ===`);
+ } catch (e) { console.error('Fatal:', e); process.exit(1); }
+ finally { await pool.end(); }
+})();
diff --git a/DW-Programming/scrapers/designers-guild/parse.js b/DW-Programming/scrapers/designers-guild/parse.js
new file mode 100644
index 00000000..8f61bff7
--- /dev/null
+++ b/DW-Programming/scrapers/designers-guild/parse.js
@@ -0,0 +1,163 @@
+'use strict';
+/**
+ * Designers Guild — product-page parser (feed-first, $0, anonymous).
+ *
+ * designersguild.com is an ASP.NET MVC + Knockout.js commerce site (Cloudflare
+ * front). Every product page (/en-us/.../pNNNN) server-renders a rich, HTML-
+ * entity-encoded product model inside the page — including a `TechInfo` block
+ * (width / weight / repeats / composition / patternMatch / rollLength /
+ * additionalInfo), an `attributes` array (Brand / Pattern / Type), the parent
+ * `collection`, GTIN, product code, and gallery image ids. NO Browserbase,
+ * proxy, or captcha needed — plain https GET returns HTTP 200 with x-bot:false.
+ *
+ * PRICE is the ONE field NOT present anonymously: priceMin/priceMax/displayPrice
+ * are null on every region (en-us AND en-gb) because retail pricing is gated
+ * behind the trade login and fetched by a post-auth AJAX call. See README.md.
+ */
+
+function unescapeHtml(s) {
+ if (!s) return s;
+ return s
+ .replace(/"/g, '"')
+ .replace(/'/g, "'")
+ .replace(/'/g, "'")
+ .replace(/&/g, '&')
+ .replace(/</g, '<')
+ .replace(/>/g, '>')
+ .replace(/ /g, ' ');
+}
+
+// Pull the first "key":"value" (string) match out of a decoded blob.
+function strField(dec, key) {
+ const m = dec.match(new RegExp('"' + key + '"\\s*:\\s*"([^"]*)"'));
+ return m ? m[1].trim() : null;
+}
+
+// Pull the first "key":<number|null> match.
+function numOrNull(dec, key) {
+ const m = dec.match(new RegExp('"' + key + '"\\s*:\\s*([0-9.]+|null)'));
+ if (!m || m[1] === 'null') return null;
+ return parseFloat(m[1]);
+}
+
+function parseInches(val) {
+ if (!val) return null;
+ const inMatch = String(val).match(/([\d.]+)\s*(?:in|inch|")/i);
+ if (inMatch) return parseFloat(inMatch[1]);
+ const cmMatch = String(val).match(/([\d.]+)\s*cm/i);
+ if (cmMatch) return +(parseFloat(cmMatch[1]) / 2.54).toFixed(2);
+ const numMatch = String(val).match(/([\d.]+)/);
+ return numMatch ? parseFloat(numMatch[1]) : null;
+}
+
+/**
+ * Parse a Designers Guild product page HTML into a normalized product object.
+ * Returns null if it doesn't look like a product page (no code/orderFormName).
+ */
+function parseProduct(html, productUrl) {
+ const dec = unescapeHtml(html);
+
+ const code = strField(dec, 'code');
+ const orderFormName = strField(dec, 'orderFormName');
+ if (!code && !orderFormName) return null;
+
+ // --- TechInfo (specs) ---
+ const width = strField(dec, 'width');
+ const weight = strField(dec, 'weight');
+ const hRepeat = strField(dec, 'horizontalPatternRepeat');
+ const vRepeat = strField(dec, 'verticalPatternRepeat');
+ const composition = strField(dec, 'composition');
+ const patternMatch = strField(dec, 'patternMatch');
+ const rollLength = strField(dec, 'rollLength');
+ const additionalInfo = strField(dec, 'additionalInfo');
+ const aftercare = strField(dec, 'aftercare');
+
+ // --- brand / collection ---
+ const brand = strField(dec, 'brand');
+ let collection = null;
+ const collM = dec.match(/"collection":\{"collectionId":\d+,"heading":"([^"]+)"/);
+ if (collM) collection = collM[1];
+ if (!collection) {
+ const cn = dec.match(/"collections":\[\{"collectionId":\d+,"name":"([^"]+)"/);
+ if (cn) collection = cn[1];
+ }
+
+ // --- attributes: Pattern / Type / Design ---
+ let pattern = null, typeAttr = null, design = null;
+ const attrsM = dec.match(/"attributes":(\[[^\]]*\])/);
+ if (attrsM) {
+ try {
+ const attrs = JSON.parse(attrsM[1]);
+ for (const a of attrs) {
+ const g = (a.attributeGroup || '').toLowerCase();
+ if (g === 'pattern') pattern = a.attribute;
+ else if (g === 'type') typeAttr = a.attribute;
+ else if (g === 'design') design = a.attribute;
+ }
+ } catch (_) {}
+ }
+
+ // --- product type (wallcovering vs fabric) from URL ---
+ let productType = 'wallcovering';
+ if (/\/fabric\//i.test(productUrl || '')) productType = 'fabric';
+ else if (/\/wallpaper\//i.test(productUrl || '')) productType = 'wallcovering';
+
+ // --- pattern + color name (orderFormName is "Pattern - Color") ---
+ let patternName = orderFormName ? orderFormName.trim() : null;
+ let colorName = null;
+ if (orderFormName && orderFormName.includes(' - ')) {
+ const parts = orderFormName.split(' - ');
+ colorName = parts[parts.length - 1].trim();
+ }
+ if (patternName) patternName = patternName.toLowerCase();
+ if (colorName) colorName = colorName.toLowerCase();
+
+ // --- images (imageId list; primary from first) ---
+ const imageIds = [...new Set((dec.match(/"imageId":(\d+)/g) || []).map(s => s.replace(/\D/g, '')))];
+ const imgBase = 'https://www.designersguild.com/image-nonwebp/1024/';
+ const allImages = imageIds.map(id => imgBase + id);
+ const imageUrl = imageIds.length ? 'https://www.designersguild.com/image-nonwebp/600/' + imageIds[0] : null;
+
+ // --- room-setting / lifestyle images ---
+ const roomImages = [];
+ const roomRe = /image-nonwebp\/\d+\/(\d+)[^"']*(?:room|lifestyle|interior|setting)/gi;
+ let rm;
+ while ((rm = roomRe.exec(dec))) roomImages.push(imgBase + rm[1]);
+
+ const gtin = strField(dec, 'gtin13');
+
+ // --- PRICE (gated; null anonymously) ---
+ const priceRetail = numOrNull(dec, 'priceMin') || numOrNull(dec, 'displayPrice');
+
+ return {
+ mfr_sku: code || null,
+ pattern_name: patternName,
+ color_name: colorName,
+ collection: collection,
+ product_type: productType,
+ width: width,
+ width_inches: parseInches(width),
+ length: rollLength,
+ roll_length: rollLength,
+ repeat_v: vRepeat,
+ repeat_h: hRepeat,
+ material: composition,
+ match_type: patternMatch,
+ weight: weight,
+ care_instructions: aftercare || additionalInfo,
+ design: design || pattern,
+ features: additionalInfo,
+ price_retail: priceRetail, // null anonymously — filled only via authed run
+ image_url: imageUrl,
+ all_images: allImages.length ? JSON.stringify(allImages) : null,
+ room_setting_urls: roomImages.length ? JSON.stringify([...new Set(roomImages)]) : null,
+ product_url: productUrl,
+ // extras retained for reference
+ _brand: brand,
+ _gtin: gtin,
+ _pattern_attr: pattern,
+ _type_attr: typeAttr,
+ };
+}
+
+module.exports = { parseProduct, unescapeHtml, parseInches };
diff --git a/DW-Programming/scrapers/designers-guild/scrape.js b/DW-Programming/scrapers/designers-guild/scrape.js
new file mode 100644
index 00000000..15caa645
--- /dev/null
+++ b/DW-Programming/scrapers/designers-guild/scrape.js
@@ -0,0 +1,175 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Designers Guild — dedicated scraper (STAGING ONLY → designers_guild_catalog).
+ *
+ * Per Steve's "every vendor gets its own dedicated scraper" rule, this is the
+ * DG-scoped module. Feed-first, $0, anonymous (no login, no Browserbase).
+ *
+ * Modes:
+ * node scrape.js --pilot [N] re-fetch N (default 20) existing rows, upsert
+ * node scrape.js --full re-fetch ALL existing rows, upsert
+ * node scrape.js --discover enumerate new product URLs via search paging
+ * (adds any /pNNNN not already in the table)
+ * node scrape.js --url <url> scrape a single product URL (debug)
+ *
+ * Flags: --limit N --sleep MS(default 1000) --db <conn>
+ *
+ * PRICE: null anonymously (trade-login gated). This scraper captures every other
+ * field; a separate authed pass (Browserbase) is required to fill price_retail.
+ */
+
+const https = require('https');
+const { Pool } = require('pg');
+const { parseProduct } = require('./parse');
+
+const argv = process.argv.slice(2);
+const has = (f) => argv.includes(f);
+const val = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+
+const DB = val('--db', process.env.DATABASE_URL || 'postgresql://macstudio3@127.0.0.1:5432/dw_unified');
+const SLEEP = parseInt(val('--sleep', '1000'), 10);
+const pool = new Pool({ connectionString: DB });
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+function fetchPage(url, redirects = 0) {
+ return new Promise((resolve, reject) => {
+ if (redirects > 5) return reject(new Error('too many redirects'));
+ const u = new URL(url);
+ const req = https.get({
+ hostname: u.hostname,
+ path: u.pathname + u.search,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36',
+ 'Accept': 'text/html,application/xhtml+xml',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ },
+ }, res => {
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+ const next = res.headers.location.startsWith('http') ? res.headers.location : u.origin + res.headers.location;
+ res.resume();
+ return fetchPage(next, redirects + 1).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', c => (data += c));
+ res.on('end', () => resolve({ status: res.statusCode, body: data }));
+ });
+ req.on('error', reject);
+ req.setTimeout(25000, () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+
+// Columns we write. mfr_sku is the UNIQUE upsert key.
+const UPSERT_COLS = [
+ 'mfr_sku', 'pattern_name', 'color_name', 'collection', 'product_type',
+ 'width', 'width_inches', 'length', 'roll_length', 'repeat_v', 'repeat_h',
+ 'material', 'match_type', 'weight', 'care_instructions', 'design', 'features',
+ 'image_url', 'all_images', 'room_setting_urls', 'product_url',
+];
+
+async function upsert(p) {
+ if (!p || !p.mfr_sku) return { skipped: true };
+ const cols = UPSERT_COLS.slice();
+ const vals = cols.map(c => (p[c] === undefined ? null : p[c]));
+ // price_retail only written when non-null (never null-out an authed price)
+ if (p.price_retail != null) { cols.push('price_retail'); vals.push(p.price_retail); }
+ cols.push('last_scraped'); // set via now(), no bind param
+ // Build placeholders: real columns get $N binds, last_scraped uses now().
+ const ph = [];
+ let pi = 1;
+ for (const c of cols) { if (c === 'last_scraped') ph.push('now()'); else ph.push('$' + pi++); }
+ const updates = cols
+ .filter(c => c !== 'mfr_sku')
+ .map(c => (c === 'last_scraped' ? 'last_scraped = now()' : `${c} = EXCLUDED.${c}`))
+ .join(', ');
+ const sql = `INSERT INTO designers_guild_catalog (${cols.join(', ')}) VALUES (${ph.join(', ')})
+ ON CONFLICT (mfr_sku) DO UPDATE SET ${updates}, updated_at = now()`;
+ await pool.query(sql, vals);
+ return { ok: true, hadPrice: p.price_retail != null };
+}
+
+async function scrapeUrl(url) {
+ const { status, body } = await fetchPage(url);
+ if (status !== 200) return { url, status, error: `HTTP ${status}` };
+ const p = parseProduct(body, url);
+ if (!p) return { url, status, error: 'no-product' };
+ const r = await upsert(p);
+ return { url, status, mfr_sku: p.mfr_sku, hadPrice: !!r.hadPrice, imaged: !!p.image_url };
+}
+
+async function getExistingUrls(limit) {
+ const q = `SELECT product_url FROM designers_guild_catalog
+ WHERE product_url IS NOT NULL AND product_url <> '' ORDER BY id
+ ${limit ? 'LIMIT ' + parseInt(limit, 10) : ''}`;
+ const { rows } = await pool.query(q);
+ return rows.map(r => r.product_url);
+}
+
+// --discover: enumerate product URLs from the anonymous server-rendered search
+async function discoverUrls(term = 'wallpaper', maxPages = 60) {
+ const found = new Set();
+ for (let page = 1; page <= maxPages; page++) {
+ const u = `https://www.designersguild.com/en-us/search-results/l76?search-term=${encodeURIComponent(term)}&page=${page}`;
+ let body;
+ try { ({ body } = await fetchPage(u)); } catch (e) { break; }
+ const links = (body.match(/\/en-us\/(?:wallpaper|fabric)\/[a-z0-9/-]+\/p[0-9]+/gi) || []);
+ if (!links.length) break;
+ let added = 0;
+ for (const l of links) { const full = 'https://www.designersguild.com' + l; if (!found.has(full)) { found.add(full); added++; } }
+ process.stdout.write(` [discover ${term}] page ${page}: +${added} (total ${found.size})\n`);
+ if (added === 0 && page > 2) break; // pagination exhausted (loops back)
+ await sleep(SLEEP);
+ }
+ return [...found];
+}
+
+async function run(urls, label) {
+ const limit = val('--limit', null);
+ if (limit) urls = urls.slice(0, parseInt(limit, 10));
+ console.log(`\n=== Designers Guild scrape [${label}] — ${urls.length} urls ===\n`);
+ let ok = 0, fail = 0, priced = 0, imaged = 0;
+ for (let i = 0; i < urls.length; i++) {
+ try {
+ const r = await scrapeUrl(urls[i]);
+ if (r.error) { fail++; if (fail <= 8) console.log(` FAIL ${r.error} ${urls[i]}`); }
+ else { ok++; if (r.hadPrice) priced++; if (r.imaged) imaged++; }
+ } catch (e) { fail++; if (fail <= 8) console.log(` ERROR ${e.message} ${urls[i]}`); }
+ if ((i + 1) % 25 === 0) console.log(` [${i + 1}/${urls.length}] ok:${ok} fail:${fail} priced:${priced} imaged:${imaged}`);
+ await sleep(SLEEP);
+ }
+ console.log(`\n=== DONE [${label}] — ok:${ok} fail:${fail} priced:${priced}/${ok} imaged:${imaged}/${ok} ===`);
+ return { ok, fail, priced, imaged };
+}
+
+(async () => {
+ try {
+ if (has('--url')) {
+ const r = await scrapeUrl(val('--url'));
+ console.log(JSON.stringify(r, null, 2));
+ } else if (has('--pilot')) {
+ const n = parseInt(val('--pilot', '20'), 10) || 20;
+ const urls = await getExistingUrls(n);
+ await run(urls, `pilot ${n}`);
+ } else if (has('--full')) {
+ const urls = await getExistingUrls(null);
+ await run(urls, 'full');
+ } else if (has('--discover')) {
+ const term = val('--discover-term', 'wallpaper');
+ const discovered = await discoverUrls(term);
+ // filter to NEW ones
+ const { rows } = await pool.query('SELECT product_url FROM designers_guild_catalog');
+ const known = new Set(rows.map(r => r.product_url));
+ const fresh = discovered.filter(u => !known.has(u));
+ console.log(`\ndiscovered ${discovered.length}, new ${fresh.length}`);
+ await run(fresh, `discover ${term}`);
+ } else {
+ console.log('Usage: node scrape.js --pilot [N] | --full | --discover [--discover-term wallpaper] | --url <url>');
+ }
+ } catch (e) {
+ console.error('Fatal:', e);
+ process.exit(1);
+ } finally {
+ await pool.end();
+ }
+})();
diff --git a/_dw-login-record.mjs b/_dw-login-record.mjs
new file mode 100644
index 00000000..908df2c5
--- /dev/null
+++ b/_dw-login-record.mjs
@@ -0,0 +1,86 @@
+import { chromium } from 'playwright';
+import fs from 'fs';
+
+const EXE = '/Users/macstudio3/Library/Caches/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-mac-arm64/chrome-headless-shell';
+const EMAIL = process.env.LOGIN_EMAIL || 'steve@designerwallcoverings.com';
+const OTP_FILE = '/tmp/dw-otp.txt';
+const LOG = m => { console.log(new Date().toISOString().slice(11,19), m); };
+try { fs.unlinkSync(OTP_FILE); } catch {}
+
+const browser = await chromium.launch({ headless: true, executablePath: EXE });
+const ctx = await browser.newContext({
+ viewport: { width: 1280, height: 800 },
+ recordVideo: { dir: '/tmp/dw-login-video', size: { width: 1280, height: 800 } },
+ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36'
+});
+const page = await ctx.newPage();
+const out = { email: EMAIL, steps: [] };
+const step = (n, extra={}) => { out.steps.push({ n, url: page.url(), ...extra }); LOG(n + ' | ' + page.url().slice(0,80)); };
+
+try {
+ await page.goto('https://designerwallcoverings.com', { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(2500); step('home');
+ const trig = await page.$('[data-dw-signin], a[href$="#dw-signin"]');
+ if (trig) { await trig.click().catch(()=>{}); }
+ await page.waitForTimeout(1000);
+ await page.screenshot({ path: '/tmp/dw-login-01-modal.png' }).catch(()=>{});
+ const loginBtn = await page.$('[data-dwsm-login]');
+ const loginUrl = loginBtn ? await loginBtn.getAttribute('data-dwsm-login') : 'https://designerwallcoverings.com/account';
+ await page.goto(loginUrl.startsWith('http') ? loginUrl : 'https://designerwallcoverings.com'+loginUrl, { waitUntil: 'domcontentloaded', timeout: 45000 }).catch(()=>{});
+ await page.waitForTimeout(4000); step('native-login');
+ await page.screenshot({ path: '/tmp/dw-login-02-emailpage.png' }).catch(()=>{});
+
+ const emailInput = await page.$('input[type="email"], input[name="account[email]"], input[autocomplete="email"]');
+ if (!emailInput) throw new Error('no email input found on login page');
+ await emailInput.fill(EMAIL);
+ await page.waitForTimeout(400);
+ await emailInput.press('Enter').catch(()=>{});
+ await page.waitForTimeout(4000); step('after-email');
+ const useEmail = await page.$('button:has-text("Use email instead"), a:has-text("Use email instead"), :text("Use email instead")');
+ if (useEmail) { await useEmail.click({ timeout: 3000 }).catch(()=>{}); out.usedEmailInstead = true; await page.waitForTimeout(4000); }
+ const sub = await page.$('button[type="submit"], button:has-text("Continue"), button:has-text("Submit")');
+ if (sub) await sub.click({ timeout: 3000 }).catch(()=>{});
+ await page.waitForTimeout(5000); step('code-requested');
+ await page.screenshot({ path: '/tmp/dw-login-03-codepage.png' }).catch(()=>{});
+
+ let code = null;
+ for (let i = 0; i < 60; i++) {
+ if (fs.existsSync(OTP_FILE)) {
+ const v = fs.readFileSync(OTP_FILE, 'utf8').trim();
+ if (/^\d{6}$/.test(v)) { code = v; break; }
+ }
+ await page.waitForTimeout(3000);
+ }
+ out.gotCode = !!code;
+ if (!code) { step('NO-CODE-TIMEOUT'); throw new Error('OTP not delivered to poll file in time'); }
+ LOG('got code ' + code);
+
+ const digitInputs = await page.$$('input[inputmode="numeric"], input[autocomplete="one-time-code"], input[maxlength="1"]');
+ if (digitInputs.length >= 6) {
+ for (let i = 0; i < 6; i++) { await digitInputs[i].fill(code[i]); await page.waitForTimeout(120); }
+ } else {
+ const one = await page.$('input[name="code"], input[inputmode="numeric"], input[type="text"], input[type="tel"]');
+ if (one) { await one.fill(code); } else { await page.keyboard.type(code, { delay: 120 }); }
+ }
+ // Shopify OTP auto-submits on the 6th digit; wait for the URL to leave /code.
+ try { await page.waitForURL(u => !/\/code(\?|$)/.test(String(u)), { timeout: 25000 }); } catch {}
+ await page.waitForTimeout(6000);
+ try { await page.waitForLoadState('networkidle', { timeout: 12000 }); } catch {}
+ await page.waitForTimeout(3000);
+ step('LANDING');
+ out.landingUrl = page.url();
+ out.landingTitle = await page.title().catch(()=> '');
+ out.landingText = ((await page.evaluate(()=> document.body?document.body.innerText:'').catch(()=> ''))||'').replace(/\s+/g,' ').slice(0,500);
+ await page.screenshot({ path: '/tmp/dw-login-04-landing.png', fullPage: false }).catch(()=>{});
+ out.ok = true;
+} catch (e) {
+ out.error = e.message;
+ await page.screenshot({ path: '/tmp/dw-login-ERR.png' }).catch(()=>{});
+} finally {
+ const vid = page.video();
+ await ctx.close();
+ try { out.videoPath = vid ? await vid.path() : null; } catch {}
+ await browser.close();
+ fs.writeFileSync('/tmp/dw-login-result.json', JSON.stringify(out, null, 2));
+ console.log('RESULT ' + JSON.stringify({ ok: out.ok, landingUrl: out.landingUrl, landingTitle: out.landingTitle, error: out.error }));
+}
← f8c34683 auto-save: 2026-07-10T11:07:57 (7 files) — pending-approval/
·
back to Designer Wallcoverings
·
auto-save: 2026-07-10T12:08:12 (6 files) — DW-Programming/sc 45d52c93 →