← back to Designer Wallcoverings
Jeffrey Stevens reconcile: openclaw-driven authed yorkwall.com per-SKU scraper (current/disco + MAP + specs -> york_catalog)
4d1ec9780b99932652301020c56bfd941189ffbb · 2026-06-25 18:10:53 -0700 · Steve Abrams
Files touched
A scripts/jeffrey-stevens-reconcile/scrape-js-via-openclaw.mjs
Diff
commit 4d1ec9780b99932652301020c56bfd941189ffbb
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 18:10:53 2026 -0700
Jeffrey Stevens reconcile: openclaw-driven authed yorkwall.com per-SKU scraper (current/disco + MAP + specs -> york_catalog)
---
.../scrape-js-via-openclaw.mjs | 208 +++++++++++++++++++++
1 file changed, 208 insertions(+)
diff --git a/scripts/jeffrey-stevens-reconcile/scrape-js-via-openclaw.mjs b/scripts/jeffrey-stevens-reconcile/scrape-js-via-openclaw.mjs
new file mode 100644
index 00000000..62904a29
--- /dev/null
+++ b/scripts/jeffrey-stevens-reconcile/scrape-js-via-openclaw.mjs
@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+/**
+ * Jeffrey Stevens (York/Brewster trade) LIVE portal scrape — DTD verdict C, step 3.
+ * Owner: vp-dw-commerce.
+ *
+ * WHY THIS EXISTS (and the staged runbook does not work):
+ * The lib/scrapers/york-*.ts scrapers hit the PUBLIC yorkwallcoverings.com
+ * nopCommerce storefront — they are collection/grid scrapers with NO argv
+ * parsing and NO per-SKU authenticated lookup. The Jeffrey Stevens reconcile
+ * needs ground-truth CURRENT/DISCONTINUED + current MAP per MFR SKU, which only
+ * the AUTHENTICATED yorkwall.com trade portal exposes, via
+ * /Product/Search?sku=<MFR_SKU> ("Inquiry - <SKU>" page).
+ *
+ * yorkwall.com bounces headless/Playwright logins (automation detection), so we
+ * drive the lookups THROUGH the already-authenticated openclaw REAL-Chrome
+ * session (customer 6003784 "STEVE ABRAMS DESIGNS"). We NEVER harvest the
+ * session cookie — we just navigate + read the rendered DOM via openclaw.
+ *
+ * WRITES: dw_unified.york_catalog (reversible data refresh). NO Shopify writes.
+ * COST: $0 (local + openclaw real Chrome; no paid API).
+ *
+ * Usage:
+ * node scrape-js-via-openclaw.mjs # full 91xxx cohort (491 SKUs)
+ * node scrape-js-via-openclaw.mjs --limit 5 # smoke / partial
+ * node scrape-js-via-openclaw.mjs --skus 4081-26348,2988-70202
+ */
+import { execFileSync } from 'node:child_process';
+import { writeFileSync, appendFileSync } from 'node:fs';
+import { Client } from 'pg';
+
+const ARGS = process.argv.slice(2);
+const argVal = (f) => { const i = ARGS.indexOf(f); return i >= 0 ? ARGS[i + 1] : null; };
+const LIMIT = argVal('--limit') ? parseInt(argVal('--limit'), 10) : null;
+const EXPLICIT = argVal('--skus') ? argVal('--skus').split(',').map(s => s.trim()).filter(Boolean) : null;
+const DELAY_MS = argVal('--delay') ? parseInt(argVal('--delay'), 10) : 1500;
+const AUDIT = '/Users/stevestudio2/Projects/Designer-Wallcoverings/scripts/jeffrey-stevens-reconcile/js-scrape-audit.jsonl';
+
+// Connect via the unix socket (/tmp) so trust auth applies — matches working `psql -d dw_unified`.
+// TCP (127.0.0.1) demands a SCRAM password the local dw_admin creds no longer satisfy.
+const PG = { host: '/tmp', port: 5432, database: 'dw_unified', user: process.env.USER };
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+function oc(args, timeout = 45000) {
+ return execFileSync('openclaw', args, { encoding: 'utf8', timeout, maxBuffer: 8 * 1024 * 1024 });
+}
+
+// Retry wrapper — openclaw's gateway occasionally times out transiently.
+async function ocRetry(args, { tries = 3, timeout = 45000 } = {}) {
+ let last;
+ for (let t = 0; t < tries; t++) {
+ try { return oc(args, timeout); }
+ catch (e) { last = e; await sleep(1200 * (t + 1)); }
+ }
+ throw last;
+}
+
+// Navigate the authed tab to the SKU inquiry page and return structured fields.
+async function lookup(sku) {
+ await ocRetry(['browser', 'navigate', `https://www.yorkwall.com/Product/Search?sku=${encodeURIComponent(sku)}`, '--timeout', '30000'], { tries: 3, timeout: 45000 });
+ await sleep(DELAY_MS);
+ // The extractor runs in-page; returns JSON. openclaw prints the JSON value.
+ const fn = `() => {
+ const T = document.body.innerText;
+ const norm = s => (s||"").replace(/\\s+/g," ").trim();
+ const g = re => { const m = T.match(re); return m ? norm(m[1]) : null; };
+ const found = /PRODUCT DETAILS/.test(T);
+ const imgs = Array.from(document.querySelectorAll("img.img-thumbnail"))
+ .map(i => i.src).filter(s => s && /bhf-cdn|azureedge|product/i.test(s));
+ return {
+ found,
+ authed: /Hello, STEVE ABRAMS/.test(T),
+ sku: g(/SKU #:\\s*([0-9A-Za-z\\-]+)/),
+ title: g(/PRODUCT DETAILS\\s+([^\\n]+?)\\s+SKU #:/s),
+ msrp: g(/MSRP:\\s*\\$?([0-9.,]+)/),
+ uom: g(/MSRP:\\s*\\$?[0-9.,]+\\s+([A-Za-z ]+?)\\s+COLLECTION/),
+ collection: g(/COLLECTION:\\s*([^\\n]+?)\\s+PRICE CODE/),
+ price_code: g(/PRICE CODE:\\s*([A-Z0-9]+)/),
+ net_price: g(/CASE PRICING[\\s\\S]*?NET PRICE\\s+\\d+\\s+([0-9.]+)/) || g(/PRICING[\\s\\S]*?NET PRICE\\s+\\d+\\s+([0-9.]+)/),
+ discontinued: /Discontinued Item/i.test(T),
+ low_inventory: /Low Inventory Alert/i.test(T),
+ width: g(/Roll Width\\s+([0-9.]+\\")/),
+ repeat: g(/Repeat\\s+([0-9.]+)/),
+ match: g(/Match\\s+([A-Za-z ]+?)\\s+Roll Width/),
+ material: g(/Material\\s+([A-Za-z ]+?)\\s+Installation/),
+ coverage: g(/Roll Coverage\\s+([^\\n]+?)\\s+Washability/),
+ roll_length: g(/Roll Length\\s+([^\\n]+?)\\s+(?:INVENTORY|BATCH|$)/),
+ washability: g(/Washability\\s+([A-Za-z ]+?)\\s+Removability/),
+ removability: g(/Removability\\s+([A-Za-z ]+?)\\s+Roll Length/),
+ installation: g(/Installation\\s+([A-Za-z ]+?)\\s+Repeat/),
+ images: imgs.slice(0, 6),
+ };
+ }`;
+ const raw = await ocRetry(['browser', 'evaluate', '--timeout', '25000', '--fn', fn], { tries: 3, timeout: 45000 });
+ // openclaw prints the JSON object (possibly with a leading status line). Grab the JSON block.
+ const start = raw.indexOf('{');
+ const end = raw.lastIndexOf('}');
+ if (start < 0 || end < 0) throw new Error('no JSON in openclaw output: ' + raw.slice(0, 200));
+ return JSON.parse(raw.slice(start, end + 1));
+}
+
+// Title -> pattern + color (e.g. "CHILTON YELLOW WILDFLOWERS WALLPAPER")
+function titleCase(s) {
+ return (s || '').toLowerCase().replace(/\b\w/g, c => c.toUpperCase());
+}
+
+async function main() {
+ const c = new Client(PG);
+ await c.connect();
+
+ let rows;
+ if (EXPLICIT) {
+ rows = EXPLICIT.map(sku => ({ mfr_sku: sku, eff: null }));
+ } else {
+ const q = `
+ SELECT s.mfr_sku,
+ COALESCE(NULLIF(s.dw_sku,''), regexp_replace(s.sku,'-Sample$','','i')) AS eff
+ FROM shopify_products s
+ WHERE s.vendor ILIKE 'jeffrey stevens' AND s.status='ACTIVE'
+ AND COALESCE(NULLIF(s.dw_sku,''), regexp_replace(s.sku,'-Sample$','','i')) LIKE 'DWJS-91%'
+ AND s.mfr_sku IS NOT NULL AND s.mfr_sku <> ''
+ GROUP BY 1,2
+ ORDER BY 2`;
+ const r = await c.query(q);
+ rows = r.rows;
+ }
+ if (LIMIT) rows = rows.slice(0, LIMIT);
+
+ writeFileSync(AUDIT, '');
+ console.log(`[js-scrape] ${rows.length} SKUs to look up (delay ${DELAY_MS}ms)`);
+ let ok = 0, disco = 0, current = 0, notfound = 0, err = 0;
+
+ for (let i = 0; i < rows.length; i++) {
+ const { mfr_sku, eff } = rows[i];
+ try {
+ const d = await lookup(mfr_sku);
+ if (!d.authed) { console.error(' !! SESSION DE-AUTHED — stopping.'); break; }
+ if (!d.found) {
+ notfound++;
+ appendFileSync(AUDIT, JSON.stringify({ mfr_sku, eff, status: 'NOT_FOUND', ...d }) + '\n');
+ console.log(` [${i + 1}/${rows.length}] ${mfr_sku} (${eff}) -> NOT FOUND on portal`);
+ await sleep(300);
+ continue;
+ }
+ const status = d.discontinued ? 'Discontinued' : 'Active';
+ if (d.discontinued) disco++; else current++;
+ // Pattern/color: portal gives full title; keep full title as pattern_name source,
+ // do NOT invent color (DW rule). color_name only if confidently splittable — leave null otherwise.
+ const patternFull = titleCase((d.title || '').replace(/\bWALLPAPER\b/ig, '').trim());
+
+ await c.query(`
+ INSERT INTO york_catalog
+ (mfr_sku, dw_sku, pattern_name, collection, width, pattern_repeat, roll_length,
+ match_type, material, washability, removability, coverage, application,
+ us_msrp, us_map, sku_status, image_url, all_images, product_url, uom,
+ brand, crawled_at, updated_at)
+ VALUES
+ ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,NOW(),NOW())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ dw_sku=COALESCE(york_catalog.dw_sku, EXCLUDED.dw_sku),
+ pattern_name=EXCLUDED.pattern_name,
+ collection=EXCLUDED.collection,
+ width=EXCLUDED.width,
+ pattern_repeat=EXCLUDED.pattern_repeat,
+ roll_length=EXCLUDED.roll_length,
+ match_type=EXCLUDED.match_type,
+ material=EXCLUDED.material,
+ washability=EXCLUDED.washability,
+ removability=EXCLUDED.removability,
+ coverage=EXCLUDED.coverage,
+ application=EXCLUDED.application,
+ us_msrp=EXCLUDED.us_msrp,
+ us_map=EXCLUDED.us_map,
+ sku_status=EXCLUDED.sku_status,
+ image_url=COALESCE(EXCLUDED.image_url, york_catalog.image_url),
+ all_images=EXCLUDED.all_images,
+ product_url=EXCLUDED.product_url,
+ uom=EXCLUDED.uom,
+ brand=EXCLUDED.brand,
+ crawled_at=NOW(),
+ updated_at=NOW()
+ `, [
+ mfr_sku, eff, patternFull || null, d.collection || null, d.width || null,
+ d.repeat || null, d.roll_length || null, d.match || null, d.material || null,
+ d.washability || null, d.removability || null, d.coverage || null, d.installation || null,
+ d.msrp ? parseFloat(d.msrp.replace(/,/g, '')) : null,
+ d.net_price ? parseFloat(d.net_price) : null,
+ status, (d.images && d.images[0]) || null,
+ d.images && d.images.length ? JSON.stringify(d.images) : null,
+ `https://www.yorkwall.com/Product/Search?sku=${mfr_sku}`,
+ d.uom || null, d.collection ? d.collection.split(' ')[0] : null,
+ ]);
+ ok++;
+ appendFileSync(AUDIT, JSON.stringify({ mfr_sku, eff, status, msrp: d.msrp, net: d.net_price, ...d }) + '\n');
+ console.log(` [${i + 1}/${rows.length}] ${mfr_sku} (${eff}) -> ${status}${d.low_inventory ? ' [low-inv]' : ''} | ${patternFull} | MAP $${d.net_price || '?'}`);
+ } catch (e) {
+ err++;
+ appendFileSync(AUDIT, JSON.stringify({ mfr_sku, eff, status: 'ERROR', error: String(e).slice(0, 300) }) + '\n');
+ console.error(` [${i + 1}/${rows.length}] ${mfr_sku} -> ERROR: ${String(e).slice(0, 120)}`);
+ }
+ }
+
+ await c.end();
+ console.log(`\n[js-scrape] DONE. upserted=${ok} (current=${current}, discontinued=${disco}), not_found=${notfound}, errors=${err}`);
+ console.log(`[js-scrape] audit: ${AUDIT}`);
+}
+
+main().catch(e => { console.error(e); process.exit(1); });
← 205a826b auto-save: 2026-06-25T18:06:20 (6 files) — pending-approval/
·
back to Designer Wallcoverings
·
JS reconcile: newline-agnostic title regex + post-scrape 91x d52584b4 →