← back to Designer Wallcoverings
Onboard Daisy Bennett Naturals Vol 2 (WallQuest→Phillipe Romano): scrape 65 SKUs, stage to daisy_bennett_catalog (DWAY-40xxx), settlement-flag bird chinoiserie
95ef94b5650990446f7c5108f8166bc826bdf638 · 2026-07-06 11:18:40 -0700 · Steve
Files touched
A scripts/wallquest-refresh/load-db-daisy-bennett-v2.cjsA scripts/wallquest-refresh/normalize-daisy-bennett-v2.cjsA scripts/wallquest-refresh/onboard-daisy-bennett-v2.cjs
Diff
commit 95ef94b5650990446f7c5108f8166bc826bdf638
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jul 6 11:18:40 2026 -0700
Onboard Daisy Bennett Naturals Vol 2 (WallQuest→Phillipe Romano): scrape 65 SKUs, stage to daisy_bennett_catalog (DWAY-40xxx), settlement-flag bird chinoiserie
---
.../wallquest-refresh/load-db-daisy-bennett-v2.cjs | 74 +++++++++++
.../normalize-daisy-bennett-v2.cjs | 66 ++++++++++
.../wallquest-refresh/onboard-daisy-bennett-v2.cjs | 141 +++++++++++++++++++++
3 files changed, 281 insertions(+)
diff --git a/scripts/wallquest-refresh/load-db-daisy-bennett-v2.cjs b/scripts/wallquest-refresh/load-db-daisy-bennett-v2.cjs
new file mode 100644
index 00000000..b98dac7f
--- /dev/null
+++ b/scripts/wallquest-refresh/load-db-daisy-bennett-v2.cjs
@@ -0,0 +1,74 @@
+// Phase 6/7 — stage the 65 Daisy Bennett Naturals VOL 2 rows into the EXISTING (sibling-owned)
+// dw_unified.daisy_bennett_catalog. Convention matches Volume 1: DWAY- prefix, shared table,
+// Phillipe Romano private label, on_shopify=false (staging; publish is Steve-gated).
+// NAMING IS LEFT TO THE SIBLING SESSION (Steve's call): pattern_name stays NULL, the real vendor
+// design name goes to src_pattern so the namer has the source. We DO set settlement_status.
+// V2 mfr codes (DB#####→DWAY-40xxx) do not collide with V1 (BX#####→DWAY-10xxx).
+// Idempotent on mfr_sku; on conflict we refresh raw/spec/settlement fields but NEVER clobber
+// a pattern_name/collection the naming session may have set.
+const fs = require('fs');
+const { Client } = require('pg');
+
+const CONN = process.env.DATABASE_URL || 'postgresql://dw_admin:DW2024SecurePass@127.0.0.1:5432/dw_unified';
+const recs = JSON.parse(fs.readFileSync('/tmp/wq-daisy-bennett-v2-normalized.json', 'utf8'));
+const SOURCE_BOOK = 'Daisy Bennett Naturals Volume 2';
+
+// Per-family settlement triage (visual review; formal gate runs before any publish).
+// BLOCK = do not publish. REVIEW = needs formal settlement verdict. OK = clean texture/geometric.
+const SETTLE = {
+ 'Blossom Citrus Floral': 'BLOCK', // chinoiserie WITH A BIRD (Part B) + full Part A
+ 'Huanzhu Leaf Stripe': 'REVIEW', // tonal single-ink → likely OK, confirm via formal gate
+ 'Anhe Abaca': 'OK', 'Enya Sisal': 'OK', 'Meilian Raffia': 'OK', 'Meilin Grasscloth': 'OK',
+ 'Pei Ling Lattice': 'OK', 'Yujin Cork': 'OK', 'Zinzhu Jute': 'OK',
+};
+
+// Only additive, non-destructive column guards (won't disturb sibling's schema/rows).
+const GUARD = `
+ALTER TABLE daisy_bennett_catalog ADD COLUMN IF NOT EXISTS settlement_status text DEFAULT 'REVIEW';
+ALTER TABLE daisy_bennett_catalog ADD COLUMN IF NOT EXISTS src_pattern text;`;
+
+(async () => {
+ const db = new Client({ connectionString: CONN });
+ await db.connect();
+ await db.query(GUARD);
+ // Confirm we are ADDING to an existing V1 table, not creating a fresh one.
+ const pre = await db.query(`SELECT count(*) n, count(*) FILTER (WHERE source_book=$1) v2 FROM daisy_bennett_catalog`, [SOURCE_BOOK]);
+ console.log(`pre-load: table has ${pre.rows[0].n} rows (${pre.rows[0].v2} already tagged as V2)`);
+
+ let n = 0; const tally = {};
+ for (const r of recs) {
+ const settlement = SETTLE[r.pattern] || 'REVIEW';
+ tally[settlement] = (tally[settlement] || 0) + 1;
+ await db.query(`
+ INSERT INTO daisy_bennett_catalog
+ (mfr_sku,dw_sku,src_pattern,color_name,material,product_type,
+ price_retail,our_price,price_dw,width,length,repeat_v,match_type,backing,specs,
+ image_url,all_images,settlement_status,private_label_vendor,real_vendor_name,
+ source_book,on_shopify,product_url,updated_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,'Phillipe Romano','WallQuest',$19,false,$20,now())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ src_pattern=EXCLUDED.src_pattern, color_name=EXCLUDED.color_name, material=EXCLUDED.material,
+ price_retail=EXCLUDED.price_retail, our_price=EXCLUDED.our_price, price_dw=EXCLUDED.price_dw,
+ width=EXCLUDED.width, length=EXCLUDED.length, repeat_v=EXCLUDED.repeat_v,
+ match_type=EXCLUDED.match_type, backing=EXCLUDED.backing, specs=EXCLUDED.specs,
+ image_url=EXCLUDED.image_url, all_images=EXCLUDED.all_images,
+ settlement_status=EXCLUDED.settlement_status, source_book=EXCLUDED.source_book,
+ product_url=EXCLUDED.product_url, updated_at=now()
+ -- NOTE: pattern_name / collection intentionally NOT touched (sibling names them)`,
+ [r.mfr_sku, r.dw_sku, r.pattern, r.color || '', r.material || '', r.product_type || 'Wallcovering',
+ r.price_retail ?? null, r.our_price ?? null, r.price_retail ? +(r.price_retail / 0.55).toFixed(2) : null,
+ r.width || '', r.length || '', r.repeat || '', r.match || '', r.backing || '', JSON.stringify(r.specs || {}),
+ (r.images || [])[0] || '', JSON.stringify(r.images || []), settlement, SOURCE_BOOK, r.product_url || '']);
+ n++;
+ }
+ const { rows } = await db.query(`SELECT
+ count(*) FILTER (WHERE source_book=$1) v2_total,
+ count(*) FILTER (WHERE source_book=$1 AND price_retail>0) v2_priced,
+ count(*) FILTER (WHERE source_book=$1 AND settlement_status='OK') v2_ok,
+ count(*) FILTER (WHERE source_book=$1 AND settlement_status='REVIEW') v2_review,
+ count(*) FILTER (WHERE source_book=$1 AND settlement_status='BLOCK') v2_block,
+ count(*) total_all FROM daisy_bennett_catalog`, [SOURCE_BOOK]);
+ console.log(`staged ${n} V2 rows. settlement:`, tally);
+ console.log('post-load tally:', rows[0]);
+ await db.end();
+})().catch(e => { console.error('DB ERROR:', e.message); process.exit(1); });
diff --git a/scripts/wallquest-refresh/normalize-daisy-bennett-v2.cjs b/scripts/wallquest-refresh/normalize-daisy-bennett-v2.cjs
new file mode 100644
index 00000000..92af4067
--- /dev/null
+++ b/scripts/wallquest-refresh/normalize-daisy-bennett-v2.cjs
@@ -0,0 +1,66 @@
+// Phase 2 — normalize the raw Daisy Bennett Naturals VOL 2 scrape → structured records.
+// RAW source = WallQuest (2 scrape passes merged). TARGET private label = Phillipe Romano.
+// Named-pattern book (Design Name + Colorway Name), NOT plain textures — so we surface the
+// distinct pattern families for the beach-city naming map. NO rename/SKU-final here beyond
+// the DW numeric SKU (PREFIX via env, provisional until Steve confirms). NO publish.
+const fs = require('fs');
+const PREFIX = process.env.PREFIX || 'DWKA-'; // provisional phone-safe prefix (DB is forbidden: B,D)
+const INS = (process.env.INS || '/tmp/wq-daisy-bennett-v2.jsonl,/tmp/wq-daisy-bennett-v2-part2.jsonl').split(',');
+const OUT = process.env.OUT || '/tmp/wq-daisy-bennett-v2-normalized.json';
+
+// Merge all passes, dedupe by url, prefer the priced row.
+const byUrl = new Map();
+for (const f of INS) {
+ if (!fs.existsSync(f)) continue;
+ for (const line of fs.readFileSync(f, 'utf8').trim().split('\n').filter(Boolean)) {
+ let r; try { r = JSON.parse(line); } catch { continue; }
+ if (!r.url) continue;
+ const prev = byUrl.get(r.url);
+ const isGood = r.title && r.priceValue;
+ if (!prev || (isGood && !(prev.title && prev.priceValue))) byUrl.set(r.url, r);
+ }
+}
+const rows = [...byUrl.values()].filter(r => r.title && r.priceValue);
+
+const rec = rows.map(r => {
+ const slug = r.url.replace(/https?:\/\/[^/]+\//i, '').split('?')[0].replace(/\/$/, '');
+ const mfr_sku = slug.toUpperCase(); // db40022 → DB40022
+ const numTail = (mfr_sku.match(/\d+/) || ['0'])[0]; // 40022
+ const dw_sku = `${PREFIX}${numTail}`;
+ const s = r.specs || {};
+ const pattern = (s['Design Name'] || (r.title.split(' - ')[0] || '')).trim();
+ const color = (s['Colorway Name'] || (r.title.split(' - ')[1] || '')).trim();
+ const cost = parseFloat(String(r.priceValue).replace(/,/g, ''));
+ return {
+ mfr_sku, dw_sku,
+ src_title: r.title,
+ pattern, color,
+ price_retail: cost, // = our COST (WallQuest portal convention)
+ our_price: +(cost / 0.65 / 0.85).toFixed(2), // new-product retail formula
+ material: s['Material'] || s['Design Style'] || '',
+ backing: s['Backing Material'] || '',
+ width: s['Roll Width'] || s['Width'] || '',
+ length: s['Roll Length'] || s['Length'] || '',
+ repeat: s['Design Repeat'] || s['Vertical Repeat'] || '',
+ match: s['Design Match'] || s['Match'] || '',
+ design_style: s['Design Style'] || '',
+ product_type: 'Wallcovering', // WallQuest labels "Wallpaper" — DW banned word; store as Wallcovering
+ specs: s,
+ images: r.imgs || [],
+ product_url: r.url,
+ };
+});
+
+fs.writeFileSync(OUT, JSON.stringify(rec, null, 2));
+
+// Report distinct pattern families (drives the beach-city naming map + settlement triage)
+const byPattern = {};
+for (const x of rec) (byPattern[x.pattern] = byPattern[x.pattern] || []).push(x.color);
+console.log(`normalized ${rec.length} rows (prefix ${PREFIX}) → ${OUT}`);
+console.log(`\nPATTERN FAMILIES (${Object.keys(byPattern).length}):`);
+const FLORAL = /floral|leaf|blossom|botanic|vine|flower|bird|palm|tropic|fern|garden/i;
+for (const [p, cols] of Object.entries(byPattern).sort()) {
+ const flag = FLORAL.test(p) ? ' ⚠SETTLEMENT-REVIEW' : '';
+ console.log(` • ${p} (${cols.length}): ${cols.join(', ')}${flag}`);
+}
+console.log(`\ncost range: $${Math.min(...rec.map(r=>r.price_retail))}–$${Math.max(...rec.map(r=>r.price_retail))}`);
diff --git a/scripts/wallquest-refresh/onboard-daisy-bennett-v2.cjs b/scripts/wallquest-refresh/onboard-daisy-bennett-v2.cjs
new file mode 100644
index 00000000..569f6e7c
--- /dev/null
+++ b/scripts/wallquest-refresh/onboard-daisy-bennett-v2.cjs
@@ -0,0 +1,141 @@
+// Daisy Bennett — Naturals VOLUME 2 — collection ONBOARD scraper.
+// Mirrors onboard-carl-robinson.cjs / onboard-daisy-bennett.cjs exactly: discovers
+// product-detail URLs from the collection listing, then login-scrapes each
+// (specs/price/images/title/sku). Captures RAW vendor (WallQuest) data only — NO rename,
+// NO SKU assignment, NO publish. Private-label routing happens in later phases.
+// MODE=discover → paginate listing, save product URLs (cheap; ~1 bb session)
+// MODE=scrape → login-scrape each discovered URL → JSONL
+const Browserbase = require('@browserbasehq/sdk').default;
+const { chromium } = require('playwright-core');
+const fs = require('fs');
+require('dotenv').config({ path: require('os').homedir() + '/.claude/skills/browserbase/.env' });
+
+const COLLECTION = 'https://www.wallquest.com/daisy-bennett-naturals-volume-2';
+const WPW = '*Wallquestaccess911*';
+const WUSER = 'info@designerwallcoverings.com';
+const URLS_FILE = '/tmp/wq-daisy-bennett-v2-urls.json';
+const OUT = '/tmp/wq-daisy-bennett-v2.jsonl';
+const MODE = process.env.MODE || 'discover';
+// Any Daisy Bennett collection-landing slug is NOT a product (the priceless WQ-DAISYBENNETT* rows).
+const COLLECTION_SLUG = /daisy-bennett/i;
+
+const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
+const goto = async (p, u) => { for (let i = 0; i < 3; i++) { try { await p.goto(u, { waitUntil: 'domcontentloaded', timeout: 45000 }); return 1; } catch (e) { await p.waitForTimeout(3000); } } return 0; };
+const isLoggedIn = (page) => page.evaluate(() =>
+ /log\s*out|sign\s*out/i.test(document.body.innerText) ||
+ !!document.querySelector('a.ico-logout, .header-links a[href*="logout"]')
+).catch(() => false);
+
+async function login(page) {
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ await goto(page, 'https://www.wallquest.com/login');
+ await page.waitForTimeout(7000);
+ try {
+ await page.fill('#Email', WUSER);
+ await page.fill('#Password', WPW);
+ await page.press('#Password', 'Enter');
+ } catch (e) { /* form not ready; loop retries */ }
+ await page.waitForTimeout(9000);
+ if (await isLoggedIn(page)) { console.error(` login OK (attempt ${attempt})`); return true; }
+ console.error(` login attempt ${attempt} not confirmed, retrying…`);
+ await page.waitForTimeout(3000);
+ }
+ console.error(' LOGIN FAILED after 3 attempts — chunk will likely be priceless');
+ return false;
+}
+
+// Collect product-detail URLs from a nopCommerce collection listing page.
+const discoverOnPage = (p) => p.evaluate(() => {
+ const bad = /\/(login|logout|cart|wishlist|compare|register|account|search)|#/i;
+ const set = new Set();
+ document.querySelectorAll('.item-box a, .product-item a, .product-title a, .picture a, h2.product-title a').forEach(a => {
+ const href = a.href || '';
+ if (!href || bad.test(href)) return;
+ if (!/wallquest\.com\//i.test(href)) return;
+ const path = href.replace(/https?:\/\/[^/]+/i, '').split('?')[0].replace(/\/$/, '');
+ if (!path || path.split('/').length > 2) return;
+ if (/daisy-bennett/i.test(path)) return; // collection page/anchors, not products
+ set.add(href.split('?')[0]);
+ });
+ return [...set];
+});
+
+const extract = (p) => p.evaluate(() => {
+ const Q = s => [...document.querySelectorAll(s)];
+ const specs = {};
+ Q('.spec-name').forEach(n => { const v = n.nextElementSibling; if (v) specs[n.textContent.trim()] = v.textContent.trim(); });
+ const imgs = [...new Set(
+ Q('a[data-full-image-url]').map(a => a.getAttribute('data-full-image-url'))
+ .concat(Q('#cloudZoomImage,.picture img').map(i => i.src))
+ )].filter(u => u && /jpe?g|png/i.test(u));
+ const pv = ((Q('[class*=price-value]')[0] || {}).textContent || '').match(/[\d,]+\.\d{2}/);
+ const title = (document.querySelector('.product-name h1, h1.product-title, .product-essential h1, h1') || {}).textContent || '';
+ const sku = (document.querySelector('[itemprop=sku], .sku .value, .sku-number, .product-sku .value') || {}).textContent || '';
+ return { title: title.trim(), sku: sku.trim(), specs, imgs: imgs.slice(0, 20), priceValue: pv ? pv[0] : null };
+});
+
+(async () => {
+ if (MODE === 'discover') {
+ const s = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, proxies: true, browserSettings: { solveCaptchas: true } });
+ const br = await chromium.connectOverCDP(s.connectUrl);
+ const page = br.contexts()[0].pages()[0] || await br.contexts()[0].newPage();
+ try {
+ await login(page);
+ const all = new Set();
+ for (let pn = 1; pn <= 10; pn++) {
+ if (!await goto(page, `${COLLECTION}?orderby=50&pagesize=100&viewmode=grid&pagenumber=${pn}`)) break;
+ await page.waitForTimeout(6000);
+ const found = await discoverOnPage(page);
+ const before = all.size; found.forEach(u => all.add(u));
+ console.error(`[page ${pn}] found=${found.length} total=${all.size}`);
+ if (all.size === before) break; // no new products → past the end
+ }
+ const urls = [...all];
+ fs.writeFileSync(URLS_FILE, JSON.stringify(urls, null, 2));
+ console.error(`DISCOVER DONE — ${urls.length} product URLs → ${URLS_FILE}`);
+ console.log(JSON.stringify({ count: urls.length, sample: urls.slice(0, 12) }, null, 2));
+ } finally { await br.close().catch(() => {}); }
+ return;
+ }
+
+ // MODE=scrape (env-tunable: SRC url-file, OUTF output, CHUNK, WAITMS)
+ const SRC = process.env.SRC || URLS_FILE;
+ const OUTF = process.env.OUTF || OUT;
+ const CHUNK = parseInt(process.env.CHUNK || '40', 10);
+ const WAITMS = parseInt(process.env.WAITMS || '9000', 10);
+ const urls = JSON.parse(fs.readFileSync(SRC, 'utf8'));
+ fs.writeFileSync(OUTF, '');
+ const out = fs.createWriteStream(OUTF, { flags: 'a' });
+ let done = 0, priced = 0;
+ for (let i = 0; i < urls.length; i += CHUNK) {
+ const chunk = urls.slice(i, i + CHUNK); let br, page, authed = false;
+ for (let sTry = 1; sTry <= 3 && !authed; sTry++) {
+ try {
+ if (br) await br.close().catch(() => {});
+ const s = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, proxies: true, browserSettings: { solveCaptchas: true } });
+ br = await chromium.connectOverCDP(s.connectUrl);
+ page = br.contexts()[0].pages()[0] || await br.contexts()[0].newPage();
+ authed = await login(page);
+ if (!authed) console.error(` session ${sTry} did not authenticate — recreating`);
+ } catch (e) { console.error('session create err', String(e).slice(0, 120)); }
+ }
+ if (!authed) { console.error(`[chunk ${Math.floor(i / CHUNK) + 1}] SKIP — no authed session`); if (br) await br.close().catch(() => {}); continue; }
+ try {
+ for (const url of chunk) {
+ try {
+ if (!await goto(page, url)) { out.write(JSON.stringify({ url, nav: 0 }) + '\n'); done++; continue; }
+ await page.waitForTimeout(WAITMS);
+ let x = await extract(page);
+ if (!x.priceValue) { await page.waitForTimeout(6000); x = await extract(page); }
+ if (x.priceValue) priced++;
+ out.write(JSON.stringify({ url, ...x }) + '\n');
+ } catch (e) { out.write(JSON.stringify({ url, err: String(e).slice(0, 120) }) + '\n'); }
+ done++;
+ }
+ } catch (e) { console.error('session err', String(e).slice(0, 160)); }
+ finally { if (br) await br.close().catch(() => {}); }
+ console.error(`[chunk ${Math.floor(i / CHUNK) + 1}] done=${done}/${urls.length} priced=${priced}`);
+ }
+ out.end();
+ console.error(`SCRAPE DONE done=${done} priced=${priced} → ${OUTF}`);
+})();
← 532c47be Onboard Daisy Bennett Natural Wallcoverings (WallQuest→Phill
·
back to Designer Wallcoverings
·
Daisy Bennett: review viewer + Shopify draft payloads (63 pr 0862568a →