[object Object]

← back to Designer Wallcoverings

wallquest: onboard specialty grasscloths & veneers (87 SKUs staged DWQW-61453..61539)

95e17fd7b70663ef714e4cb382ec19cbf4e8bdef · 2026-07-06 10:40:23 -0700 · Steve

- discover-grasscloth.cjs: browserbase login + paginate collection grid
- scrape-grasscloth.cjs: full-page price/specs/images/pattern scrape (chunked, retry-safe)
- insert-grasscloth.cjs: private-label scrub + DWQW SKU assign + jsonb tags, staged unpublished
- price unresolved (WallQuest has not price-published these); activation blocked on cost

Files touched

Diff

commit 95e17fd7b70663ef714e4cb382ec19cbf4e8bdef
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 6 10:40:23 2026 -0700

    wallquest: onboard specialty grasscloths & veneers (87 SKUs staged DWQW-61453..61539)
    
    - discover-grasscloth.cjs: browserbase login + paginate collection grid
    - scrape-grasscloth.cjs: full-page price/specs/images/pattern scrape (chunked, retry-safe)
    - insert-grasscloth.cjs: private-label scrub + DWQW SKU assign + jsonb tags, staged unpublished
    - price unresolved (WallQuest has not price-published these); activation blocked on cost
---
 scripts/wallquest-refresh/discover-grasscloth.cjs | 76 +++++++++++++++++++++
 scripts/wallquest-refresh/insert-grasscloth.cjs   | 61 +++++++++++++++++
 scripts/wallquest-refresh/scrape-grasscloth.cjs   | 82 +++++++++++++++++++++++
 3 files changed, 219 insertions(+)

diff --git a/scripts/wallquest-refresh/discover-grasscloth.cjs b/scripts/wallquest-refresh/discover-grasscloth.cjs
new file mode 100644
index 00000000..92efede4
--- /dev/null
+++ b/scripts/wallquest-refresh/discover-grasscloth.cjs
@@ -0,0 +1,76 @@
+// Discover products on WallQuest "specialty-grasscloths-and-veneers" collection.
+// Login (browserbase clears CF) -> paginate grid pagesize=100 -> capture URL/title/thumb/sku-hint.
+// Writes /tmp/wq-grasscloth-discover.jsonl. NO writes to dw_unified.
+const Browserbase = require('@browserbasehq/sdk').default;
+const { chromium } = require('playwright-core');
+const fs = require('fs');
+const { execSync } = require('child_process');
+require('dotenv').config({ path: require('os').homedir() + '/.claude/skills/browserbase/.env' });
+
+const PSQL = '/opt/homebrew/opt/postgresql@14/bin/psql';
+const ENVL = fs.readFileSync(require('os').homedir() + '/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/.env.local', 'utf8');
+const PGPW = (ENVL.match(/^POSTGRES_PASSWORD=(.+)$/m) || [])[1];
+const q = sql => execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -tAF'|' -c "${sql}"`, { encoding: 'utf8', maxBuffer: 1e8 });
+const WPW = q("SELECT trade_password FROM vendor_registry WHERE vendor_code='wallquest';").trim();
+
+const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
+const BASE = 'https://www.wallquest.com/specialty-grasscloths-and-veneers';
+const OUT = '/tmp/wq-grasscloth-discover.jsonl';
+fs.writeFileSync(OUT, '');
+
+const goto = async (p, u) => { for (let i = 0; i < 3; i++) { try { await p.goto(u, { waitUntil: 'domcontentloaded', timeout: 45000 }); return 1; } catch (e) { await p.waitForTimeout(3000); } } return 0; };
+
+async function login(page) {
+  await goto(page, 'https://www.wallquest.com/login');
+  await page.waitForTimeout(8000);
+  await page.fill('#Email', 'info@designerwallcoverings.com');
+  await page.fill('#Password', WPW);
+  await page.press('#Password', 'Enter');
+  await page.waitForTimeout(9000);
+}
+
+const extractTiles = p => p.evaluate(() => {
+  const abs = h => { try { return new URL(h, location.origin).href; } catch { return h; } };
+  const items = [...document.querySelectorAll('.item-box, .product-item')];
+  const out = [];
+  for (const it of items) {
+    const a = it.querySelector('.product-title a, h2 a, .product-title a[href], a.product-title, a[href]');
+    if (!a) continue;
+    const href = a.getAttribute('href');
+    if (!href || /\/(login|register|cart|wishlist|compare|account)/i.test(href)) continue;
+    const img = it.querySelector('.picture img, img');
+    out.push({
+      url: abs(href),
+      title: (a.textContent || '').trim(),
+      thumb: img ? abs(img.getAttribute('src') || img.getAttribute('data-src') || '') : null,
+      sku_hint: (it.querySelector('.sku, [class*=sku]') || {}).textContent?.trim() || null,
+    });
+  }
+  return out;
+});
+
+(async () => {
+  let br;
+  const seen = new Set();
+  try {
+    const s = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, proxies: true, browserSettings: { solveCaptchas: true } });
+    br = await chromium.connectOverCDP(s.connectUrl);
+    const ctx = br.contexts()[0];
+    const page = ctx.pages()[0] || await ctx.newPage();
+    await login(page);
+    const outS = fs.createWriteStream(OUT, { flags: 'a' });
+    for (let pn = 1; pn <= 15; pn++) {
+      const u = `${BASE}?orderby=50&pagesize=100&viewmode=grid&pagenumber=${pn}`;
+      if (!await goto(page, u)) { console.error(`page ${pn} nav fail`); break; }
+      await page.waitForTimeout(4000);
+      const tiles = await extractTiles(page);
+      const fresh = tiles.filter(t => t.url && !seen.has(t.url));
+      fresh.forEach(t => { seen.add(t.url); outS.write(JSON.stringify(t) + '\n'); });
+      console.error(`page ${pn}: tiles=${tiles.length} fresh=${fresh.length} total=${seen.size}`);
+      if (tiles.length === 0 || fresh.length === 0) break;
+    }
+    outS.end();
+  } catch (e) { console.error('ERR', e.message); }
+  finally { if (br) await br.close().catch(() => {}); }
+  console.error(`DISCOVER DONE total=${seen.size} -> ${OUT}`);
+})();
diff --git a/scripts/wallquest-refresh/insert-grasscloth.cjs b/scripts/wallquest-refresh/insert-grasscloth.cjs
new file mode 100644
index 00000000..8c4e140f
--- /dev/null
+++ b/scripts/wallquest-refresh/insert-grasscloth.cjs
@@ -0,0 +1,61 @@
+// Stage the 87 new grasscloth/veneer products into wallquest_catalog with DWQW SKUs.
+// Private-label scrub, spec parse, description+tags. price fields NULL (cost unresolved).
+// shopify_product_id stays NULL (NOT published). Idempotent via ON CONFLICT(mfr_sku).
+const fs = require('fs');
+const { execSync } = require('child_process');
+const PSQL = '/opt/homebrew/opt/postgresql@14/bin/psql';
+const ENVL = fs.readFileSync(require('os').homedir() + '/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/.env.local', 'utf8');
+const PGPW = (ENVL.match(/^POSTGRES_PASSWORD=(.+)$/m) || [])[1];
+const q = sql => execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -tAF'|' -c "${sql.replace(/"/g, '\\"')}"`, { encoding: 'utf8', maxBuffer: 1e8 });
+
+const BANNED = /wallquest|chesapeake|brewster|nextwall|seabrook|voyage|william\s*morris|miyabi/ig;
+const scrub = s => String(s == null ? '' : s).replace(BANNED, '').replace(/\s{2,}/g, ' ').trim();
+const esc = s => String(s == null ? '' : s).replace(/'/g, "''");
+const sql = v => (v == null || v === '') ? 'NULL' : "'" + esc(v) + "'";
+const numOrNull = v => (v == null || isNaN(v)) ? 'NULL' : v;
+
+const rows = fs.readFileSync('/tmp/wq-grasscloth-final.jsonl', 'utf8').split('\n').filter(Boolean).map(JSON.parse)
+  .filter(r => r.imgs && r.imgs.length)
+  .sort((a, b) => a.mfr_sku.localeCompare(b.mfr_sku));
+
+// next dw_sku
+const maxNum = parseInt((q("SELECT max(NULLIF(regexp_replace(dw_sku,'\\D','','g'),'')::int) FROM wallquest_catalog WHERE dw_sku ~ '^DWQW';").trim()) || '61452', 10);
+let next = maxNum + 1;
+
+const parseInches = w => { const m = String(w || '').match(/([\d.]+)\s*in/i); return m ? parseFloat(m[1]) : null; };
+
+const values = rows.map(r => {
+  const s = r.specs || {};
+  const pattern = scrub(r.pattern_name);
+  const color = scrub(r.name);
+  const material = scrub(s.Material);
+  const width = s['Roll Width'] || null;
+  const width_in = parseInches(width);
+  const img0 = r.imgs[0];
+  const allImgs = JSON.stringify(r.imgs);
+  // description (for 5-field gate) — customer-safe, no vendor name
+  const bits = [];
+  if (material) bits.push(material.toLowerCase());
+  bits.push('wallcovering');
+  if (color) bits.push(`in ${color}`);
+  const specLine = ['Design Repeat', 'Match', 'Roll Width', 'Roll Length', 'Coverage'].map(k => s[k] ? `${k}: ${s[k]}` : null).filter(Boolean).join('; ');
+  const desc = scrub(`${pattern || 'Specialty grasscloth'} ${bits.join(' ')}. ${specLine}`.trim());
+  // tags (>=2): material family, 'Grasscloth', colorway, pattern
+  const tagSet = new Set(['Grasscloth', 'Specialty Wallcovering']);
+  if (material) tagSet.add(material);
+  if (color) tagSet.add(color);
+  if (pattern) tagSet.add(pattern);
+  const tagArr = [...tagSet].map(scrub).filter(Boolean);
+  const tagsJson = "'" + esc(JSON.stringify(tagArr)) + "'::jsonb";
+  const dw = 'DWQW-' + (next++);
+  return `(${sql(r.mfr_sku)},${sql(dw)},${sql(pattern)},${sql(color)},${sql(pattern)},'Wallcovering',${sql(width)},${sql(material)},${sql(img0)},'${esc(allImgs)}',${sql(r.url)},${numOrNull(width_in)},${sql(desc)},${tagsJson},false)`;
+});
+
+const stmt = `INSERT INTO wallquest_catalog
+ (mfr_sku, dw_sku, pattern_name, color_name, collection, product_type, width, material, image_url, all_images, product_url, width_inches, ai_description, ai_tags, on_shopify)
+ VALUES ${values.join(',\n')}
+ ON CONFLICT (mfr_sku) DO NOTHING;`;
+fs.writeFileSync('/tmp/wq-grasscloth-insert.sql', stmt);
+const res = execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -c "$(cat /tmp/wq-grasscloth-insert.sql)"`, { encoding: 'utf8', maxBuffer: 1e8 });
+console.log('INSERT result:', res.trim());
+console.log(`assigned DWQW-${maxNum + 1} .. DWQW-${next - 1} (${rows.length} rows)`);
diff --git a/scripts/wallquest-refresh/scrape-grasscloth.cjs b/scripts/wallquest-refresh/scrape-grasscloth.cjs
new file mode 100644
index 00000000..7f4302ab
--- /dev/null
+++ b/scripts/wallquest-refresh/scrape-grasscloth.cjs
@@ -0,0 +1,82 @@
+// Full-page scrape of the 87 NEW grasscloth/veneer products (from /tmp/wq-grasscloth-unique.jsonl).
+// Login -> per product: price (9s AJAX wait) + all specs + all images + pattern name + ratings.
+// Writes /tmp/wq-grasscloth-scraped.jsonl. NO DB writes here.
+const Browserbase = require('@browserbasehq/sdk').default;
+const { chromium } = require('playwright-core');
+const fs = require('fs');
+const { execSync } = require('child_process');
+require('dotenv').config({ path: require('os').homedir() + '/.claude/skills/browserbase/.env' });
+
+const PSQL = '/opt/homebrew/opt/postgresql@14/bin/psql';
+const ENVL = fs.readFileSync(require('os').homedir() + '/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/.env.local', 'utf8');
+const PGPW = (ENVL.match(/^POSTGRES_PASSWORD=(.+)$/m) || [])[1];
+const q = sql => execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -tAF'|' -c "${sql}"`, { encoding: 'utf8', maxBuffer: 1e8 });
+const WPW = q("SELECT trade_password FROM vendor_registry WHERE vendor_code='wallquest';").trim();
+
+const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
+const INFILE = process.env.WQ_INFILE || '/tmp/wq-grasscloth-unique.jsonl';
+let rows = fs.readFileSync(INFILE, 'utf8').split('\n').filter(Boolean).map(JSON.parse);
+if (process.env.WQ_LIMIT) rows = rows.slice(0, parseInt(process.env.WQ_LIMIT));
+const OUT = process.env.WQ_OUTFILE || '/tmp/wq-grasscloth-scraped.jsonl';
+if (!process.env.WQ_APPEND) fs.writeFileSync(OUT, '');
+const out = fs.createWriteStream(OUT, { flags: 'a' });
+const CHUNK = parseInt(process.env.WQ_CHUNK || '45'); // re-login per chunk (smaller = safer vs session lifetime)
+
+const goto = async (p, u) => { for (let i = 0; i < 3; i++) { try { await p.goto(u, { waitUntil: 'domcontentloaded', timeout: 45000 }); return 1; } catch (e) { await p.waitForTimeout(3000); } } return 0; };
+async function login(page) {
+  await goto(page, 'https://www.wallquest.com/login');
+  await page.waitForTimeout(8000);
+  await page.fill('#Email', 'info@designerwallcoverings.com');
+  await page.fill('#Password', WPW);
+  await page.press('#Password', 'Enter');
+  await page.waitForTimeout(9000);
+}
+const extract = p => p.evaluate(() => {
+  const Q = s => [...document.querySelectorAll(s)];
+  const txt = s => (Q(s)[0]?.textContent || '').trim();
+  const specs = {};
+  Q('.spec-name').forEach(n => { const v = n.nextElementSibling; if (v) specs[n.textContent.trim().replace(/:$/,'')] = v.textContent.trim(); });
+  const imgs = [...new Set(
+    Q('a[data-full-image-url]').map(a => a.getAttribute('data-full-image-url'))
+    .concat(Q('#cloudZoomImage,.picture img,.thumb-item img').map(i => i.getAttribute('src') || i.getAttribute('data-src')))
+  )].filter(u => u && /jpe?g|png/i.test(u)).map(u => u.startsWith('http') ? u : new URL(u, location.origin).href);
+  const pv = (txt('[class*=price-value]') || '').match(/[\d,]+\.\d{2}/);
+  // pattern name: prefer product h1, strip trailing sku
+  const name = txt('h1.product-name, h1[itemprop=name], .product-name h1, h1') || null;
+  const rating = txt('.rating, [itemprop=ratingValue]') || null;
+  const shortDesc = txt('.short-description, [itemprop=description]') || null;
+  // breadcrumb: clean whitespace, dedup; taxonomy = Home / Brands & Collections / <Pattern> / <Colorway>
+  const bc = [...new Set(Q('.breadcrumb a').map(a => a.textContent.replace(/\s+/g, ' ').trim()).filter(x => x && x !== '/'))];
+  const pattern_name = bc.length >= 2 ? bc[bc.length - 1] : null; // last breadcrumb link = pattern/collection
+  return { name, pattern_name, specs, imgs: imgs.slice(0, 25), priceValue: pv ? pv[0] : null, rating, shortDesc, breadcrumb: bc };
+});
+
+(async () => {
+  let done = 0, priced = 0, imaged = 0;
+  for (let i = 0; i < rows.length; i += CHUNK) {
+    const chunk = rows.slice(i, i + CHUNK);
+    let br;
+    try {
+      const s = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, proxies: true, browserSettings: { solveCaptchas: true } });
+      br = await chromium.connectOverCDP(s.connectUrl);
+      const ctx = br.contexts()[0];
+      const page = ctx.pages()[0] || await ctx.newPage();
+      await login(page);
+      for (const r of chunk) {
+        try {
+          if (!await goto(page, r.url)) { out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, nav: 0 }) + '\n'); done++; continue; }
+          await page.waitForTimeout(9000);
+          const x = await extract(page);
+          if (x.priceValue) priced++;
+          if (x.imgs && x.imgs.length) imaged++;
+          out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, thumb: r.thumb, ...x }) + '\n');
+        } catch (e) { out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, err: e.message }) + '\n'); }
+        done++;
+        if (done % 10 === 0) console.error(`... ${done}/${rows.length} priced=${priced} imaged=${imaged}`);
+      }
+    } catch (e) { console.error('session err', e.message); }
+    finally { if (br) await br.close().catch(() => {}); }
+  }
+  out.end();
+  console.error(`SCRAPE DONE done=${done} priced=${priced} imaged=${imaged} -> ${OUT}`);
+})();

← bd1f7ba7 chore: macstudio3 migration — reconcile from mac2 + repoint  ·  back to Designer Wallcoverings  ·  Carl Robinson E20 → Phillipe Romano onboard pipeline (scrape 60fb5850 →