← back to Designer Wallcoverings
WallQuest scraper: harden login vs NEW Cloudflare challenge (+reCAPTCHA)
564caf21bc292fd11fe1a8361511ff380a0ebb76 · 2026-07-09 19:38:52 -0700 · Steve
WallQuest escalated to Cloudflare 'Just a moment' on /login (curl now 403) ON TOP
of the reCAPTCHA. Both prior paths broke (Browserbase-Enter clone: no captcha solve;
grasscloth template: page.$ navigation race while Cloudflare redirected).
Fix in scrape-book-hardened.cjs login(): waitForLoadState + 16s Cloudflare-clear wait
+ waitForSelector('#Email') guard + try-wrapped page.$ so a mid-nav check can't kill
the session. Browserbase solveCaptchas:true + proxies handles Cloudflare AND reCAPTCHA.
Verified: login attempt 1 priceVisible=true, 3/3 priced+imaged (Sisal/Cork, real textile).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A scripts/wallquest-refresh/scrape-book-hardened.cjs
Diff
commit 564caf21bc292fd11fe1a8361511ff380a0ebb76
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jul 9 19:38:52 2026 -0700
WallQuest scraper: harden login vs NEW Cloudflare challenge (+reCAPTCHA)
WallQuest escalated to Cloudflare 'Just a moment' on /login (curl now 403) ON TOP
of the reCAPTCHA. Both prior paths broke (Browserbase-Enter clone: no captcha solve;
grasscloth template: page.$ navigation race while Cloudflare redirected).
Fix in scrape-book-hardened.cjs login(): waitForLoadState + 16s Cloudflare-clear wait
+ waitForSelector('#Email') guard + try-wrapped page.$ so a mid-nav check can't kill
the session. Browserbase solveCaptchas:true + proxies handles Cloudflare AND reCAPTCHA.
Verified: login attempt 1 priceVisible=true, 3/3 priced+imaged (Sisal/Cork, real textile).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/wallquest-refresh/scrape-book-hardened.cjs | 106 +++++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/scripts/wallquest-refresh/scrape-book-hardened.cjs b/scripts/wallquest-refresh/scrape-book-hardened.cjs
new file mode 100644
index 00000000..7ecf0683
--- /dev/null
+++ b/scripts/wallquest-refresh/scrape-book-hardened.cjs
@@ -0,0 +1,106 @@
+// Full-page scrape of the 87 NEW grasscloth/veneer products (from /tmp/wq-grasscloth-unique.jsonl).
+// Login -> per product: price (9s AJAX wait) + all specs + all images + pattern name + ratings.
+// Writes /tmp/wq-grasscloth-scraped.jsonl. NO DB writes here.
+const Browserbase = require('@browserbasehq/sdk').default;
+const { chromium } = require('playwright-core');
+const fs = require('fs');
+const { execSync } = require('child_process');
+require('dotenv').config({ path: require('os').homedir() + '/.claude/skills/browserbase/.env' });
+
+const PSQL = '/opt/homebrew/opt/postgresql@14/bin/psql';
+const ENVL = fs.readFileSync(require('os').homedir() + '/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/.env.local', 'utf8');
+const PGPW = (ENVL.match(/^POSTGRES_PASSWORD=(.+)$/m) || [])[1];
+const q = sql => execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -tAF'|' -c "${sql}"`, { encoding: 'utf8', maxBuffer: 1e8 });
+const WPW = q("SELECT trade_password FROM vendor_registry WHERE vendor_code='wallquest';").trim();
+
+const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
+const INFILE = process.env.WQ_INFILE || '/tmp/wq-grasscloth-unique.jsonl';
+let rows = fs.readFileSync(INFILE, 'utf8').split('\n').filter(Boolean).map(JSON.parse);
+if (process.env.WQ_LIMIT) rows = rows.slice(0, parseInt(process.env.WQ_LIMIT));
+const OUT = process.env.WQ_OUTFILE || '/tmp/wq-grasscloth-scraped.jsonl';
+if (!process.env.WQ_APPEND) fs.writeFileSync(OUT, '');
+const out = fs.createWriteStream(OUT, { flags: 'a' });
+const CHUNK = parseInt(process.env.WQ_CHUNK || '45'); // re-login per chunk (smaller = safer vs session lifetime)
+
+const goto = async (p, u) => { for (let i = 0; i < 3; i++) { try { await p.goto(u, { waitUntil: 'domcontentloaded', timeout: 45000 }); return 1; } catch (e) { await p.waitForTimeout(3000); } } return 0; };
+async function login(page) {
+ await goto(page, 'https://www.wallquest.com/login');
+ await page.waitForLoadState('domcontentloaded').catch(()=>{});
+ await page.waitForTimeout(16000); // let Cloudflare challenge clear + Browserbase solve
+ try { await page.waitForSelector('#Email', { timeout: 20000 }); } catch(e) { console.error(' #Email not present (cloudflare?):', e.message.slice(0,50)); return; }
+ await page.fill('#Email', 'info@designerwallcoverings.com').catch(()=>{});
+ await page.fill('#Password', WPW).catch(()=>{});
+ // FIX 2026-07-06: the login BUTTON submits; press-Enter silently no-ops (price stays login-gated).
+ let clicked = false;
+ for (const sel of ['input.login-button', '.login-button', 'button.login-button']) {
+ let el=null; try{ el=await page.$(sel);}catch(e){}
+ if (el) { try { await page.click(sel, { timeout: 5000 }); clicked = true; break; } catch (e) {} }
+ }
+ if (!clicked) await page.press('#Password', 'Enter');
+ await page.waitForTimeout(9000);
+}
+// verify login by loading a KNOWN-priced product; returns true if a real price rendered
+async function loginVerified(page, tries = 3) {
+ for (let t = 0; t < tries; t++) {
+ await login(page);
+ await goto(page, 'https://www.wallquest.com/sy21843');
+ await page.waitForTimeout(parseInt(process.env.WQ_PWAIT || '14000'));
+ const gotPrice = await page.evaluate(() => {
+ const pv = ([...document.querySelectorAll('[class*=price-value]')][0] || {}).textContent || '';
+ return /[\d,]+\.\d{2}/.test(pv) && !/sign in to view/i.test(document.body.innerText);
+ });
+ console.error(` login attempt ${t + 1}: priceVisible=${gotPrice}`);
+ if (gotPrice) return true;
+ }
+ return false;
+}
+const extract = p => p.evaluate(() => {
+ const Q = s => [...document.querySelectorAll(s)];
+ const txt = s => (Q(s)[0]?.textContent || '').trim();
+ const specs = {};
+ Q('.spec-name').forEach(n => { const v = n.nextElementSibling; if (v) specs[n.textContent.trim().replace(/:$/,'')] = v.textContent.trim(); });
+ const imgs = [...new Set(
+ Q('a[data-full-image-url]').map(a => a.getAttribute('data-full-image-url'))
+ .concat(Q('#cloudZoomImage,.picture img,.thumb-item img').map(i => i.getAttribute('src') || i.getAttribute('data-src')))
+ )].filter(u => u && /jpe?g|png/i.test(u)).map(u => u.startsWith('http') ? u : new URL(u, location.origin).href);
+ const pv = (txt('[class*=price-value]') || '').match(/[\d,]+\.\d{2}/);
+ // pattern name: prefer product h1, strip trailing sku
+ const name = txt('h1.product-name, h1[itemprop=name], .product-name h1, h1') || null;
+ const rating = txt('.rating, [itemprop=ratingValue]') || null;
+ const shortDesc = txt('.short-description, [itemprop=description]') || null;
+ // breadcrumb: clean whitespace, dedup; taxonomy = Home / Brands & Collections / <Pattern> / <Colorway>
+ const bc = [...new Set(Q('.breadcrumb a').map(a => a.textContent.replace(/\s+/g, ' ').trim()).filter(x => x && x !== '/'))];
+ const pattern_name = bc.length >= 2 ? bc[bc.length - 1] : null; // last breadcrumb link = pattern/collection
+ return { name, pattern_name, specs, imgs: imgs.slice(0, 25), priceValue: pv ? pv[0] : null, rating, shortDesc, breadcrumb: bc };
+});
+
+(async () => {
+ let done = 0, priced = 0, imaged = 0;
+ for (let i = 0; i < rows.length; i += CHUNK) {
+ const chunk = rows.slice(i, i + CHUNK);
+ let br;
+ try {
+ const s = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, proxies: true, browserSettings: { solveCaptchas: true } });
+ br = await chromium.connectOverCDP(s.connectUrl);
+ const ctx = br.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ const authed = await loginVerified(page);
+ if (!authed) { console.error(' !! login unverified this session, skipping chunk to avoid unpriced rows'); continue; }
+ for (const r of chunk) {
+ try {
+ if (!await goto(page, r.url)) { out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, nav: 0 }) + '\n'); done++; continue; }
+ await page.waitForTimeout(parseInt(process.env.WQ_PWAIT || '13000'));
+ const x = await extract(page);
+ if (x.priceValue) priced++;
+ if (x.imgs && x.imgs.length) imaged++;
+ out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, thumb: r.thumb, ...x }) + '\n');
+ } catch (e) { out.write(JSON.stringify({ mfr_sku: r.mfr_sku, url: r.url, err: e.message }) + '\n'); }
+ done++;
+ if (done % 10 === 0) console.error(`... ${done}/${rows.length} priced=${priced} imaged=${imaged}`);
+ }
+ } catch (e) { console.error('session err', e.message); }
+ finally { if (br) await br.close().catch(() => {}); }
+ }
+ out.end();
+ console.error(`SCRAPE DONE done=${done} priced=${priced} imaged=${imaged} -> ${OUT}`);
+})();
← c2c68ac6 auto-save: 2026-07-09T18:41:42 (5 files) — pending-approval/
·
back to Designer Wallcoverings
·
auto-save: 2026-07-09T19:41:55 (5 files) — pending-approval/ 42ffdd9a →