wallquest real-chrome scraper: discontinued-detection + honest realPriced metric + active-only filter + free-mode (no-2captcha, reuse cached session) + resumable. Pilots: active 20/20 priced, tiers verified (102.38/60.64/45.92).

5a738537bad346a74eade49911c4d11bf1f19c22 · 2026-07-06 13:36:25 -0700 · Steve
Files touched

A scripts/wallquest-refresh/diag-price.cjs
A scripts/wallquest-refresh/diag-variant.cjs
A scripts/wallquest-refresh/scrape-realchrome.cjs
Diff

commit 5a738537bad346a74eade49911c4d11bf1f19c22
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 6 13:36:25 2026 -0700

    wallquest real-chrome scraper: discontinued-detection + honest realPriced metric + active-only filter + free-mode (no-2captcha, reuse cached session) + resumable. Pilots: active 20/20 priced, tiers verified (102.38/60.64/45.92).
---
 scripts/wallquest-refresh/diag-price.cjs        | 30 +++++++++++
 scripts/wallquest-refresh/diag-variant.cjs      | 23 +++++++++
 scripts/wallquest-refresh/scrape-realchrome.cjs | 68 +++++++++++++++++++++++++
 3 files changed, 121 insertions(+)

diff --git a/scripts/wallquest-refresh/diag-price.cjs b/scripts/wallquest-refresh/diag-price.cjs
new file mode 100644
index 00000000..710372a7
--- /dev/null
+++ b/scripts/wallquest-refresh/diag-price.cjs
@@ -0,0 +1,30 @@
+// Diagnostic: does the extractor work, or are these products genuinely $0? Loads a few KNOWN-priced SKUs
+// (price_retail>0 in DB) + one $0.00 SKU via the authed profile, dumps ALL price-related DOM elements.
+const {chromium}=require('playwright-core'); const fs=require('fs'); const {execSync}=require('child_process'); const os=require('os');
+const PROFILE='/tmp/wq-chrome-profile'; const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+const goto=async(p,u)=>{for(let i=0;i<3;i++){try{await p.goto(u,{waitUntil:'domcontentloaded',timeout:45000});return 1;}catch(e){await sleep(3000);}}return 0;};
+const TESTS=[
+  {sku:'TG60010',db:'102.38'},  // different tier — prove varied prices, not one fixed value
+  {sku:'DA60001',db:'45.92'},   // different tier
+  {sku:'1301900',db:'60.64'},   // control (known $60.64)
+];
+(async()=>{
+  const ctx=await chromium.launchPersistentContext(PROFILE,{channel:'chrome',headless:false,viewport:{width:1280,height:900}});
+  const page=ctx.pages()[0]||await ctx.newPage();
+  // confirm still authed
+  await goto(page,'https://www.wallquest.com/sy21843'); await sleep(14000);
+  const authed=await page.evaluate(()=>/[\d,]+\.\d{2}/.test(([...document.querySelectorAll('[class*=price-value]')][0]||{}).textContent||'')&&!/sign in to view/i.test(document.body.innerText));
+  console.error('auth check sy21843 priced='+authed);
+  for(const t of TESTS){
+    await goto(page,'https://www.wallquest.com/'+t.sku); await sleep(14000);
+    const dump=await page.evaluate(()=>{
+      const els=[...document.querySelectorAll('[class*=price]')].map(e=>({cls:e.className,txt:(e.textContent||'').replace(/\s+/g,' ').trim().slice(0,50)})).filter(e=>e.txt);
+      const pv=([...document.querySelectorAll('[class*=price-value]')][0]||{}).textContent||'';
+      const m=pv.match(/[\d,]+\.\d{2}/);
+      return {extractPV:m?m[0]:null, signIn:/sign in to view/i.test(document.body.innerText), priceEls:els.slice(0,12)};
+    });
+    console.error(`\n=== ${t.sku} (DB=${t.db}) → extractPV=${dump.extractPV} signIn=${dump.signIn} ===`);
+    dump.priceEls.forEach(e=>console.error(`   [${e.cls}] "${e.txt}"`));
+  }
+  await ctx.close().catch(()=>{});
+})();
diff --git a/scripts/wallquest-refresh/diag-variant.cjs b/scripts/wallquest-refresh/diag-variant.cjs
new file mode 100644
index 00000000..3073cc75
--- /dev/null
+++ b/scripts/wallquest-refresh/diag-variant.cjs
@@ -0,0 +1,23 @@
+// Dump the product options/form structure on a "Please make a selection" product to find the real control.
+const {chromium}=require('playwright-core'); const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+const PROFILE='/tmp/wq-chrome-profile';
+const goto=async(p,u)=>{for(let i=0;i<3;i++){try{await p.goto(u,{waitUntil:'domcontentloaded',timeout:45000});return 1;}catch(e){await sleep(3000);}}return 0;};
+(async()=>{
+  const ctx=await chromium.launchPersistentContext(PROFILE,{channel:'chrome',headless:false,viewport:{width:1280,height:900}});
+  const page=ctx.pages()[0]||await ctx.newPage();
+  await goto(page,'https://www.wallquest.com/1820502'); await sleep(12000);
+  const info=await page.evaluate(()=>{
+    const pick=sel=>{const el=document.querySelector(sel);return el?el.outerHTML.replace(/\s+/g,' ').slice(0,900):null;};
+    // clickable things in the product area
+    const clickables=[...document.querySelectorAll('.product-essential a, .product-essential button, .overview a, .overview button, .attributes *[onclick], form[method] a, form[method] button')].map(e=>({tag:e.tagName,cls:e.className,txt:(e.textContent||'').trim().slice(0,30),href:e.getAttribute('href')||'',onclick:(e.getAttribute('onclick')||'').slice(0,40)})).filter(e=>e.txt||e.onclick).slice(0,20);
+    return {
+      attributes: pick('.attributes')||pick('.product-attributes')||pick('.product-variant-attributes'),
+      overviewSnippet: pick('.overview'),
+      clickables
+    };
+  });
+  console.error('ATTRIBUTES BLOCK:\n'+(info.attributes||'(none)'));
+  console.error('\nOVERVIEW SNIPPET:\n'+(info.overviewSnippet||'(none)').slice(0,700));
+  console.error('\nCLICKABLES:'); (info.clickables||[]).forEach(c=>console.error(`  <${c.tag}.${c.cls}> "${c.txt}" href=${c.href} onclick=${c.onclick}`));
+  await ctx.close().catch(()=>{});
+})();
diff --git a/scripts/wallquest-refresh/scrape-realchrome.cjs b/scripts/wallquest-refresh/scrape-realchrome.cjs
new file mode 100644
index 00000000..c01e1965
--- /dev/null
+++ b/scripts/wallquest-refresh/scrape-realchrome.cjs
@@ -0,0 +1,68 @@
+// WallQuest refresh via REAL local Chrome, reusing the authenticated persistent profile
+// (/tmp/wq-chrome-profile). Headless channel:chrome carries the live session cookies → no per-SKU
+// captcha. Re-auths via 2captcha only if the session lapsed. Same row query + output keys as
+// scrape.cjs so write.cjs is unchanged. NO paid LLM. WQ_LIMIT caps for pilots. $0 unless a re-login
+// captcha solve is needed (~$0.003).
+const {chromium}=require('playwright-core'); const fs=require('fs'); const {execSync}=require('child_process'); const os=require('os');
+const PSQL='/opt/homebrew/opt/postgresql@14/bin/psql';
+const ENVL=fs.readFileSync(os.homedir()+'/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/.env.local','utf8');
+const PGPW=(ENVL.match(/^POSTGRES_PASSWORD=(.+)$/m)||[])[1];
+const q=sql=>execSync(`PGPASSWORD='${PGPW}' ${PSQL} -h localhost -U dw_admin -d dw_unified -tAF'|' -c "${sql}"`,{encoding:'utf8',maxBuffer:1e8});
+const WPW=q("SELECT trade_password FROM vendor_registry WHERE vendor_code='wallquest';").trim();
+const CAPKEY=(fs.readFileSync(os.homedir()+'/.claude/skills/2captcha/.env','utf8').match(/^TWOCAPTCHA_API_KEY=(.+)$/m)||[])[1]?.trim();
+const PROFILE='/tmp/wq-chrome-profile';
+const OUT='/tmp/wq-refresh.jsonl';
+if(process.env.WQ_FRESH || !fs.existsSync(OUT)) fs.writeFileSync(OUT,''); // WQ_FRESH=1 starts clean; else RESUME (skip done)
+const done=new Set(); fs.readFileSync(OUT,'utf8').split('\n').filter(Boolean).forEach(l=>{try{const o=JSON.parse(l);if(o.sku)done.add(o.sku);}catch(e){}});
+const out=fs.createWriteStream(OUT,{flags:'a'});
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+const PWAIT=parseInt(process.env.WQ_PWAIT||'14000');
+const goto=async(p,u)=>{for(let i=0;i<3;i++){try{await p.goto(u,{waitUntil:'domcontentloaded',timeout:45000});return 1;}catch(e){await sleep(3000);}}return 0;};
+
+const priceFilter=process.env.WQ_ONLY_PRICED?'AND price_retail>0':''; // pilot pricing on ACTIVE SKUs
+let rows=q(`SELECT mfr_sku||'|'||product_url FROM wallquest_catalog WHERE product_url ~ 'wallquest.com' ${priceFilter} ORDER BY (price_retail IS NULL OR price_retail=0) DESC, mfr_sku;`).split('\n').filter(Boolean).map(l=>{const[sku,url]=l.split('|');return{sku,url};});
+if(process.env.WQ_LIMIT)rows.splice(parseInt(process.env.WQ_LIMIT));
+
+const extract=p=>p.evaluate(()=>{const Q=s=>[...document.querySelectorAll(s)];const specs={};Q('.spec-name').forEach(n=>{const v=n.nextElementSibling;if(v)specs[n.textContent.trim()]=v.textContent.trim();});const imgs=[...new Set(Q('a[data-full-image-url]').map(a=>a.getAttribute('data-full-image-url')).concat(Q('#cloudZoomImage,.picture img').map(i=>i.src)))].filter(u=>u&&/jpe?g|png/i.test(u));const pv=((Q('[class*=price-value]')[0]||{}).textContent||'').match(/[\d,]+\.\d{2}/);const discontinued=!!document.querySelector('.discontinued-product')||/no longer available/i.test(document.body.innerText);return{specs,imgs:imgs.slice(0,20),priceValue:pv?pv[0]:null,discontinued};});
+
+async function solve(sitekey,pageurl){
+  const j=await (await fetch(`http://2captcha.com/in.php?key=${CAPKEY}&method=userrecaptcha&googlekey=${sitekey}&pageurl=${encodeURIComponent(pageurl)}&json=1`,{method:'POST'})).json();
+  if(j.status!==1) throw new Error('2captcha in: '+JSON.stringify(j)); const id=j.request; console.error('2captcha job='+id);
+  for(let i=0;i<40;i++){await sleep(5000);const r=await (await fetch(`http://2captcha.com/res.php?key=${CAPKEY}&action=get&id=${id}&json=1`)).json();if(r.status===1)return r.request;if(r.request!=='CAPCHA_NOT_READY')throw new Error('2captcha res: '+JSON.stringify(r));}
+  throw new Error('2captcha timeout');
+}
+async function priced(page){await goto(page,'https://www.wallquest.com/sy21843');await sleep(PWAIT);return page.evaluate(()=>/[\d,]+\.\d{2}/.test(([...document.querySelectorAll('[class*=price-value]')][0]||{}).textContent||'')&&!/sign in to view/i.test(document.body.innerText));}
+async function reLogin(page){
+  if(!process.env.WQ_ALLOW_2CAPTCHA){ console.error('FREE MODE: session lapsed and 2captcha disabled — re-establish the session manually (node auto-login-2captcha.cjs once, OR solve in a window) then re-run to continue. No spend.'); return; }
+  console.error('session lapsed — re-authenticating via 2captcha');
+  await goto(page,'https://www.wallquest.com/login'); await sleep(6000);
+  await page.fill('#Email','info@designerwallcoverings.com'); await page.fill('#Password',WPW);
+  const sk=await page.evaluate(()=>{const el=document.querySelector('[data-sitekey]');if(el)return el.dataset.sitekey;const ifr=[...document.querySelectorAll('iframe')].find(f=>/recaptcha/.test(f.src||''));if(ifr){const m=(ifr.src||'').match(/[?&]k=([^&]+)/);return m?m[1]:null;}return null;});
+  if(!sk)throw new Error('no sitekey on re-login'); const tok=await solve(sk,page.url());
+  await page.evaluate(t=>{let ta=document.querySelector('textarea[name="g-recaptcha-response"]');if(!ta){ta=document.createElement('textarea');ta.name='g-recaptcha-response';ta.style.display='none';document.body.appendChild(ta);}ta.value=t;ta.innerHTML=t;},tok);
+  for(const sel of ['input.login-button','.login-button','button.login-button']){if(await page.$(sel)){try{await page.click(sel,{timeout:5000});break;}catch(e){}}}
+  await sleep(9000);
+}
+
+(async()=>{
+  console.error(`real-Chrome refresh: ${rows.length} rows, profile=${PROFILE}`);
+  // MUST be headful — headless system-Chrome is still anti-bot-flagged on WallQuest (blank login/product
+  // pages). The human fingerprint that unlocks pricing comes from a VISIBLE Chrome, not just channel:chrome.
+  const ctx=await chromium.launchPersistentContext(PROFILE,{channel:'chrome',headless:false,viewport:{width:1280,height:900}});
+  const page=ctx.pages()[0]||await ctx.newPage();
+  if(!await priced(page)){ await reLogin(page); if(!await priced(page)){ console.error('!! could not authenticate — aborting'); await ctx.close().catch(()=>{}); process.exit(2); } }
+  console.error('session authenticated ✓ — crawling');
+  let d=0,pr=0,dc=0,sk=0;
+  for(const r of rows){
+    if(done.has(r.sku)){d++;sk++;continue;} // resume: already scraped
+    try{
+      if(!await goto(page,r.url)){out.write(JSON.stringify({sku:r.sku,nav:0})+'\n');d++;continue;}
+      await sleep(PWAIT); const x=await extract(page);
+      const realPrice=x.priceValue && x.priceValue!=='0.00' && !x.discontinued; if(realPrice)pr++; if(x.discontinued)dc++;
+      out.write(JSON.stringify({sku:r.sku,...x})+'\n');
+    }catch(e){out.write(JSON.stringify({sku:r.sku,err:1})+'\n');}
+    d++; if(d%10===0)console.error(`... ${d}/${rows.length} realPriced=${pr} discontinued=${dc}`);
+  }
+  out.end(); await ctx.close().catch(()=>{});
+  console.error(`SCRAPE DONE done=${d} realPriced=${pr} discontinued=${dc} resumed-skipped=${sk}`);
+})();
← 68d732b0 sangetsu dossier: record Goodrich sole/exclusive distributor · back to Designer Wallcoverings · wallquest login: check Remember-me → persistent session (one 7c206ae0 →