wallquest refresh: port login fix into scrape.cjs (.login-button submit + 14s price-AJAX wait + loginVerified skip-unpriced-chunk + CHUNK≤15 + WQ_LIMIT test hook) — per memory wallquest-login-price-fix

1df17882854526a0461ce4f053853029c97aaf2b · 2026-07-06 12:13:02 -0700 · Steve
Files touched

M scripts/wallquest-refresh/scrape.cjs
Diff

commit 1df17882854526a0461ce4f053853029c97aaf2b
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Jul 6 12:13:02 2026 -0700

    wallquest refresh: port login fix into scrape.cjs (.login-button submit + 14s price-AJAX wait + loginVerified skip-unpriced-chunk + CHUNK≤15 + WQ_LIMIT test hook) — per memory wallquest-login-price-fix
---
 scripts/wallquest-refresh/scrape.cjs | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/wallquest-refresh/scrape.cjs b/scripts/wallquest-refresh/scrape.cjs
index c8b53f1f..401c94b7 100644
--- a/scripts/wallquest-refresh/scrape.cjs
+++ b/scripts/wallquest-refresh/scrape.cjs
@@ -10,7 +10,13 @@ const bb=new Browserbase({apiKey:process.env.BROWSERBASE_API_KEY});
 const goto=async(p,u)=>{for(let i=0;i<3;i++){try{await p.goto(u,{waitUntil:'domcontentloaded',timeout:45000});return 1;}catch(e){await p.waitForTimeout(3000);}}return 0;};
 const OUT='/tmp/wq-refresh.jsonl'; fs.writeFileSync(OUT,'');
 const rows=q("SELECT mfr_sku||'|'||product_url FROM wallquest_catalog WHERE product_url ~ 'wallquest.com' ORDER BY (price_retail IS NULL OR price_retail=0) DESC, mfr_sku;").split('\n').filter(Boolean).map(l=>{const[sku,url]=l.split('|');return{sku,url};});
-const out=fs.createWriteStream(OUT,{flags:'a'}); const CHUNK=55;
-async function login(page){await goto(page,'https://www.wallquest.com/login');await page.waitForTimeout(8000);await page.fill('#Email','info@designerwallcoverings.com');await page.fill('#Password',WPW);await page.press('#Password','Enter');await page.waitForTimeout(9000);}
+if(process.env.WQ_LIMIT)rows.splice(parseInt(process.env.WQ_LIMIT)); // test hook: WQ_LIMIT=10 → first 10 SKUs only
+const out=fs.createWriteStream(OUT,{flags:'a'}); const CHUNK=parseInt(process.env.WQ_CHUNK||'15'); // ≤15 per memory wallquest-login-price-fix (browserbase session lifetime)
+async function login(page){await goto(page,'https://www.wallquest.com/login');await page.waitForTimeout(8000);await page.fill('#Email','info@designerwallcoverings.com');await page.fill('#Password',WPW);
+  // FIX 2026-07-06 (memory: wallquest-login-price-fix): the login BUTTON submits; press-Enter silently no-ops so price stays login-gated.
+  let clicked=false;for(const sel of ['input.login-button','.login-button','button.login-button']){if(await page.$(sel)){try{await page.click(sel,{timeout:5000});clicked=true;break;}catch(e){}}}
+  if(!clicked)await page.press('#Password','Enter');await page.waitForTimeout(9000);}
+// verify login via a KNOWN-priced product (sy21843); retry up to 3x; caller skips the chunk if unverifiable (avoids emitting unpriced rows)
+async function loginVerified(page,tries=3){for(let t=0;t<tries;t++){await login(page);await goto(page,'https://www.wallquest.com/sy21843');await page.waitForTimeout(parseInt(process.env.WQ_PWAIT||'14000'));const gotPrice=await page.evaluate(()=>{const pv=([...document.querySelectorAll('[class*=price-value]')][0]||{}).textContent||'';return /[\d,]+\.\d{2}/.test(pv)&&!/sign in to view/i.test(document.body.innerText);});console.error(`  login attempt ${t+1}: priceVisible=${gotPrice}`);if(gotPrice)return true;}return false;}
 const extract=p=>p.evaluate(()=>{const Q=s=>[...document.querySelectorAll(s)];const specs={};Q('.spec-name').forEach(n=>{const v=n.nextElementSibling;if(v)specs[n.textContent.trim()]=v.textContent.trim();});const imgs=[...new Set(Q('a[data-full-image-url]').map(a=>a.getAttribute('data-full-image-url')).concat(Q('#cloudZoomImage,.picture img').map(i=>i.src)))].filter(u=>u&&/jpe?g|png/i.test(u));const pv=((Q('[class*=price-value]')[0]||{}).textContent||'').match(/[\d,]+\.\d{2}/);return{specs,imgs:imgs.slice(0,20),priceValue:pv?pv[0]:null};});
-(async()=>{let d=0,pr=0;for(let i=0;i<rows.length;i+=CHUNK){const chunk=rows.slice(i,i+CHUNK);let br;try{const s=await bb.sessions.create({projectId:process.env.BROWSERBASE_PROJECT_ID,proxies:true,browserSettings:{solveCaptchas:true}});br=await chromium.connectOverCDP(s.connectUrl);const ctx=br.contexts()[0];const page=ctx.pages()[0]||await ctx.newPage();await login(page);for(const r of chunk){try{if(!await goto(page,r.url)){out.write(JSON.stringify({sku:r.sku,nav:0})+'\n');d++;continue;}await page.waitForTimeout(9000);const x=await extract(page);if(x.priceValue)pr++;out.write(JSON.stringify({sku:r.sku,...x})+'\n');}catch(e){out.write(JSON.stringify({sku:r.sku,err:1})+'\n');}d++;}}catch(e){}finally{if(br)await br.close().catch(()=>{});}console.error(`[${Math.floor(i/CHUNK)+1}] done=${d} priced=${pr}`);}out.end();console.error(`SCRAPE DONE done=${d} priced=${pr}`);})();
+(async()=>{let d=0,pr=0;for(let i=0;i<rows.length;i+=CHUNK){const chunk=rows.slice(i,i+CHUNK);let br;try{const s=await bb.sessions.create({projectId:process.env.BROWSERBASE_PROJECT_ID,proxies:true,browserSettings:{solveCaptchas:true}});br=await chromium.connectOverCDP(s.connectUrl);const ctx=br.contexts()[0];const page=ctx.pages()[0]||await ctx.newPage();const authed=await loginVerified(page);if(!authed){console.error('  !! login unverified this session, skipping chunk to avoid unpriced rows');continue;}for(const r of chunk){try{if(!await goto(page,r.url)){out.write(JSON.stringify({sku:r.sku,nav:0})+'\n');d++;continue;}await page.waitForTimeout(parseInt(process.env.WQ_PWAIT||'14000'));const x=await extract(page);if(x.priceValue)pr++;out.write(JSON.stringify({sku:r.sku,...x})+'\n');}catch(e){out.write(JSON.stringify({sku:r.sku,err:1})+'\n');}d++;}}catch(e){}finally{if(br)await br.close().catch(()=>{});}console.error(`[${Math.floor(i/CHUNK)+1}] done=${d} priced=${pr}`);}out.end();console.error(`SCRAPE DONE done=${d} priced=${pr}`);})();
← eaec8222 Carl Robinson: SKU numbers start above 100000 (GRS-100001…) · back to Designer Wallcoverings · Carl Robinson: unique running 6-digit SKU numbers across boo 366e5801 →