Schumacher cost via Browserbase scraper (residential IP, real login form); login-submit still needs fix (see browserbase/SCHUMACHER-NOTES.md)

6a69015ddac792202fdd046b67cb8b498d70bbb7 · 2026-06-11 09:50:01 -0700 · SteveStudio2

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files touched

A shopify/scripts/schu-browserbase-scrape.js

Diff

commit 6a69015ddac792202fdd046b67cb8b498d70bbb7
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Thu Jun 11 09:50:01 2026 -0700

    Schumacher cost via Browserbase scraper (residential IP, real login form); login-submit still needs fix (see browserbase/SCHUMACHER-NOTES.md)
    
    Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 shopify/scripts/schu-browserbase-scrape.js | 94 ++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/shopify/scripts/schu-browserbase-scrape.js b/shopify/scripts/schu-browserbase-scrape.js
new file mode 100644
index 00000000..b8c27f29
--- /dev/null
+++ b/shopify/scripts/schu-browserbase-scrape.js
@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+/**
+ * Schumacher cost scraper via BROWSERBASE (cloud browser, residential IP).
+ *
+ * Why: schumacher.com gates ALL pricing behind trade login (anonymous priceUsd=null),
+ * AND blocks datacenter IPs (Kamatera) + the Mac2 headless browser segfaults.
+ * Browserbase = residential-IP real Chromium → drives the login form, then reads
+ * priceUsd from __NEXT_DATA__ per SKU. Discount confirmed 0% so priceUsd(trade) = net cost.
+ *
+ * Writes schumacher_catalog.trade_price (+ price_fetched_at) for rows WHERE retail_price IS NULL.
+ *
+ *   node schu-browserbase-scrape.js --test            # 1 SKU, login + price proof, no DB write
+ *   node schu-browserbase-scrape.js --limit 50 --write
+ */
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+const NM = os.homedir() + '/.claude/skills/browserbase/node_modules';
+require(path.join(NM, 'dotenv')).config({ path: os.homedir() + '/.claude/skills/browserbase/.env' });
+const Browserbase = require(path.join(NM, '@browserbasehq/sdk')).default;
+const { chromium } = require(path.join(NM, 'playwright-core'));
+const { execFileSync } = require('child_process');
+
+const args = process.argv.slice(2);
+const TEST = args.includes('--test');
+const WRITE = args.includes('--write');
+const lIdx = args.indexOf('--limit');
+const LIMIT = TEST ? 1 : (lIdx >= 0 ? parseInt(args[lIdx + 1], 10) : 25);
+const EMAIL = 'info@designerwallcoverings.com', PW = '*Schumacheraccess911*';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36';
+
+function psql(sql) { return execFileSync('psql', ['-At','-F','\t','-d','dw_unified','-c',sql], { encoding:'utf8', maxBuffer:1<<28 }).trim(); }
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+function priceFromNextData(html, sku) {
+  const m = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/); if (!m) return null;
+  const re = new RegExp('"itemNumber":"' + sku + '","priceUsd":([^,}]+)'); const mm = m[1].match(re);
+  return mm ? mm[1].replace(/"/g,'') : null;
+}
+
+(async () => {
+  const skus = psql(`SELECT mfr_sku FROM schumacher_catalog WHERE retail_price IS NULL AND mfr_sku ~ '^[0-9]+$' ORDER BY mfr_sku LIMIT ${LIMIT};`).split('\n').filter(Boolean);
+  console.log(`SKUs to price: ${skus.length} (test=${TEST} write=${WRITE})`);
+  const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
+  const session = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID });
+  console.log('Browserbase session:', session.id, '| live:', session.debuggerUrl || session.debuggerFullscreenUrl || '(see dashboard)');
+  const browser = await chromium.connectOverCDP(session.connectUrl);
+  const ctx = browser.contexts()[0] || await browser.newContext();
+  const page = ctx.pages()[0] || await ctx.newPage();
+  const out = [];
+  try {
+    // ---- login ----
+    await page.goto('https://www.schumacher.com/sign-in', { waitUntil: 'networkidle', timeout: 60000 });
+    await page.waitForTimeout(4000);
+    const inputs = await page.$$eval('input', els => els.map(e => ({ name:e.name, type:e.type, id:e.id, ph:e.placeholder, aria:e.getAttribute('aria-label') })));
+    console.log('sign-in inputs:', JSON.stringify(inputs));
+    try {
+      // type (not fill) so React's onChange fires and the submit button enables
+      await page.click('#email'); await page.locator('#email').pressSequentially(EMAIL, { delay: 30 });
+      await page.click('#password'); await page.locator('#password').pressSequentially(PW, { delay: 30 });
+      await page.waitForTimeout(500);
+      // submit via Enter first (most reliable), then fall back to an enabled button
+      await page.press('#password', 'Enter');
+      await page.waitForTimeout(2500);
+      if (/sign-in/i.test(page.url())) {
+        const btn = page.locator('button:has-text("Sign In"), button:has-text("Sign in"), button:has-text("Log In"), button[type=submit]').first();
+        try { await btn.click({ timeout: 8000 }); } catch {}
+      }
+      await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(()=>{});
+      await page.waitForTimeout(4000);
+    } catch (e) { console.log('LOGIN fill error:', e.message.split('\n')[0]); }
+    const loggedIn = !/sign-in/i.test(page.url());
+    console.log('post-login url:', page.url(), '| loggedIn:', loggedIn);
+
+    // ---- per-SKU price ----
+    for (const sku of skus) {
+      try {
+        await page.goto(`https://www.schumacher.com/catalog/products/${sku}`, { waitUntil:'domcontentloaded', timeout:30000 });
+        await page.waitForTimeout(1800);
+        const html = await page.content();
+        const price = priceFromNextData(html, sku);
+        out.push({ sku, price });
+        console.log(`  ${sku}  priceUsd=${price}`);
+        if (WRITE && price && price !== 'null' && !isNaN(parseFloat(price))) {
+          const p = parseFloat(price);
+          psql(`UPDATE schumacher_catalog SET trade_price=${p}, price_fetched_at=now() WHERE mfr_sku='${sku.replace(/'/g,"''")}';`);
+        }
+        await sleep(1200);
+      } catch (e) { out.push({ sku, err: e.message.split('\n')[0] }); console.log(`  ${sku} ERR ${e.message.split('\n')[0]}`); }
+    }
+  } finally { await browser.close(); }
+  fs.writeFileSync('/tmp/schu-bb-out.json', JSON.stringify(out, null, 1));
+  const got = out.filter(o => o.price && o.price!=='null' && !isNaN(parseFloat(o.price))).length;
+  console.log(`\nDONE: ${got}/${out.length} priced${WRITE?' (written to schumacher_catalog.trade_price)':' (no write — add --write)'}`);
+})();

← 7818ccb4 Quarantine 205 mispriced (needs-reprice tag); Romo 262 + Thi · back to Designer Wallcoverings · Add launchd auto-resume agent (com.steve.dw-pricing-resume): a51fa0c8 →