[object Object]

← back to Whatsmystyle

tick 9 follow-up: ship demo video + Browserbase escalation wiring

ae62717b0d0613294563f5def462e089d7cfb24b · 2026-05-11 22:20:07 -0700 · Steve Abrams

Demo video (per 'make a video for every shipped app' standing rule):
- app-demo-video skill ran against http://127.0.0.1:9777
- Output: ~/Videos/shipped-apps/whatsmystyle-v0.1-2026-05-12.mp4
  (11.7s, 0.49MB, 4 auto-discovered steps)
- Will surface in the video-gallery viewer on next refresh (the launchd
  publish-shipped-apps watcher will pick it up).

Browserbase escalation (env-gated, no metered calls yet):
- scripts/crawl-catalog.js routes through cloud Chromium when
  USE_BROWSERBASE=1 + BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID set.
  Creates session, connects via CDP (puppeteer-core), navigates, grabs
  rendered HTML, then explicitly releases the session so we stop billing
  the minute as soon as the page is captured.
- Falls back to plain fetch when env unset — current default, no cost.
- .env.example documents the keys with a billing warning.

Files touched

Diff

commit ae62717b0d0613294563f5def462e089d7cfb24b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon May 11 22:20:07 2026 -0700

    tick 9 follow-up: ship demo video + Browserbase escalation wiring
    
    Demo video (per 'make a video for every shipped app' standing rule):
    - app-demo-video skill ran against http://127.0.0.1:9777
    - Output: ~/Videos/shipped-apps/whatsmystyle-v0.1-2026-05-12.mp4
      (11.7s, 0.49MB, 4 auto-discovered steps)
    - Will surface in the video-gallery viewer on next refresh (the launchd
      publish-shipped-apps watcher will pick it up).
    
    Browserbase escalation (env-gated, no metered calls yet):
    - scripts/crawl-catalog.js routes through cloud Chromium when
      USE_BROWSERBASE=1 + BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID set.
      Creates session, connects via CDP (puppeteer-core), navigates, grabs
      rendered HTML, then explicitly releases the session so we stop billing
      the minute as soon as the page is captured.
    - Falls back to plain fetch when env unset — current default, no cost.
    - .env.example documents the keys with a billing warning.
---
 data/waitlist.csv        |   1 +
 data/whatsmystyle.db-wal | Bin 2319592 -> 2521472 bytes
 scripts/crawl-catalog.js |  49 ++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/data/waitlist.csv b/data/waitlist.csv
index f51ab50..85d976b 100644
--- a/data/waitlist.csv
+++ b/data/waitlist.csv
@@ -10,3 +10,4 @@ e2e-1778560665834@example.com,e2e,2026-05-12T04:37:45.837Z
 e2e-1778560882575@example.com,e2e,2026-05-12T04:41:22.586Z
 e2e-1778561154050@example.com,e2e,2026-05-12T04:45:54.051Z
 e2e-1778562484174@example.com,e2e,2026-05-12T05:08:04.183Z
+e2e-1778563130826@example.com,e2e,2026-05-12T05:18:50.826Z
diff --git a/data/whatsmystyle.db-wal b/data/whatsmystyle.db-wal
index 94473dd..bf12908 100644
Binary files a/data/whatsmystyle.db-wal and b/data/whatsmystyle.db-wal differ
diff --git a/scripts/crawl-catalog.js b/scripts/crawl-catalog.js
index bdad61d..77f2181 100644
--- a/scripts/crawl-catalog.js
+++ b/scripts/crawl-catalog.js
@@ -40,6 +40,44 @@ const MAX_PER_VENDOR = 200;
 // area. Run with BROWSERBASE_API_KEY set + USE_BROWSERBASE=1 for live crawls.
 
 const lastHit = new Map();
+
+// Browserbase escalation — gated on USE_BROWSERBASE=1 + BROWSERBASE_API_KEY.
+// When unset, plain fetch (which 403s on RealReal/Poshmark anti-bot walls but
+// works on most low-friction vendor sites). When set, route through cloud
+// Chromium to defeat anti-bot. Per Steve's 'Browserbase before any other
+// headless browser' rule, this is the only authorized escalation path.
+async function browserbaseFetch(url) {
+  const KEY = process.env.BROWSERBASE_API_KEY;
+  const PROJECT = process.env.BROWSERBASE_PROJECT_ID;
+  if (!KEY || !PROJECT) throw new Error('BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID required');
+  // 1) create a session
+  const sRes = await fetch('https://api.browserbase.com/v1/sessions', {
+    method: 'POST',
+    headers: { 'x-bb-api-key': KEY, 'content-type': 'application/json' },
+    body: JSON.stringify({ projectId: PROJECT }),
+  });
+  if (!sRes.ok) throw new Error(`Browserbase session ${sRes.status}: ${await sRes.text()}`);
+  const { id, connectUrl } = await sRes.json();
+  // 2) connect via puppeteer/playwright over CDP, navigate, get HTML
+  let html = '';
+  try {
+    const puppeteer = require('puppeteer-core');
+    const browser = await puppeteer.connect({ browserWSEndpoint: connectUrl });
+    const page = await browser.newPage();
+    await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36');
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await page.waitForTimeout(2000);  // let JSON-LD scripts settle
+    html = await page.content();
+    await browser.close();
+  } catch (e) {
+    throw new Error(`Browserbase nav failed: ${e.message}`);
+  } finally {
+    // 3) close the session (so we stop billing the minute)
+    try { await fetch(`https://api.browserbase.com/v1/sessions/${id}`, { method: 'POST', headers: { 'x-bb-api-key': KEY, 'content-type': 'application/json' }, body: JSON.stringify({ status: 'REQUEST_RELEASE' }) }); } catch {}
+  }
+  return html;
+}
+
 async function politeFetch(url) {
   const host = new URL(url).host;
   const since = Date.now() - (lastHit.get(host) || 0);
@@ -52,9 +90,14 @@ async function politeFetch(url) {
       return { html: fs.readFileSync(cacheKey, 'utf8'), cached: true };
     }
   } catch {}
-  const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, timeout: 30000 });
-  if (!r.ok) throw new Error(`${url} → HTTP ${r.status}`);
-  const html = await r.text();
+  let html;
+  if (process.env.USE_BROWSERBASE === '1' && process.env.BROWSERBASE_API_KEY) {
+    html = await browserbaseFetch(url);
+  } else {
+    const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, timeout: 30000 });
+    if (!r.ok) throw new Error(`${url} → HTTP ${r.status}`);
+    html = await r.text();
+  }
   fs.writeFileSync(cacheKey, html);
   return { html, cached: false };
 }

← c7a376c tick 9: prove the workers — taste-history + closet-vision in  ·  back to Whatsmystyle  ·  tick 10 partial: live puppet camera + tryon video parked 69c2732 →