`scripts/clothing-apis/headless-spa.js`

189 lines
/**
 * Headless-SPA adapter — scrape rendered DOM via Browserbase cloud Chrome.
 *
 * For brands whose products live behind a React/Next.js SPA that doesn't
 * expose /products.json or /wp-json (Reformation, Kotn, Pact, Frank+Oak,
 * Burton). The /products page renders client-side, so a static fetch sees
 * only the SPA shell. Browserbase serves a real Chromium that runs the JS
 * and gives us the rendered DOM.
 *
 * Cost: ~$0.001 per page (Steve already pays the Browserbase subscription).
 * Gate this adapter behind the `headless_spa_enabled` config flag so it
 * doesn't surprise-bill if Steve hasn't opted in for a given tick.
 *
 * Per-brand selector dict — each brand needs a small CSS-selector spec to
 * pluck product cards out of the rendered DOM (title, price, image, href).
 * That spec lives in brands.json under `headless_spa[]` and is fragile to
 * vendor redesigns; expect ~quarterly maintenance.
 *
 * Env:
 *   BROWSERBASE_API_KEY     — required
 *   BROWSERBASE_PROJECT_ID  — required
 *
 * The adapter loads them from the secrets-managed skill .env so we don't
 * have to duplicate. See ~/.claude/skills/browserbase/.env.
 */
const fetch = global.fetch || require('node-fetch');
const fs = require('fs');
const path = require('path');

// Load BB creds from the skill .env if not already in process.env.
function loadSkillEnv() {
  if (process.env.BROWSERBASE_API_KEY) return;
  try {
    const envFile = fs.readFileSync(path.join(process.env.HOME, '.claude', 'skills', 'browserbase', '.env'), 'utf8');
    envFile.split('\n').forEach(line => {
      const m = line.match(/^([A-Z_]+)=(.+)$/);
      if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
    });
  } catch {}
}
loadSkillEnv();

const BB_KEY = process.env.BROWSERBASE_API_KEY;
const BB_PROJECT = process.env.BROWSERBASE_PROJECT_ID;
const COST_CENTS_PER_PAGE = Number(process.env.BB_COST_CENTS_PER_PAGE || 0.1);

function available() { return !!(BB_KEY && BB_PROJECT); }

/**
 * Open a Browserbase session, navigate to URL, return rendered HTML.
 * Uses the BB REST API directly (no SDK) to keep the dep surface tiny.
 */
async function fetchRenderedHtml(url, { waitMs = 3000 } = {}) {
  if (!available()) throw new Error('BROWSERBASE_API_KEY or _PROJECT_ID not set');

  // 1. Create a session
  const s = await fetch('https://api.browserbase.com/v1/sessions', {
    method: 'POST',
    headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
    body: JSON.stringify({ projectId: BB_PROJECT }),
  });
  if (!s.ok) throw new Error(`BB session create ${s.status}`);
  const session = await s.json();
  const sessionId = session.id;

  try {
    // 2. Navigate via the BB Connect (CDP) endpoint isn't trivial without
    //    playwright; cleanest path is the Stagehand-less HTTP browser-action
    //    API, but that's also Beta. For v0.1 we use the official live debug
    //    URL pattern + a wait, then fetch the rendered HTML from the
    //    session's recording.
    //
    //    SHORTCUT: Browserbase supports a `goto` action via the Sessions
    //    actions endpoint. Format may shift — wrap in try/catch.
    const r = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/goto`, {
      method: 'POST',
      headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
      body: JSON.stringify({ url, waitUntil: 'networkidle', timeout: 30000 }),
    });
    if (!r.ok) throw new Error(`BB goto ${r.status}: ${await r.text()}`);
    await new Promise(res => setTimeout(res, waitMs));

    // 3. Pull the rendered HTML
    const h = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/html`, {
      headers: { 'x-bb-api-key': BB_KEY },
    });
    if (!h.ok) throw new Error(`BB html ${h.status}: ${await h.text()}`);
    return await h.text();
  } finally {
    // 4. Always close the session — leaving them open burns credit.
    await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}`, {
      method: 'PATCH',
      headers: { 'x-bb-api-key': BB_KEY, 'content-type': 'application/json' },
      body: JSON.stringify({ status: 'REQUEST_RELEASE' }),
    }).catch(() => {});
  }
}

/**
 * Parse rendered DOM with cheerio per a brand's selector spec.
 *
 * spec shape (in brands.json):
 *   {
 *     id, name, domain, products_path, sustain_tier, pro_grade,
 *     selectors: {
 *       card:  '.product-card',
 *       title: '.product-card__title',
 *       price: '.product-card__price',
 *       image: 'img',
 *       link:  'a'
 *     }
 *   }
 */
function parseProducts(html, spec) {
  let cheerio;
  try { cheerio = require('cheerio'); }
  catch { throw new Error('cheerio not installed — run `npm i cheerio`'); }
  const $ = cheerio.load(html);
  const out = [];
  $(spec.selectors.card).each((i, el) => {
    const $el = $(el);
    const title = $el.find(spec.selectors.title).first().text().trim();
    const priceRaw = $el.find(spec.selectors.price).first().text().trim();
    const m = priceRaw.match(/\$?\s*([0-9,]+\.?\d{0,2})/);
    const price_cents = m ? Math.round(Number(m[1].replace(/,/g, '')) * 100) : null;
    const imgEl = $el.find(spec.selectors.image).first();
    const image = imgEl.attr('src') || imgEl.attr('data-src') || imgEl.attr('srcset')?.split(',')[0]?.trim().split(' ')[0];
    const href = $el.find(spec.selectors.link).first().attr('href') || $el.attr('href');
    if (!title || !image || !price_cents) return;
    out.push({
      external_id: `spa:${spec.domain}:${Buffer.from(href || title).toString('base64url').slice(0, 32)}`,
      title: title.slice(0, 200),
      price_cents,
      image_url: image.startsWith('//') ? 'https:' + image : (image.startsWith('http') ? image : `https://${spec.domain}${image}`),
      product_url: href ? (href.startsWith('http') ? href : `https://${spec.domain}${href}`) : null,
    });
  });
  return out;
}

async function importBrand(db, brandEntry) {
  if (!available()) return { brand: brandEntry.name, ok: false, skipped: 'BB creds not configured' };
  const { name, domain, products_path, sustain_tier, pro_grade, selectors } = brandEntry;
  if (!selectors) return { brand: name, ok: false, skipped: 'no selectors spec — needs hand-craft per brand' };

  const t0 = Date.now();
  const url = `https://${domain}${products_path || '/products'}`;
  let html;
  try { html = await fetchRenderedHtml(url); }
  catch (e) { return { brand: name, ok: false, err: e.message, ms: Date.now() - t0 }; }

  const parsed = parseProducts(html, brandEntry);
  const insert = db.prepare(`
    INSERT OR IGNORE INTO items
      (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
    VALUES
      (@source, @external_id, @title, @brand, 'top', @price_cents, 'USD', @image_url, @product_url, @tags, @pro_grade)
  `);
  let imported = 0;
  const tx = db.transaction(() => {
    for (const p of parsed) {
      const res = insert.run({
        source: `spa:${domain}`,
        external_id: p.external_id,
        title: p.title,
        brand: name,
        price_cents: p.price_cents,
        image_url: p.image_url,
        product_url: p.product_url,
        tags: JSON.stringify([]),
        pro_grade,
      });
      if (res.changes > 0) imported++;
    }
  });
  tx();

  // Log the BB cost.
  try {
    db.prepare(`INSERT INTO provider_costs (provider, kind, cost_cents, meta)
                VALUES ('browserbase', 'catalog-scrape', ?, ?)`)
      .run(COST_CENTS_PER_PAGE, JSON.stringify({ domain, fetched: parsed.length }));
  } catch {}

  return { brand: name, domain, ok: true, fetched: parsed.length, imported, ms: Date.now() - t0, cost_cents: COST_CENTS_PER_PAGE };
}

module.exports = { importBrand, available, fetchRenderedHtml };