`scrapers/fetch-browserbase.js`

108 lines
'use strict';
// Cloud Chromium fetch via Browserbase — bypasses Cloudflare / anti-bot for sites like Wallquest.
// Credentials live in ~/.claude/skills/browserbase/.env on Mac2 and /root/.env on Kamatera.

const path = require('path');
const fs = require('fs');
const os = require('os');

// Try local skill env first, then ~/.env, then process.env
function loadCreds() {
  if (process.env.BROWSERBASE_API_KEY && process.env.BROWSERBASE_PROJECT_ID) {
    return {
      apiKey: process.env.BROWSERBASE_API_KEY,
      projectId: process.env.BROWSERBASE_PROJECT_ID,
    };
  }
  const candidates = [
    // Project-local .env (preferred — populated by secrets-manager)
    path.join(__dirname, '..', '.env'),
    path.join(os.homedir(), '.claude/skills/browserbase/.env'),
    path.join(os.homedir(), '.env'),
    '/root/.env',
  ];
  for (const f of candidates) {
    try {
      const txt = fs.readFileSync(f, 'utf8');
      const obj = {};
      txt.split('\n').forEach(line => {
        const m = line.match(/^([A-Z_]+)=(.+)$/);
        if (m) obj[m[1]] = m[2].replace(/^["']|["']$/g, '');
      });
      if (obj.BROWSERBASE_API_KEY && obj.BROWSERBASE_PROJECT_ID) {
        return { apiKey: obj.BROWSERBASE_API_KEY, projectId: obj.BROWSERBASE_PROJECT_ID };
      }
    } catch {}
  }
  return null;
}

async function fetchViaBrowserbase(url) {
  const creds = loadCreds();
  if (!creds) throw new Error('Browserbase creds not found in env or .env files');

  // Lazy-load to avoid pulling chromium when not needed
  let Browserbase, chromium;
  try {
    Browserbase = require('@browserbasehq/sdk').default || require('@browserbasehq/sdk').Browserbase;
    chromium = require('playwright-core').chromium;
  } catch (e) {
    // Fallback: load from the browserbase skill's node_modules
    const skillNm = path.join(os.homedir(), '.claude/skills/browserbase/node_modules');
    try {
      Browserbase = require(path.join(skillNm, '@browserbasehq/sdk')).default;
      chromium = require(path.join(skillNm, 'playwright-core')).chromium;
    } catch (e2) {
      throw new Error('Browserbase SDK not installed: ' + e.message);
    }
  }

  const bb = new Browserbase({ apiKey: creds.apiKey });

  let trackedEnd = () => {};
  try {
    const { trackedSession } = require(path.join(os.homedir(), '.claude/skills/browserbase/track-session.js'));
    const wrapped = await trackedSession(bb, creds.projectId, { app: 'abrams-report', note: url.slice(0, 120) });
    var session = wrapped.session;
    trackedEnd = wrapped.end;
  } catch {
    // helper missing — fall back to raw session creation (no cost log)
    var session = await bb.sessions.create({ projectId: creds.projectId });
  }
  const browser = await chromium.connectOverCDP(session.connectUrl);

  try {
    const ctx = browser.contexts()[0] || (await browser.newContext());
    const page = ctx.pages()[0] || (await ctx.newPage());

    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 });
    // Give Cloudflare a moment to clear the challenge if any
    await page.waitForTimeout(2500);

    // If this is an XML/sitemap URL, Chrome wraps it in pretty-print HTML.
    // Try fetching the raw resource through the page context (bypasses the wrapper
    // and preserves the same-site cookies set by the initial navigation).
    if (/\.xml(\?|$)|sitemap/i.test(url)) {
      try {
        const xml = await page.evaluate(async (u) => {
          const r = await fetch(u, {
            headers: { Accept: 'application/xml, text/xml, */*' },
          });
          return await r.text();
        }, url);
        if (xml && xml.length > 200 && xml.trim().startsWith('<')) {
          return xml;
        }
      } catch {}
    }

    const html = await page.content();
    return html;
  } finally {
    try { await browser.close(); } catch {}
    try { await trackedEnd(); } catch {}
  }
}

module.exports = { fetchViaBrowserbase };