`scrapers/fetch-local-browser.js`

86 lines
'use strict';
// Local real-Chrome fetch — $0 replacement for paid Browserbase (TK-10655, prefer-local rule).
// Drives the installed Google Chrome via Playwright with a PERSISTENT profile so Cloudflare's
// cf_clearance cookie survives between sources/runs. A real, residential-IP Chrome clears the
// Cloudflare "managed challenge" that bounces headless / datacenter browsers — the exact reason
// Browserbase was used for sites like Wallquest. Same contract as the old fetchViaBrowserbase(url):
//   fetchViaLocalBrowser(url) -> Promise<html-or-xml string>
//
// Headed by default: CF's non-interactive challenge only auto-clears for a real visible Chrome.
// Set ABR_BROWSER_HEADLESS=1 to force headless (works for non-CF sites; will miss CF-hard ones).
const path = require('path');
const os = require('os');
const fs = require('fs');

const HEADLESS = process.env.ABR_BROWSER_HEADLESS === '1';
const PROFILE = process.env.ABR_CHROME_PROFILE || path.join(os.homedir(), '.cache', 'abrams-report-chrome');

// Resolve playwright-core: project-local first, then the browserbase skill's copy (legacy),
// then the globally-installed full playwright. Mirrors the old lazy-load fallback chain.
function loadChromium() {
  const tries = [
    () => require('playwright-core').chromium,
    () => require(path.join(os.homedir(), '.claude/skills/browserbase/node_modules/playwright-core')).chromium,
    () => require(path.join(os.homedir(), '.npm-global/lib/node_modules/playwright')).chromium,
  ];
  for (const t of tries) {
    try { const c = t(); if (c) return c; } catch { /* try next */ }
  }
  throw new Error('playwright-core / playwright not found for local Chrome fetch');
}

function isChallenge(html) {
  return /just a moment|challenge-platform|cf-browser-verification|enable javascript and cookies/i.test(html || '');
}

// Poll page content until the Cloudflare managed challenge clears (or give up after ~25s).
async function clearChallenge(page) {
  for (let i = 0; i < 10; i++) {
    const c = await page.content().catch(() => '');
    if (!isChallenge(c)) return true;
    await page.waitForTimeout(2500);
  }
  return false;
}

async function fetchViaLocalBrowser(url) {
  const chromium = loadChromium();
  try { fs.mkdirSync(PROFILE, { recursive: true }); } catch { /* ignore */ }

  const ctx = await chromium.launchPersistentContext(PROFILE, {
    channel: 'chrome',
    headless: HEADLESS,
    viewport: { width: 1280, height: 900 },
    args: ['--no-first-run', '--no-default-browser-check', '--disable-blink-features=AutomationControlled'],
  });

  try {
    const page = ctx.pages()[0] || (await ctx.newPage());
    page.setDefaultTimeout(45000);

    // For XML/sitemap URLs, a bare navigation to the .xml returns Cloudflare's challenge page
    // (which never auto-clears on a raw XML response). So clear CF on the site's HTML origin FIRST,
    // then navigate to the sitemap and return the RAW HTTP response body — page.goto() gives us the
    // real bytes before Chrome's XML pretty-print viewer transforms the DOM. (An out-of-page
    // request.get() can't be used: CF binds cf_clearance to the browser's TLS fingerprint.)
    if (/\.xml(\?|$)|sitemap/i.test(url)) {
      const origin = new URL(url).origin + '/';
      await page.goto(origin, { waitUntil: 'domcontentloaded' }).catch(() => {});
      await clearChallenge(page);
      const resp = await page.goto(url, { waitUntil: 'domcontentloaded' });
      const body = resp ? await resp.text() : '';
      if (body && body.trim().startsWith('<')) return body;
      return await page.content();
    }

    await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => {});
    await clearChallenge(page);
    await page.waitForTimeout(2000); // let late anti-bot JS settle
    return await page.content();
  } finally {
    try { await ctx.close(); } catch { /* ignore */ }
  }
}

module.exports = { fetchViaLocalBrowser };