← back to Abrams Report
abrams-report: reroute anti-bot scrapes off Browserbase to local real Chrome ($0)
4c1bd9558b9b92f8ff9dc5c101ce6e2254195842 · 2026-08-18 11:41:33 -0700 · Steve Abrams
TK-10655 (Steve-approved): the 2 Cloudflare-protected sources (Schumacher blog,
Wallquest sitemap) went through paid Browserbase (~$46.80 all-time, the #1 spend
line). Replace with scrapers/fetch-local-browser.js — Playwright driving the
installed Google Chrome via a persistent profile (residential IP + real-Chrome
fingerprint clears CF's managed challenge where headless/datacenter is bounced).
Raw XML captured from page.goto().text() before Chrome's viewer transforms it.
Verified live: Schumacher -> 13 blog links, Wallquest -> 394 sitemap URLs, $0.
Deleted fetch-browserbase.js + dropped @browserbasehq/sdk dep (now unused).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M package-lock.jsonM package.jsonD scrapers/fetch-browserbase.jsA scrapers/fetch-local-browser.jsM scrapers/scrape-html.jsM scrapers/sources.js
Diff
commit 4c1bd9558b9b92f8ff9dc5c101ce6e2254195842
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Aug 18 11:41:33 2026 -0700
abrams-report: reroute anti-bot scrapes off Browserbase to local real Chrome ($0)
TK-10655 (Steve-approved): the 2 Cloudflare-protected sources (Schumacher blog,
Wallquest sitemap) went through paid Browserbase (~$46.80 all-time, the #1 spend
line). Replace with scrapers/fetch-local-browser.js — Playwright driving the
installed Google Chrome via a persistent profile (residential IP + real-Chrome
fingerprint clears CF's managed challenge where headless/datacenter is bounced).
Raw XML captured from page.goto().text() before Chrome's viewer transforms it.
Verified live: Schumacher -> 13 blog links, Wallquest -> 394 sitemap URLs, $0.
Deleted fetch-browserbase.js + dropped @browserbasehq/sdk dep (now unused).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
package-lock.json | 213 +---------------------------------------
package.json | 9 +-
scrapers/fetch-browserbase.js | 107 --------------------
scrapers/fetch-local-browser.js | 85 ++++++++++++++++
scrapers/scrape-html.js | 15 +--
scrapers/sources.js | 7 +-
6 files changed, 106 insertions(+), 330 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index 296ab3b..22972c0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,61 +8,15 @@
"name": "abrams-report",
"version": "1.0.0",
"dependencies": {
- "@browserbasehq/sdk": "^2.0.0",
"better-sqlite3": "^9.4.3",
"cheerio": "^1.0.0-rc.12",
"express": "^4.18.2",
"node-cron": "^3.0.3",
"node-fetch": "^2.7.0",
- "playwright-core": "^1.49.0",
+ "playwright-core": "^1.62.1",
"rss-parser": "^3.13.0"
}
},
- "node_modules/@browserbasehq/sdk": {
- "version": "2.10.0",
- "resolved": "https://registry.npmjs.org/@browserbasehq/sdk/-/sdk-2.10.0.tgz",
- "integrity": "sha512-pOL4yW8P8AI2+N5y6zEP6XXKqIXtYyKunr1JXppqQDOyKLxxvZEDqQCHJXWUzqgx3R1tGWpn7m9AjXN7MeYInA==",
- "dependencies": {
- "@types/node": "^18.11.18",
- "@types/node-fetch": "^2.6.4",
- "abort-controller": "^3.0.0",
- "agentkeepalive": "^4.2.1",
- "form-data-encoder": "1.7.2",
- "formdata-node": "^4.3.2",
- "node-fetch": "^2.6.7"
- }
- },
- "node_modules/@types/node": {
- "version": "18.19.130",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
- "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
- "license": "MIT",
- "dependencies": {
- "undici-types": "~5.26.4"
- }
- },
- "node_modules/@types/node-fetch": {
- "version": "2.6.13",
- "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz",
- "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==",
- "license": "MIT",
- "dependencies": {
- "@types/node": "*",
- "form-data": "^4.0.4"
- }
- },
- "node_modules/abort-controller": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
- "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
- "license": "MIT",
- "dependencies": {
- "event-target-shim": "^5.0.0"
- },
- "engines": {
- "node": ">=6.5"
- }
- },
"node_modules/accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -76,30 +30,12 @@
"node": ">= 0.6"
}
},
- "node_modules/agentkeepalive": {
- "version": "4.6.0",
- "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
- "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",
- "license": "MIT",
- "dependencies": {
- "humanize-ms": "^1.2.1"
- },
- "engines": {
- "node": ">= 8.0.0"
- }
- },
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
"license": "MIT"
},
- "node_modules/asynckit": {
- "version": "0.4.0",
- "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
- "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
- "license": "MIT"
- },
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -303,18 +239,6 @@
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"license": "ISC"
},
- "node_modules/combined-stream": {
- "version": "1.0.8",
- "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
- "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
- "license": "MIT",
- "dependencies": {
- "delayed-stream": "~1.0.0"
- },
- "engines": {
- "node": ">= 0.8"
- }
- },
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -412,15 +336,6 @@
"node": ">=4.0.0"
}
},
- "node_modules/delayed-stream": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
- "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
- "license": "MIT",
- "engines": {
- "node": ">=0.4.0"
- }
- },
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -597,21 +512,6 @@
"node": ">= 0.4"
}
},
- "node_modules/es-set-tostringtag": {
- "version": "2.1.0",
- "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
- "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
- "license": "MIT",
- "dependencies": {
- "es-errors": "^1.3.0",
- "get-intrinsic": "^1.2.6",
- "has-tostringtag": "^1.0.2",
- "hasown": "^2.0.2"
- },
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -627,15 +527,6 @@
"node": ">= 0.6"
}
},
- "node_modules/event-target-shim": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
- "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
- "license": "MIT",
- "engines": {
- "node": ">=6"
- }
- },
"node_modules/expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -715,41 +606,6 @@
"node": ">= 0.8"
}
},
- "node_modules/form-data": {
- "version": "4.0.5",
- "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
- "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
- "license": "MIT",
- "dependencies": {
- "asynckit": "^0.4.0",
- "combined-stream": "^1.0.8",
- "es-set-tostringtag": "^2.1.0",
- "hasown": "^2.0.2",
- "mime-types": "^2.1.12"
- },
- "engines": {
- "node": ">= 6"
- }
- },
- "node_modules/form-data-encoder": {
- "version": "1.7.2",
- "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
- "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
- "license": "MIT"
- },
- "node_modules/formdata-node": {
- "version": "4.4.1",
- "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
- "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
- "license": "MIT",
- "dependencies": {
- "node-domexception": "1.0.0",
- "web-streams-polyfill": "4.0.0-beta.3"
- },
- "engines": {
- "node": ">= 12.20"
- }
- },
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -850,21 +706,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/has-tostringtag": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
- "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
- "license": "MIT",
- "dependencies": {
- "has-symbols": "^1.0.3"
- },
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
"node_modules/hasown": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz",
@@ -928,15 +769,6 @@
"url": "https://opencollective.com/express"
}
},
- "node_modules/humanize-ms": {
- "version": "1.2.1",
- "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
- "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
- "license": "MIT",
- "dependencies": {
- "ms": "^2.0.0"
- }
- },
"node_modules/iconv-lite": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@@ -1131,26 +963,6 @@
"node": ">=6.0.0"
}
},
- "node_modules/node-domexception": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
- "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
- "deprecated": "Use your platform's native DOMException instead",
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/jimmywarting"
- },
- {
- "type": "github",
- "url": "https://paypal.me/jimmywarting"
- }
- ],
- "license": "MIT",
- "engines": {
- "node": ">=10.5.0"
- }
- },
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -1281,15 +1093,15 @@
"license": "MIT"
},
"node_modules/playwright-core": {
- "version": "1.60.0",
- "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz",
- "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==",
+ "version": "1.62.1",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
+ "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
- "node": ">=18"
+ "node": ">=20"
}
},
"node_modules/prebuild-install": {
@@ -1760,12 +1572,6 @@
"node": ">=20.18.1"
}
},
- "node_modules/undici-types": {
- "version": "5.26.5",
- "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
- "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
- "license": "MIT"
- },
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -1809,15 +1615,6 @@
"node": ">= 0.8"
}
},
- "node_modules/web-streams-polyfill": {
- "version": "4.0.0-beta.3",
- "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
- "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
- "license": "MIT",
- "engines": {
- "node": ">= 14"
- }
- },
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
diff --git a/package.json b/package.json
index 33f9aca..e93da1d 100644
--- a/package.json
+++ b/package.json
@@ -9,13 +9,12 @@
"scrape:wallpaper": "node scrapers/scrape-wallpaper.js"
},
"dependencies": {
- "express": "^4.18.2",
"better-sqlite3": "^9.4.3",
- "node-fetch": "^2.7.0",
- "rss-parser": "^3.13.0",
"cheerio": "^1.0.0-rc.12",
+ "express": "^4.18.2",
"node-cron": "^3.0.3",
- "@browserbasehq/sdk": "^2.0.0",
- "playwright-core": "^1.49.0"
+ "node-fetch": "^2.7.0",
+ "playwright-core": "^1.62.1",
+ "rss-parser": "^3.13.0"
}
}
diff --git a/scrapers/fetch-browserbase.js b/scrapers/fetch-browserbase.js
deleted file mode 100644
index d2b5706..0000000
--- a/scrapers/fetch-browserbase.js
+++ /dev/null
@@ -1,107 +0,0 @@
-'use strict';
-// Cloud Chromium fetch via Browserbase — bypasses Cloudflare / anti-bot for sites like Wallquest.
-// Credentials live in ~/.claude/skills/browserbase/.env on Mac2 and /root/.env on Kamatera.
-
-const path = require('path');
-const fs = require('fs');
-const os = require('os');
-
-// Try local skill env first, then ~/.env, then process.env
-function loadCreds() {
- if (process.env.BROWSERBASE_API_KEY && process.env.BROWSERBASE_PROJECT_ID) {
- return {
- apiKey: process.env.BROWSERBASE_API_KEY,
- projectId: process.env.BROWSERBASE_PROJECT_ID,
- };
- }
- const candidates = [
- // Project-local .env (preferred — populated by secrets-manager)
- path.join(__dirname, '..', '.env'),
- path.join(os.homedir(), '.claude/skills/browserbase/.env'),
- path.join(os.homedir(), '.env'),
- '/root/.env',
- ];
- for (const f of candidates) {
- try {
- const txt = fs.readFileSync(f, 'utf8');
- const obj = {};
- txt.split('\n').forEach(line => {
- const m = line.match(/^([A-Z_]+)=(.+)$/);
- if (m) obj[m[1]] = m[2].replace(/^["']|["']$/g, '');
- });
- if (obj.BROWSERBASE_API_KEY && obj.BROWSERBASE_PROJECT_ID) {
- return { apiKey: obj.BROWSERBASE_API_KEY, projectId: obj.BROWSERBASE_PROJECT_ID };
- }
- } catch {}
- }
- return null;
-}
-
-async function fetchViaBrowserbase(url) {
- const creds = loadCreds();
- if (!creds) throw new Error('Browserbase creds not found in env or .env files');
-
- // Lazy-load to avoid pulling chromium when not needed
- let Browserbase, chromium;
- try {
- Browserbase = require('@browserbasehq/sdk').default || require('@browserbasehq/sdk').Browserbase;
- chromium = require('playwright-core').chromium;
- } catch (e) {
- // Fallback: load from the browserbase skill's node_modules
- const skillNm = path.join(os.homedir(), '.claude/skills/browserbase/node_modules');
- try {
- Browserbase = require(path.join(skillNm, '@browserbasehq/sdk')).default;
- chromium = require(path.join(skillNm, 'playwright-core')).chromium;
- } catch (e2) {
- throw new Error('Browserbase SDK not installed: ' + e.message);
- }
- }
-
- const bb = new Browserbase({ apiKey: creds.apiKey });
-
- let trackedEnd = () => {};
- try {
- const { trackedSession } = require(path.join(os.homedir(), '.claude/skills/browserbase/track-session.js'));
- const wrapped = await trackedSession(bb, creds.projectId, { app: 'abrams-report', note: url.slice(0, 120) });
- var session = wrapped.session;
- trackedEnd = wrapped.end;
- } catch {
- // helper missing — fall back to raw session creation (no cost log)
- var session = await bb.sessions.create({ projectId: creds.projectId });
- }
- const browser = await chromium.connectOverCDP(session.connectUrl);
-
- try {
- const ctx = browser.contexts()[0] || (await browser.newContext());
- const page = ctx.pages()[0] || (await ctx.newPage());
-
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 });
- // Give Cloudflare a moment to clear the challenge if any
- await page.waitForTimeout(2500);
-
- // If this is an XML/sitemap URL, Chrome wraps it in pretty-print HTML.
- // Try fetching the raw resource through the page context (bypasses the wrapper
- // and preserves the same-site cookies set by the initial navigation).
- if (/\.xml(\?|$)|sitemap/i.test(url)) {
- try {
- const xml = await page.evaluate(async (u) => {
- const r = await fetch(u, {
- headers: { Accept: 'application/xml, text/xml, */*' },
- });
- return await r.text();
- }, url);
- if (xml && xml.length > 200 && xml.trim().startsWith('<')) {
- return xml;
- }
- } catch {}
- }
-
- const html = await page.content();
- return html;
- } finally {
- try { await browser.close(); } catch {}
- try { await trackedEnd(); } catch {}
- }
-}
-
-module.exports = { fetchViaBrowserbase };
diff --git a/scrapers/fetch-local-browser.js b/scrapers/fetch-local-browser.js
new file mode 100644
index 0000000..32bbb6b
--- /dev/null
+++ b/scrapers/fetch-local-browser.js
@@ -0,0 +1,85 @@
+'use strict';
+// Local real-Chrome fetch — $0 replacement for paid Browserbase (TK-10655, prefer-local rule).
+// Drives the installed Google Chrome via Playwright with a PERSISTENT profile so Cloudflare's
+// cf_clearance cookie survives between sources/runs. A real, residential-IP Chrome clears the
+// Cloudflare "managed challenge" that bounces headless / datacenter browsers — the exact reason
+// Browserbase was used for sites like Wallquest. Same contract as the old fetchViaBrowserbase(url):
+// fetchViaLocalBrowser(url) -> Promise<html-or-xml string>
+//
+// Headed by default: CF's non-interactive challenge only auto-clears for a real visible Chrome.
+// Set ABR_BROWSER_HEADLESS=1 to force headless (works for non-CF sites; will miss CF-hard ones).
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+const HEADLESS = process.env.ABR_BROWSER_HEADLESS === '1';
+const PROFILE = process.env.ABR_CHROME_PROFILE || path.join(os.homedir(), '.cache', 'abrams-report-chrome');
+
+// Resolve playwright-core: project-local first, then the browserbase skill's copy (legacy),
+// then the globally-installed full playwright. Mirrors the old lazy-load fallback chain.
+function loadChromium() {
+ const tries = [
+ () => require('playwright-core').chromium,
+ () => require(path.join(os.homedir(), '.claude/skills/browserbase/node_modules/playwright-core')).chromium,
+ () => require(path.join(os.homedir(), '.npm-global/lib/node_modules/playwright')).chromium,
+ ];
+ for (const t of tries) {
+ try { const c = t(); if (c) return c; } catch { /* try next */ }
+ }
+ throw new Error('playwright-core / playwright not found for local Chrome fetch');
+}
+
+function isChallenge(html) {
+ return /just a moment|challenge-platform|cf-browser-verification|enable javascript and cookies/i.test(html || '');
+}
+
+// Poll page content until the Cloudflare managed challenge clears (or give up after ~25s).
+async function clearChallenge(page) {
+ for (let i = 0; i < 10; i++) {
+ const c = await page.content().catch(() => '');
+ if (!isChallenge(c)) return true;
+ await page.waitForTimeout(2500);
+ }
+ return false;
+}
+
+async function fetchViaLocalBrowser(url) {
+ const chromium = loadChromium();
+ try { fs.mkdirSync(PROFILE, { recursive: true }); } catch { /* ignore */ }
+
+ const ctx = await chromium.launchPersistentContext(PROFILE, {
+ channel: 'chrome',
+ headless: HEADLESS,
+ viewport: { width: 1280, height: 900 },
+ args: ['--no-first-run', '--no-default-browser-check', '--disable-blink-features=AutomationControlled'],
+ });
+
+ try {
+ const page = ctx.pages()[0] || (await ctx.newPage());
+ page.setDefaultTimeout(45000);
+
+ // For XML/sitemap URLs, a bare navigation to the .xml returns Cloudflare's challenge page
+ // (which never auto-clears on a raw XML response). So clear CF on the site's HTML origin FIRST,
+ // then navigate to the sitemap and return the RAW HTTP response body — page.goto() gives us the
+ // real bytes before Chrome's XML pretty-print viewer transforms the DOM. (An out-of-page
+ // request.get() can't be used: CF binds cf_clearance to the browser's TLS fingerprint.)
+ if (/\.xml(\?|$)|sitemap/i.test(url)) {
+ const origin = new URL(url).origin + '/';
+ await page.goto(origin, { waitUntil: 'domcontentloaded' }).catch(() => {});
+ await clearChallenge(page);
+ const resp = await page.goto(url, { waitUntil: 'domcontentloaded' });
+ const body = resp ? await resp.text() : '';
+ if (body && body.trim().startsWith('<')) return body;
+ return await page.content();
+ }
+
+ await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => {});
+ await clearChallenge(page);
+ await page.waitForTimeout(2000); // let late anti-bot JS settle
+ return await page.content();
+ } finally {
+ try { await ctx.close(); } catch { /* ignore */ }
+ }
+}
+
+module.exports = { fetchViaLocalBrowser };
diff --git a/scrapers/scrape-html.js b/scrapers/scrape-html.js
index 737abe9..0a1efa1 100644
--- a/scrapers/scrape-html.js
+++ b/scrapers/scrape-html.js
@@ -4,7 +4,7 @@ const cheerio = require('cheerio');
const crypto = require('crypto');
const { upsertHeadline, logScrape } = require('./db');
const { SCRAPE_SOURCES } = require('./sources');
-const { fetchViaBrowserbase } = require('./fetch-browserbase');
+const { fetchViaLocalBrowser } = require('./fetch-local-browser');
// Use a real browser UA — many sites 403/404 a bot UA
const BROWSER_UA =
@@ -63,9 +63,10 @@ function slugToTitle(url) {
}
}
-async function fetchHtml(url, useBrowserbase = false) {
- if (useBrowserbase) {
- return fetchViaBrowserbase(url);
+async function fetchHtml(url, useBrowser = false) {
+ if (useBrowser) {
+ // Local real Chrome ($0) — replaces paid Browserbase for anti-bot / Cloudflare sources (TK-10655).
+ return fetchViaLocalBrowser(url);
}
const res = await fetch(url, FETCH_OPTS);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
@@ -167,7 +168,7 @@ async function scrapeSite(src) {
let errorMsg = null;
try {
- const html = await fetchHtml(src.url, !!src.browserbase);
+ const html = await fetchHtml(src.url, !!(src.browser || src.browserbase));
if (src.parser === 'sitemap') {
items = parseSitemapXml(html, src);
@@ -214,8 +215,8 @@ async function scrapeAllHtml(category = null) {
const count = await scrapeSite(src);
console.log(`[html] ${src.name}: +${count}`);
results.push({ source: src.key, added: count });
- // Polite delay; Browserbase sources need a longer beat
- await new Promise(r => setTimeout(r, src.browserbase ? 1500 : 500));
+ // Polite delay; real-browser sources need a longer beat
+ await new Promise(r => setTimeout(r, (src.browser || src.browserbase) ? 1500 : 500));
}
return results;
}
diff --git a/scrapers/sources.js b/scrapers/sources.js
index 76cfc07..ebfbf7c 100644
--- a/scrapers/sources.js
+++ b/scrapers/sources.js
@@ -183,7 +183,8 @@ const SCRAPE_SOURCES = [
// --- Wallpaper Houses ---
// Each entry has: url, baseUrl, selector, linkFilter (optional reject regex via excludeFilter)
// Some use `parser: 'sitemap'` to read a sitemap.xml instead of HTML
- // `browserbase: true` routes through Browserbase cloud chromium (anti-bot bypass)
+ // `browserbase: true` (legacy) / `browser: true` route through LOCAL real Chrome for anti-bot
+ // (Cloudflare) bypass — $0, residential IP. See scrapers/fetch-local-browser.js (TK-10655).
{
key: 'schumacher',
name: 'Schumacher',
@@ -193,8 +194,8 @@ const SCRAPE_SOURCES = [
linkFilter: /\/blog\/[a-z0-9-]+\/?$/i,
excludeFilter: /\/blog\/(category|author|search|tag|page)\//i,
baseUrl: 'https://schumacher.com',
- // CloudFront 403s data-center IPs (Kamatera) but allows residential
- // (Mac2). Force through Browserbase so prod cron works too.
+ // CloudFront 403s data-center IPs but allows residential (Mac2). The scrape
+ // runs on Mac2, so route through LOCAL real Chrome (residential IP, $0).
browserbase: true,
},
{
← 4cc35d4 auto-save: 2026-07-30T15:49:17 (1 files) — data/headlines.db
·
back to Abrams Report
·
abrams-report: bump better-sqlite3 9.4.3 -> 13.0.3 for node e85230b →