← back to Ventura Corridor
src/crawl/front_page.ts
363 lines
/**
* Front-page crawler — Phase ii.
*
* For every business with a website on the Ventura corridor, visit the page
* with Playwright, capture metadata + screenshot + HTML, insert a row into
* front_page_audits.
*
* Polite: concurrency 4, 15s navigation timeout, ignore SSL errors.
*
* Run:
* npm run crawl
* tsx src/crawl/front_page.ts -- --limit=20 # smoke test
* tsx src/crawl/front_page.ts -- --rescore # only rows missing audits
*/
import 'dotenv/config';
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { chromium, type Browser } from 'playwright';
import { pool, query } from '../../db/pool.ts';
const ROOT = new URL('../..', import.meta.url).pathname;
const SCREEN_DIR = join(ROOT, 'data', 'screenshots');
const RAW_DIR = join(ROOT, 'data', 'raw');
mkdirSync(SCREEN_DIR, { recursive: true });
mkdirSync(RAW_DIR, { recursive: true });
const CONCURRENCY = parseInt(process.env.CRAWL_CONCURRENCY || '4', 10);
const TIMEOUT_MS = parseInt(process.env.CRAWL_TIMEOUT_MS || '15000', 10);
interface BusinessRow {
id: number;
name: string;
website: string;
}
async function pickQueue(opts: { limit: number; rescoreOnly: boolean; retryFailed: boolean }): Promise<BusinessRow[]> {
const where = ['b.on_corridor', "b.website IS NOT NULL", "b.website ~ '^https?://'"];
if (opts.rescoreOnly) {
where.push(`NOT EXISTS (SELECT 1 FROM front_page_audits a WHERE a.business_id = b.id)`);
}
if (opts.retryFailed) {
// Re-pick rows that have an audit row but with a recoverable error.
// DNS_DEAD won't resolve no matter how long we wait — skip those.
where.push(`EXISTS (
SELECT 1 FROM front_page_audits a
WHERE a.business_id = b.id
AND a.error_message IS NOT NULL
AND a.error_message NOT LIKE '%ERR_NAME_NOT_RESOLVED%'
AND a.error_message NOT LIKE '%ERR_CONNECTION_REFUSED%'
)`);
}
const r = await query<BusinessRow>(`
SELECT b.id, b.name, b.website
FROM businesses b
WHERE ${where.join(' AND ')}
ORDER BY b.id
LIMIT $1
`, [opts.limit]);
return r.rows;
}
async function deleteFailedAudit(businessId: number) {
await query(`DELETE FROM front_page_audits WHERE business_id = $1 AND error_message IS NOT NULL`, [businessId]);
}
// Static-HTML fallback for sites that block Chromium HTTP/2 (Ralphs, McDonald's chain pages)
// or have broken TLS. No screenshot, but populates the audit row with real signals.
async function staticFetchFallback(b: BusinessRow): Promise<AuditCapture> {
const out = emptyCapture();
const startedAt = Date.now();
try {
const res = await fetch(b.website, {
redirect: 'follow',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 ventura-corridor/0.1 (audit-fallback; +local)',
'Accept': 'text/html,application/xhtml+xml',
},
signal: AbortSignal.timeout(TIMEOUT_MS),
});
out.fetchedUrl = res.url;
out.statusCode = res.status;
out.hasHttps = res.url.startsWith('https:');
out.httpProtocol = out.hasHttps ? 'https' : 'http';
out.loadMs = Date.now() - startedAt;
const html = await res.text();
out.bytesHtml = Buffer.byteLength(html, 'utf8');
out.bytesTotal = out.bytesHtml;
// Lightweight regex-based extraction. Crude but enough for SEO scoring.
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
out.titleText = titleMatch ? titleMatch[1].trim().slice(0, 500) : null;
out.hasTitle = !!out.titleText;
const descMatch = html.match(/<meta\s+name=["']description["'][^>]*content=["']([^"']*)/i);
out.metaDesc = descMatch ? descMatch[1].trim().slice(0, 1000) : null;
const h1Match = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
out.h1Text = h1Match ? h1Match[1].replace(/<[^>]+>/g, '').trim().slice(0, 500) : null;
out.hasH1 = !!out.h1Text;
out.hasViewport = /<meta\s+name=["']viewport["']/i.test(html);
const ogMatch = html.match(/<meta\s+property=["']og:image["'][^>]*content=["']([^"']*)/i);
out.ogImage = ogMatch ? ogMatch[1].slice(0, 800) : null;
const ldjsonMatches = html.match(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi) || [];
out.schemaOrgCount = ldjsonMatches.length;
const text = html.replace(/<script[\s\S]*?<\/script>/gi, '').replace(/<style[\s\S]*?<\/style>/gi, '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
out.wordCount = text ? text.split(' ').length : null;
// Mark fallback in error_message so we know this row didn't get a screenshot.
out.errorMessage = '[fallback:static-fetch] no screenshot';
// Persist raw HTML so re-scoring can run later.
const htmlName = `firm-${b.id}.html`;
writeFileSync(join(RAW_DIR, htmlName), html, 'utf8');
out.rawHtml = `raw/${htmlName}`;
} catch (e: any) {
out.errorMessage = '[fallback-failed] ' + (e?.message || String(e)).slice(0, 400);
out.loadMs = Date.now() - startedAt;
}
return out;
}
interface AuditCapture {
fetchedUrl: string | null;
statusCode: number | null;
httpProtocol: string | null;
ttfbMs: number | null;
loadMs: number | null;
bytesHtml: number | null;
bytesTotal: number | null;
hasHttps: boolean;
hasViewport: boolean;
hasTitle: boolean;
titleText: string | null;
metaDesc: string | null;
hasH1: boolean;
h1Text: string | null;
schemaOrgCount: number;
schemaOrgTypes: string[];
ogImage: string | null;
altCoverage: number | null;
wordCount: number | null;
outboundLinks: number;
internalLinks: number;
errorMessage: string | null;
rawHtml: string | null;
screenshotPath: string | null;
}
function emptyCapture(err?: string): AuditCapture {
return {
fetchedUrl: null, statusCode: null, httpProtocol: null,
ttfbMs: null, loadMs: null, bytesHtml: null, bytesTotal: null,
hasHttps: false, hasViewport: false, hasTitle: false, titleText: null,
metaDesc: null, hasH1: false, h1Text: null,
schemaOrgCount: 0, schemaOrgTypes: [], ogImage: null,
altCoverage: null, wordCount: null, outboundLinks: 0, internalLinks: 0,
errorMessage: err || null, rawHtml: null, screenshotPath: null,
};
}
async function captureOne(browser: Browser, b: BusinessRow): Promise<AuditCapture> {
const out = emptyCapture();
const ctx = await browser.newContext({
ignoreHTTPSErrors: true,
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 ventura-corridor/0.1 (audit; +local)',
viewport: { width: 1280, height: 900 },
});
const page = await ctx.newPage();
let totalBytes = 0;
page.on('response', (r) => {
const len = parseInt(r.headers()['content-length'] || '0', 10);
if (Number.isFinite(len) && len > 0) totalBytes += len;
});
const startedAt = Date.now();
try {
// 'load' is more permissive than 'domcontentloaded' for sites that lazy-render.
const resp = await page.goto(b.website, { waitUntil: 'load', timeout: TIMEOUT_MS });
out.loadMs = Date.now() - startedAt;
if (!resp) {
out.errorMessage = 'no response';
await ctx.close();
return out;
}
out.fetchedUrl = page.url();
out.statusCode = resp.status();
out.httpProtocol = out.fetchedUrl.startsWith('https:') ? 'https' : 'http';
out.hasHttps = out.httpProtocol === 'https';
const reqTime = (resp.timing && resp.timing().responseStart) || 0;
out.ttfbMs = reqTime > 0 ? Math.round(reqTime) : null;
const html = await page.content();
out.bytesHtml = Buffer.byteLength(html, 'utf8');
out.bytesTotal = totalBytes || out.bytesHtml;
// Pass as string to avoid tsx injecting __name() helper calls (which the
// browser doesn't have). Plain JS only.
const meta = await page.evaluate(`(function(){
function q(sel){ return document.querySelector(sel); }
function all(sel){ return Array.prototype.slice.call(document.querySelectorAll(sel)); }
var viewport = !!q('meta[name="viewport" i]');
var title = (document.title || '').trim();
var mdEl = q('meta[name="description" i]');
var metaDesc = (mdEl && mdEl.content) ? mdEl.content.trim() : '';
var h1 = q('h1');
var ogEl = q('meta[property="og:image" i]');
var ogImg = (ogEl && ogEl.content) ? ogEl.content.trim() : null;
var ldjson = all('script[type="application/ld+json"]').map(function(s){ return s.textContent || ''; }).filter(Boolean);
var types = [];
var ldCount = 0;
for (var i = 0; i < ldjson.length; i++) {
try {
var j = JSON.parse(ldjson[i]);
var items = Array.isArray(j) ? j : [j];
for (var k = 0; k < items.length; k++) {
var it = items[k];
ldCount++;
if (it && it['@type']) {
var ts = Array.isArray(it['@type']) ? it['@type'] : [it['@type']];
for (var m = 0; m < ts.length; m++) types.push(String(ts[m]));
}
}
} catch (e) {}
}
var imgs = all('img');
var totalImgs = imgs.length;
var withAlt = 0;
for (var ii = 0; ii < imgs.length; ii++) {
var a = imgs[ii].getAttribute('alt') || '';
if (a.trim().length > 0) withAlt++;
}
var text = (document.body && document.body.innerText ? document.body.innerText : '').replace(/\\s+/g, ' ').trim();
var words = text ? text.split(' ').length : 0;
var links = all('a[href]');
var host = location.hostname;
var inLinks = 0, outLinks = 0;
for (var li = 0; li < links.length; li++) {
try {
var u = new URL(links[li].href, location.href);
if (u.hostname === host) inLinks++; else outLinks++;
} catch (e) {}
}
var uniq = [];
for (var ti = 0; ti < types.length; ti++) if (uniq.indexOf(types[ti]) < 0) uniq.push(types[ti]);
return {
viewport: viewport, title: title, metaDesc: metaDesc,
h1Text: (h1 && h1.textContent) ? h1.textContent.trim() : null,
ogImg: ogImg, ldCount: ldCount, types: uniq,
altCoverage: totalImgs > 0 ? withAlt / totalImgs : null,
wordCount: words, inLinks: inLinks, outLinks: outLinks
};
})()`);
out.hasViewport = meta.viewport;
out.hasTitle = !!meta.title;
out.titleText = meta.title || null;
out.metaDesc = meta.metaDesc || null;
out.hasH1 = !!meta.h1Text;
out.h1Text = meta.h1Text;
out.schemaOrgCount = meta.ldCount;
out.schemaOrgTypes = meta.types;
out.ogImage = meta.ogImg;
out.altCoverage = meta.altCoverage;
out.wordCount = meta.wordCount;
out.outboundLinks = meta.outLinks;
out.internalLinks = meta.inLinks;
// Screenshot — top-of-viewport only, JPEG to keep size small.
const shotName = `firm-${b.id}.jpg`;
const shotPath = join(SCREEN_DIR, shotName);
await page.screenshot({ path: shotPath, type: 'jpeg', quality: 70, fullPage: false });
out.screenshotPath = `screenshots/${shotName}`;
// Raw HTML for offline re-scoring later.
const htmlName = `firm-${b.id}.html`;
writeFileSync(join(RAW_DIR, htmlName), html, 'utf8');
out.rawHtml = `raw/${htmlName}`;
} catch (e: any) {
out.errorMessage = (e?.message || String(e)).slice(0, 500);
out.loadMs = Date.now() - startedAt;
} finally {
await ctx.close().catch(() => {});
}
return out;
}
async function insertAudit(businessId: number, c: AuditCapture) {
await query(`
INSERT INTO front_page_audits (
business_id, fetched_url, status_code, http_protocol,
ttfb_ms, load_ms, bytes_html, bytes_total,
has_https, has_viewport, has_title, title_text, meta_desc,
has_h1, h1_text, schema_org_count, schema_org_types,
og_image, alt_coverage, word_count, outbound_links, internal_links,
screenshot_path, error_message, raw_html_path
) VALUES (
$1, $2, $3, $4,
$5, $6, $7, $8,
$9, $10, $11, $12, $13,
$14, $15, $16, $17,
$18, $19, $20, $21, $22,
$23, $24, $25
)
`, [
businessId, c.fetchedUrl, c.statusCode, c.httpProtocol,
c.ttfbMs, c.loadMs, c.bytesHtml, c.bytesTotal,
c.hasHttps, c.hasViewport, c.hasTitle, c.titleText?.slice(0, 500), c.metaDesc?.slice(0, 1000),
c.hasH1, c.h1Text?.slice(0, 500), c.schemaOrgCount, c.schemaOrgTypes,
c.ogImage?.slice(0, 800), c.altCoverage, c.wordCount, c.outboundLinks, c.internalLinks,
c.screenshotPath, c.errorMessage, c.rawHtml,
]);
}
async function main() {
const argLimit = process.argv.find((a) => a.startsWith('--limit='));
const rescoreOnly = process.argv.includes('--rescore');
const retryFailed = process.argv.includes('--retry-failed');
const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 1000;
const queue = await pickQueue({ limit, rescoreOnly, retryFailed });
const mode = retryFailed ? 'retry-failed' : rescoreOnly ? 'rescore' : 'full';
console.log(`[crawl] mode=${mode} · ${queue.length} sites in queue · concurrency ${CONCURRENCY} · timeout ${TIMEOUT_MS}ms`);
const browser = await chromium.launch({ headless: true });
const startedAt = Date.now();
let done = 0, ok = 0, fail = 0, recovered = 0;
// Simple parallel worker pool.
const queueCopy = queue.slice();
await Promise.all(Array.from({ length: CONCURRENCY }).map(async () => {
while (queueCopy.length) {
const b = queueCopy.shift()!;
let c = await captureOne(browser, b);
// Try the static-fetch fallback on transport-level failures Chromium can't fix.
if (c.errorMessage && /ERR_HTTP2_PROTOCOL_ERROR|ERR_SSL_PROTOCOL_ERROR|navigating and changing/.test(c.errorMessage)) {
const fb = await staticFetchFallback(b);
if (fb.statusCode && fb.statusCode >= 200 && fb.statusCode < 400) {
c = fb;
recovered++;
}
}
if (retryFailed) await deleteFailedAudit(b.id);
await insertAudit(b.id, c);
done++;
// Treat fallback rows (errorMessage starts with [fallback:) as ok — they have real data.
const isOk = !c.errorMessage || c.errorMessage.startsWith('[fallback:');
if (isOk) ok++; else fail++;
if (done % 10 === 0 || done === queue.length) {
const tail = isOk
? `last: ${b.name} (${c.statusCode || '?'}) ${c.loadMs || 0}ms`
: `last fail: ${b.name} → ${c.errorMessage!.slice(0, 80)}`;
console.log(`[crawl] ${done}/${queue.length} ok=${ok} fail=${fail} recovered=${recovered} · ${tail}`);
}
}
}));
await browser.close();
const elapsed = Math.round((Date.now() - startedAt) / 1000);
console.log('');
console.log(`[crawl] done in ${elapsed}s · ${ok} ok · ${fail} failed`);
await pool.end();
}
main().catch((e) => {
console.error('[crawl]', e);
process.exit(1);
});