← back to Lawyer Directory Builder
src/enrich/site_audit.ts
344 lines
/**
* Site audit — for each firm with a website, fetch the homepage in headless
* Chromium, capture a 1280x800 screenshot, extract the dominant color palette,
* scan the DOM for marketing-quality signals, compute a 0-100 score, generate
* concrete suggestions, and persist all of it to the site_audits table.
*
* NOTE on "traffic ranking": real traffic numbers require paid APIs (SimilarWeb,
* SEMrush, Ahrefs). This module computes a *marketing health score* from
* publicly observable on-page signals — a defensible proxy for "is this site
* doing real marketing work" but NOT a substitute for actual visitor counts.
*
* CLI:
* npx tsx src/enrich/site_audit.ts # full pass
* npx tsx src/enrich/site_audit.ts --smoke # first 5 firms
* npx tsx src/enrich/site_audit.ts --limit 50
* npx tsx src/enrich/site_audit.ts --re-audit # re-audit firms even if recent
*/
import 'dotenv/config';
import { chromium, Browser } from 'playwright';
import sharp from 'sharp';
import fs from 'node:fs';
import path from 'node:path';
import { pool, query } from '../db/pool.ts';
const SCREENSHOT_DIR = path.resolve(process.cwd(), 'public/screenshots');
const RAW_HTML_DIR = path.resolve(process.cwd(), 'raw_html');
const USER_AGENT = process.env.USER_AGENT
|| 'LawyerDirectoryBuilder/0.1 (research; contact: steveabramsdesigns@gmail.com)';
const CONCURRENCY = Number(process.env.AUDIT_CONCURRENCY || 3);
const argv = process.argv.slice(2);
const SMOKE = argv.includes('--smoke');
const RE_AUDIT = argv.includes('--re-audit');
// --backfill-html: re-audit firms that have a site_audits row but no raw_html_path.
// Used once to populate the new column without re-doing 12k fresh crawls.
const BACKFILL_HTML = argv.includes('--backfill-html');
const LIMIT_IDX = argv.indexOf('--limit');
const LIMIT = LIMIT_IDX >= 0 ? Number(argv[LIMIT_IDX + 1]) : (SMOKE ? 5 : 0);
type Firm = { id: number; name: string; website: string };
type Signals = {
status_code: number | null;
final_url: string;
has_https: boolean;
has_favicon: boolean;
has_og_image: boolean;
has_meta_viewport: boolean;
has_analytics: boolean;
has_h1: boolean;
has_schema_org: boolean;
has_phone_visible: boolean;
page_size_bytes: number;
load_time_ms: number;
font_count: number;
image_count: number;
link_count: number;
};
async function loadFirms(): Promise<Firm[]> {
const lim = LIMIT > 0 ? `LIMIT ${LIMIT}` : '';
if (BACKFILL_HTML) {
// Backfill pass: every firm that has at least one site_audits row but no
// raw_html_path on its newest audit. Skip firms whose newest row is a 4xx/5xx,
// or that have no audit row at all (those go through the normal path).
const r = await query<Firm>(`
SELECT DISTINCT ON (o.id) o.id, o.name, o.website
FROM organizations o
JOIN site_audits a ON a.organization_id = o.id
WHERE o.type='law_firm'
AND o.website IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM site_audits a2
WHERE a2.organization_id = o.id AND a2.raw_html_path IS NOT NULL
)
ORDER BY o.id, a.audited_at DESC
${lim}
`);
return r.rows;
}
const reAuditFilter = RE_AUDIT
? ''
: `AND NOT EXISTS (SELECT 1 FROM site_audits a WHERE a.organization_id = o.id AND a.audited_at > NOW() - INTERVAL '7 days' AND a.status_code BETWEEN 200 AND 399)`;
const r = await query<Firm>(`
SELECT o.id, o.name, o.website
FROM organizations o
WHERE o.type='law_firm' AND o.website IS NOT NULL
${reAuditFilter}
ORDER BY o.id
${lim}
`);
return r.rows;
}
async function extractPalette(pngBuffer: Buffer): Promise<{ primary: string; palette: Array<{hex:string;r:number;g:number;b:number;fraction:number}> }> {
// Downsample to 80x50 (4000 px), bucket colors into a 16-step grid (4096 buckets),
// count frequencies, return top 6 by fraction.
const { data, info } = await sharp(pngBuffer)
.resize(80, 50, { fit: 'cover' })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
const total = info.width * info.height;
const buckets = new Map<string, { r: number; g: number; b: number; n: number }>();
for (let i = 0; i < data.length; i += 3) {
const r = data[i], g = data[i + 1], b = data[i + 2];
// Skip near-white & near-black to surface brand colors
const max = Math.max(r, g, b), min = Math.min(r, g, b);
if (max > 245 && min > 240) continue;
if (max < 18) continue;
const key = `${r >> 4}-${g >> 4}-${b >> 4}`;
const cur = buckets.get(key);
if (cur) { cur.r += r; cur.g += g; cur.b += b; cur.n++; }
else buckets.set(key, { r, g, b, n: 1 });
}
const sorted = [...buckets.values()].sort((a, b) => b.n - a.n).slice(0, 6).map(c => {
const r = Math.round(c.r / c.n), g = Math.round(c.g / c.n), b = Math.round(c.b / c.n);
return {
hex: '#' + [r, g, b].map(v => v.toString(16).padStart(2, '0')).join(''),
r, g, b, fraction: c.n / total,
};
});
return { primary: sorted[0]?.hex || '#000000', palette: sorted };
}
function scoreAndSuggest(s: Signals, firmName: string): { score: number; suggestions: string[] } {
const sug: string[] = [];
let score = 0;
// HTTPS — table stakes
if (s.has_https) score += 15;
else sug.push('Move site to HTTPS (free via Let\'s Encrypt) — visitors and Google penalize HTTP.');
// Mobile
if (s.has_meta_viewport) score += 10;
else sug.push('Add `<meta name="viewport" content="width=device-width">` for mobile rendering.');
// OG image (social sharing)
if (s.has_og_image) score += 10;
else sug.push('Add an Open Graph image (og:image) so links shared on LinkedIn/Facebook show a preview.');
// Analytics
if (s.has_analytics) score += 10;
else sug.push('Install GA4, Plausible, or Fathom — you can\'t improve traffic you\'re not measuring.');
// Favicon
if (s.has_favicon) score += 5;
else sug.push('Add a favicon — bare-tab pages look unfinished.');
// Content
if (s.has_h1) score += 5;
else sug.push('Add a clear H1 heading — current page has none, hurts SEO.');
if (s.has_schema_org) score += 10;
else sug.push('Add Schema.org LegalService / Attorney structured data — Google rich-results eligibility.');
if (s.has_phone_visible) score += 5;
else sug.push('Surface the firm phone number above the fold — call-driven conversions need one click.');
// Page facts
if (s.page_size_bytes > 0 && s.page_size_bytes < 2_000_000) score += 10;
else if (s.page_size_bytes >= 2_000_000) sug.push(`Page is ${(s.page_size_bytes/1e6).toFixed(1)} MB — compress images / lazy-load.`);
if (s.load_time_ms > 0 && s.load_time_ms < 3000) score += 10;
else if (s.load_time_ms >= 3000) sug.push(`Load time ${(s.load_time_ms/1000).toFixed(1)}s — visitors bounce after 3s.`);
if (s.font_count >= 2 && s.font_count <= 4) score += 5;
else if (s.font_count > 4) sug.push(`${s.font_count} font families loaded — consolidate to 2-3.`);
if (s.image_count >= 3) score += 5;
// Cap at 100
return { score: Math.min(100, score), suggestions: sug };
}
async function auditOne(browser: Browser, firm: Firm): Promise<{ ok: boolean; signals?: Signals; primary?: string; palette?: any; score?: number; suggestions?: string[]; screenshot?: string; raw_html_path?: string; error?: string }> {
const ctx = await browser.newContext({
userAgent: USER_AGENT,
viewport: { width: 1280, height: 800 },
ignoreHTTPSErrors: true,
});
const page = await ctx.newPage();
let bytesIn = 0;
page.on('response', async r => {
try {
const len = r.headers()['content-length'];
if (len) bytesIn += parseInt(len, 10);
} catch {}
});
try {
const t0 = Date.now();
let resp;
try {
resp = await page.goto(firm.website, { waitUntil: 'load', timeout: 20000 });
} catch (e) {
return { ok: false, error: `goto: ${(e as Error).message.slice(0, 120)}` };
}
const load_time_ms = Date.now() - t0;
const status_code = resp?.status() ?? null;
const final_url = page.url();
if (!status_code || status_code >= 400) {
return { ok: false, signals: { status_code, final_url, has_https: false, has_favicon: false, has_og_image: false, has_meta_viewport: false, has_analytics: false, has_h1: false, has_schema_org: false, has_phone_visible: false, page_size_bytes: 0, load_time_ms, font_count: 0, image_count: 0, link_count: 0 } };
}
// NOTE: keep this evaluate() flat — no nested arrow helpers, no TS syntax —
// tsx's transform inserts a `__name` reference for sub-functions that
// doesn't exist in the browser context.
const signals = await page.evaluate(`(function(){
var d = document;
var head = d.head || d;
var html = (d.documentElement.outerHTML || '').toLowerCase();
var bodyText = (d.body && d.body.innerText) || '';
var fonts = {};
try {
var all = d.querySelectorAll('*');
for (var i = 0; i < all.length; i++) {
var f = ((window.getComputedStyle(all[i]).fontFamily) || '').split(',')[0].trim().replace(/['"]/g, '').toLowerCase();
if (f) fonts[f] = 1;
}
} catch (e) {}
var fontCount = Object.keys(fonts).length;
var hasGA =
/google-analytics\\.com\\/(g|gtag)|googletagmanager\\.com\\/gtag\\/js|plausible\\.io\\/js|fathom\\.tech|piwik\\.|matomo\\./i.test(html)
|| !!head.querySelector('script[src*="googletagmanager"]')
|| !!head.querySelector('script[src*="plausible"]')
|| !!head.querySelector('script[src*="fathom"]');
var hasSchema =
html.indexOf('schema.org/legalservice') >= 0
|| html.indexOf('schema.org/attorney') >= 0
|| html.indexOf('schema.org/lawyer') >= 0
|| html.indexOf('"@type":"legalservice"') >= 0
|| html.indexOf('"@type":"attorney"') >= 0
|| html.indexOf('"@type":"lawyer"') >= 0;
return {
has_favicon: !!head.querySelector('link[rel*="icon"]'),
has_og_image: !!head.querySelector('meta[property="og:image"]'),
has_meta_viewport: !!head.querySelector('meta[name="viewport"]'),
has_analytics: hasGA,
has_h1: !!d.querySelector('h1'),
has_schema_org: hasSchema,
has_phone_visible: /\\b\\(?\\d{3}\\)?[\\s.\\-]?\\d{3}[\\s.\\-]?\\d{4}\\b/.test(bodyText),
font_count: fontCount,
image_count: d.querySelectorAll('img').length,
link_count: d.querySelectorAll('a[href]').length,
};
})()`) as any;
const png = await page.screenshot({ type: 'png', fullPage: false });
const filename = `firm-${firm.id}.png`;
fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
fs.writeFileSync(path.join(SCREENSHOT_DIR, filename), png);
const { primary, palette } = await extractPalette(png);
// Persist raw HTML so the offline regex pass (ad_signals.ts) can mine
// tracking-pixel fingerprints without re-crawling. INTERNAL ONLY.
let rawHtmlRel: string | undefined;
try {
const html = await page.content();
const htmlName = `firm-${firm.id}-${Date.now()}.html`;
fs.mkdirSync(RAW_HTML_DIR, { recursive: true });
fs.writeFileSync(path.join(RAW_HTML_DIR, htmlName), html);
rawHtmlRel = `raw_html/${htmlName}`;
} catch (e) {
// Non-fatal: audit still records on-page signals + screenshot.
console.warn(`[audit] raw_html capture failed for #${firm.id}: ${(e as Error).message.slice(0, 100)}`);
}
const fullSignals: Signals = {
status_code,
final_url,
has_https: final_url.startsWith('https://'),
...signals,
page_size_bytes: bytesIn,
load_time_ms,
};
const { score, suggestions } = scoreAndSuggest(fullSignals, firm.name);
return { ok: true, signals: fullSignals, primary, palette, score, suggestions, screenshot: `/screenshots/${filename}`, raw_html_path: rawHtmlRel };
} finally {
await ctx.close().catch(() => {});
}
}
async function persist(firmId: number, url: string, r: Awaited<ReturnType<typeof auditOne>>) {
const s = r.signals;
await query(`
INSERT INTO site_audits (
organization_id, url, final_url, status_code, screenshot_path,
primary_color, palette,
has_https, has_favicon, has_og_image, has_meta_viewport, has_analytics,
has_h1, has_schema_org, has_phone_visible,
page_size_bytes, load_time_ms, font_count, image_count, link_count,
marketing_score, suggestions, raw_html_path
) VALUES (
$1,$2,$3,$4,$5,
$6,$7::jsonb,
$8,$9,$10,$11,$12,
$13,$14,$15,
$16,$17,$18,$19,$20,
$21,$22,$23
)
`, [
firmId, url, s?.final_url || null, s?.status_code || null, r.screenshot || null,
r.primary || null, r.palette ? JSON.stringify(r.palette) : null,
s?.has_https ?? null, s?.has_favicon ?? null, s?.has_og_image ?? null, s?.has_meta_viewport ?? null, s?.has_analytics ?? null,
s?.has_h1 ?? null, s?.has_schema_org ?? null, s?.has_phone_visible ?? null,
s?.page_size_bytes ?? null, s?.load_time_ms ?? null, s?.font_count ?? null, s?.image_count ?? null, s?.link_count ?? null,
r.score ?? null, r.suggestions || null, r.raw_html_path || null,
]);
}
async function main() {
const firms = await loadFirms();
if (firms.length === 0) { console.log('[audit] no firms to audit'); await pool.end(); return; }
console.log(`[audit] auditing ${firms.length} firms (concurrency=${CONCURRENCY}, screenshots → ${SCREENSHOT_DIR})…`);
const browser = await chromium.launch({ headless: true });
try {
let seen = 0, ok = 0, errs = 0;
let cursor = 0;
const workers = Array.from({ length: CONCURRENCY }, async () => {
while (true) {
const idx = cursor++;
if (idx >= firms.length) break;
const firm = firms[idx];
seen++;
try {
const r = await auditOne(browser, firm);
await persist(firm.id, firm.website, r);
if (r.ok) {
ok++;
console.log(`[audit] ✓ #${firm.id} ${firm.name.slice(0,40).padEnd(40)} ${r.primary} score=${r.score}`);
} else {
console.log(`[audit] ✗ #${firm.id} ${firm.name.slice(0,40).padEnd(40)} ${r.error || 'no-200'}`);
}
} catch (e) {
errs++;
console.error(`[audit] err #${firm.id}: ${(e as Error).message}`);
}
if (seen % 25 === 0) console.log(`[audit] ${seen}/${firms.length} ok=${ok} errs=${errs}`);
}
});
await Promise.all(workers);
console.log(`[audit] done. seen=${seen} ok=${ok} errs=${errs}`);
} finally {
await browser.close();
}
await pool.end();
}
main().catch(async (err) => {
console.error('[audit] fatal:', err);
try { await pool.end(); } catch {}
process.exit(1);
});