← back to Ventura Corridor
src/enrich/website_via_brave.ts
213 lines
/**
* Website discovery for BTRC rows via Brave Search API.
*
* Picks corridor businesses with no website, runs `<name> <city> Ventura Blvd`,
* extracts the first credible non-social result, writes to businesses.website.
*
* Brave free tier: 2000 queries/month, ~1 q/sec rate limit.
*
* Reads BRAVE_SEARCH_API_KEY from ~/Projects/secrets-manager/.env
* (also falls back to ~/Projects/_ventura-recon/.env if present).
*/
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';
import { query, pool } from '../../db/pool.ts';
function loadKey(): string {
if (process.env.BRAVE_SEARCH_API_KEY) return process.env.BRAVE_SEARCH_API_KEY.trim();
for (const p of [
join(homedir(), 'Projects/secrets-manager/.env'),
join(homedir(), 'Projects/_ventura-recon/.env'),
]) {
try {
const m = readFileSync(p, 'utf8').match(/^BRAVE_SEARCH_API_KEY=(.+)$/m);
if (m) return m[1].trim();
} catch {}
}
return '';
}
const KEY = loadKey();
const BLOCK = new Set([
// Social
'yelp.com', 'facebook.com', 'instagram.com', 'twitter.com', 'x.com',
'linkedin.com', 'tiktok.com', 'youtube.com', 'pinterest.com', 'reddit.com',
// Maps/aggregators/dirs
'mapquest.com', 'google.com', 'maps.google.com', 'foursquare.com',
'tripadvisor.com', 'opentable.com', 'doordash.com', 'ubereats.com',
'grubhub.com', 'mapcarta.com', 'wikipedia.org', 'wikidata.org',
'angi.com', 'angieslist.com', 'thumbtack.com', 'alignable.com',
'signpost.com', 'nextdoor.com', 'rocketreach.co', 'apollo.io',
'zoominfo.com', 'dnb.com', 'yellowpages.com', 'whitepages.com',
'spokeo.com', 'truepeoplesearch.com', 'fastpeoplesearch.com',
// Real estate
'zillow.com', 'realtor.com', 'redfin.com', 'apartments.com', 'trulia.com',
'loopnet.com', 'costar.com',
// Reviews / business records
'bbb.org', 'manta.com', 'bizapedia.com', 'opencorporates.com',
'buzzfile.com', 'corporationwiki.com', 'companies.org',
// Gov
'lacity.org', 'data.lacity.org', 'lavote.gov', 'lacounty.gov',
'irs.gov', 'sec.gov', 'sba.gov',
// Legal
'avvo.com', 'lawyers.com', 'martindale.com', 'findlaw.com', 'justia.com',
'caselaw.findlaw.com', 'leagle.com',
// Health
'healthgrades.com', 'zocdoc.com', 'vitals.com', 'webmd.com', 'wellness.com',
// Jobs
'glassdoor.com', 'indeed.com', 'crunchbase.com', 'ziprecruiter.com',
// Misc
'amazon.com', 'ebay.com', 'pinterest.com', 'medium.com',
]);
function host(u: string): string | null {
try { return new URL(u).hostname.replace(/^www\./, '').toLowerCase(); } catch { return null; }
}
function credible(u: string): boolean {
const h = host(u);
if (!h) return false;
if (BLOCK.has(h)) return false;
for (const d of BLOCK) if (h.endsWith('.' + d)) return false;
// Block all .gov / .edu / .mil — never first-party for a small biz.
if (/\.(gov|edu|mil)$/.test(h)) return false;
return true;
}
const STOPWORDS = new Set([
'inc', 'llc', 'lp', 'llp', 'corp', 'corporation', 'co', 'company',
'and', 'the', 'of', 'a', 'an', 'in', 'on', 'at', 'for',
'group', 'services', 'service', 'enterprise', 'holdings',
'associates', 'assoc', 'partners', 'pllc', 'pc',
]);
function nameTokens(name: string): string[] {
return name
.toLowerCase()
.replace(/[^a-z0-9 ]/g, ' ')
.split(/\s+/)
.filter((t) => t.length >= 4 && !STOPWORDS.has(t));
}
// Reject if no significant word from the business name appears in the domain root.
// e.g. "FOUR PAWS DAY CARE" + fourpawsdaycare.com → match (paws/four/care)
// "NATURAL STONE RESTORATION" + calabasasstyle.com → reject
function domainMatchesName(host: string, name: string): boolean {
const root = host.split('.').slice(-2)[0]; // "fourpawsdaycare" from "fourpawsdaycare.com"
if (!root) return false;
const tokens = nameTokens(name);
if (!tokens.length) return false;
// Allow if any name token (≥4 chars) is a substring of the domain root.
return tokens.some((t) => root.includes(t));
}
async function brave(q: string): Promise<string[]> {
const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(q)}&count=10`;
const r = await fetch(url, {
headers: { 'X-Subscription-Token': KEY, Accept: 'application/json' },
signal: AbortSignal.timeout(12_000),
});
if (r.status === 429) return [];
if (!r.ok) throw new Error(`brave ${r.status}`);
const data: any = await r.json();
return (data?.web?.results || []).map((x: any) => x.url).filter(Boolean);
}
interface Row { id: number; name: string; city: string | null; }
// Categories most likely to have public-facing websites we can index +
// ingest for the news magazine. Excludes "office: company" / "office:
// estate_agent" / "office: lawyer" which are dominated by single-person
// LLCs and back-office shells where the hit rate is near zero.
const HIGH_VALUE_CATEGORY_PREFIXES = [
'amenity: restaurant', 'amenity: fast_food', 'amenity: cafe',
'amenity: bar', 'amenity: pub', 'amenity: nightclub',
'shop: hairdresser', 'shop: clothes', 'shop: shoes', 'shop: jewelry',
'shop: bakery', 'shop: butcher', 'shop: coffee', 'shop: tea',
'shop: cosmetics', 'shop: beauty', 'shop: tattoo', 'shop: pet',
'shop: bicycle', 'shop: bookstore', 'shop: convenience',
'shop: department_store', 'shop: gift', 'shop: hardware',
'shop: hifi', 'shop: optician', 'shop: photo', 'shop: stationery',
'shop: supermarket', 'shop: variety_store',
'leisure: fitness_centre', 'leisure: dance', 'leisure: bowling_alley',
'amenity: dentist', 'amenity: pharmacy', 'amenity: veterinary',
'tourism: hotel', 'tourism: motel', 'tourism: apartment',
'amenity: car_wash', 'shop: car_parts',
];
async function pick(limit: number, categoryFilter: 'high_value' | 'all'): Promise<Row[]> {
if (categoryFilter === 'high_value') {
// ANY-prefix-match against the curated whitelist. Skips "office: ..."
// shells which are 70% of the corridor and ~0% of the websites.
const conds = HIGH_VALUE_CATEGORY_PREFIXES.map((_, i) => `category ILIKE $${i + 2}`).join(' OR ');
const params: any[] = [limit, ...HIGH_VALUE_CATEGORY_PREFIXES.map(p => p + '%')];
const r = await query<Row>(`
SELECT id, name, city FROM businesses
WHERE on_corridor AND website IS NULL
AND length(name) >= 5
AND name NOT LIKE '% LLC'
AND name NOT ILIKE 'POST OFFICE BOX%'
AND (${conds})
ORDER BY id
LIMIT $1
`, params);
return r.rows;
}
const r = await query<Row>(`
SELECT id, name, city FROM businesses
WHERE on_corridor AND source = 'la_btrc' AND website IS NULL
AND length(name) >= 5
AND name NOT LIKE '% LLC'
AND name NOT ILIKE 'POST OFFICE BOX%'
ORDER BY id
LIMIT $1
`, [limit]);
return r.rows;
}
async function main() {
if (!KEY) {
console.error('[brave] BRAVE_SEARCH_API_KEY missing');
process.exit(1);
}
const argLimit = process.argv.find((a) => a.startsWith('--limit='));
const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 20;
// Default to high-value categories (consumer-facing) — restaurants, retail,
// beauty, fitness. Pass --all to fall back to every BTRC row.
const categoryFilter = process.argv.includes('--all') ? 'all' : 'high_value';
const queue = await pick(limit, categoryFilter);
console.log(`[brave] enriching ${queue.length} rows · category=${categoryFilter} · throttling 1.1s/query`);
let ok = 0, miss = 0, err = 0;
for (const row of queue) {
const q = `"${row.name}" Ventura Blvd ${row.city || 'Sherman Oaks'}`;
try {
const urls = await brave(q);
// First-pass: strict — domain root must contain a word from the name.
let winner = urls.find((u) => {
if (!credible(u)) return false;
const h = host(u);
return h && domainMatchesName(h, row.name);
});
if (winner) {
const h = host(winner);
const website = 'https://' + h;
await query(`UPDATE businesses SET website = $1, last_seen_at = now() WHERE id = $2`, [website, row.id]);
ok++;
console.log(` ✓ ${row.name.padEnd(40).slice(0, 40)} → ${website}`);
} else {
miss++;
}
} catch (e: any) {
err++;
console.log(` ⚠ ${row.name.slice(0, 40)} ${e.message?.slice(0, 60)}`);
}
await new Promise((r) => setTimeout(r, 1100));
}
console.log(`[brave] done · ${ok} found · ${miss} no-result · ${err} errors`);
await pool.end();
}
main().catch((e) => { console.error(e); process.exit(1); });