← back to Ventura Corridor
src/enrich/chambers.ts
314 lines
/**
* Scrape chamber-of-commerce member directories along the Ventura corridor and
* tag matched businesses.
*
* Two persistence sinks (kept in sync since 2026-05-05):
* 1. business_contacts — one row per (business, chamber) (legacy, audit-friendly)
* 2. business_enrichment.chamber_memberships — JSONB blob, merged not overwritten
*
* Platforms supported:
* • ChamberMaster / GrowthZone — alpha-pagination at /list/searchalpha/A..Z
* members.<chamber>.org → Calabasas, Studio City, Sherman Oaks/Encino
* • West Valley Warner Center custom PHP — new_directory.php?start=N (15/page)
*
* Skipped (intentional):
* • Encino standalone (encinochamber.org) — site is a Korean spinner stub, no
* directory. Encino is now folded into shermanoaksencinochamber.org anyway.
* • Tarzana Chamber (tarzanachamber.org) — domain NXDOMAIN, no live chamber.
*
* Matching is strict: normalized-equality on businesses.name OR raw->>business_name
* OR raw->>dba_name (lowercase, drop suffixes Inc/LLC/etc., punctuation→space).
*
* No LLM. Free. Polite (300ms inter-fetch).
*/
import { query, pool } from '../../db/pool.ts';
type Platform = 'chambermaster' | 'wvwc_custom';
interface Chamber {
name: string;
base: string;
platform: Platform;
}
const CHAMBERS: Chamber[] = [
{ name: 'Calabasas Chamber', base: 'https://www.calabasaschamber.com', platform: 'chambermaster' },
{ name: 'Studio City Chamber', base: 'https://www.studiocitychamber.com', platform: 'chambermaster' },
// Sherman Oaks + Encino merged → one combined chamber on ChamberMaster.
// Public site = shermanoaksencinochamber.org; member directory subdomain = members.…
{ name: 'Sherman Oaks Encino Chamber', base: 'https://members.shermanoaksencinochamber.org', platform: 'chambermaster' },
// West Valley Warner Center Chamber covers Tarzana / Woodland Hills / Reseda /
// Warner Center. Custom PHP, not ChamberMaster.
{ name: 'West Valley Warner Center Chamber', base: 'https://www.woodlandhillscc.net', platform: 'wvwc_custom' },
];
const ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('');
const UA = 'ventura-corridor/0.1 (chamber-directory ingest; +local)';
interface ChamberMember {
name: string;
url: string;
chamber: string;
city?: string;
website?: string;
}
// ---------- ChamberMaster / GrowthZone ----------
async function fetchAlpha(base: string, letter: string): Promise<{ name: string; url: string }[]> {
const url = `${base}/list/searchalpha/${letter}`;
const r = await fetch(url, {
headers: { 'User-Agent': UA },
signal: AbortSignal.timeout(15_000),
});
if (!r.ok) return [];
const html = await r.text();
const out: { name: string; url: string }[] = [];
// <a href="/list/member/<slug>"...>Name</a>
const re = /<a\s+href="([^"]+\/list\/member\/[^"]+)"[^>]*>([^<]+)<\/a>/g;
let m: RegExpExecArray | null;
const seen = new Set<string>();
while ((m = re.exec(html)) !== null) {
const href = m[1];
const name = m[2].replace(/\s+/g, ' ').trim();
if (!name || seen.has(href)) continue;
seen.add(href);
out.push({ name, url: href.startsWith('http') ? href : base + href });
}
return out;
}
// ---------- WVWC custom (woodlandhillscc.net) ----------
/**
* Pagination: /new_directory.php?start=N (N in {0,15,30,...}). The first page
* embeds "Results 1 - 15 of 860" — read the total to know when to stop.
*
* Per-row HTML pattern:
* <div class="inline-container directory-row">
* <div ...>
* <div id="directory_title">
* <a href="profile.php?view_id=N"><b>NAME</b></a> (or just <b>NAME</b> if no profile)
* ...
* </div>
* <div id="directory_address">
* STREET<br> CITY CA ZIP<br>
* ...
* <a href="WEBSITE" class=directory_category target=_blank>...</a> (optional)
* </div>
* </div>
* ...
* </div>
*/
async function fetchWvwcPage(
base: string,
start: number,
): Promise<{ rows: { name: string; url: string; city?: string; website?: string }[]; total: number }> {
const url = `${base}/new_directory.php?start=${start}`;
const r = await fetch(url, {
headers: { 'User-Agent': UA },
signal: AbortSignal.timeout(15_000),
});
if (!r.ok) return { rows: [], total: 0 };
const html = await r.text();
const totalMatch = html.match(/Results\s+\d+\s*-\s*\d+\s+of\s+(\d+)/i);
const total = totalMatch ? parseInt(totalMatch[1], 10) : 0;
const rows: { name: string; url: string; city?: string; website?: string }[] = [];
// Each member row spans one <div class="inline-container directory-row">…</div>
// segment. The row close is followed by `<hr class="gray-hr dashed">`.
const rowRe = /<div class="inline-container directory-row">([\s\S]*?)<hr class="gray-hr dashed">/g;
let rm: RegExpExecArray | null;
while ((rm = rowRe.exec(html)) !== null) {
const block = rm[1];
// Name: prefer linked <a href="profile.php?view_id=N"><b>NAME</b></a>, fall
// back to a bare <b>NAME</b> inside #directory_title (members without a
// public profile page still appear in the directory).
const titleM = block.match(/<div id="directory_title">([\s\S]*?)<\/div>/);
if (!titleM) continue;
const titleInner = titleM[1];
let name: string | null = null;
let viewId: string | null = null;
const linkM = titleInner.match(/<a\s+href="(profile\.php\?view_id=(\d+))"[^>]*>\s*<b>([^<]+)<\/b>\s*<\/a>/);
if (linkM) {
viewId = linkM[2];
name = linkM[3];
} else {
const bareM = titleInner.match(/<b>([^<]+)<\/b>/);
if (bareM) name = bareM[1];
}
if (!name) continue;
name = name.replace(/\s+/g, ' ').trim();
if (!name) continue;
const url = viewId
? `${base}/profile.php?view_id=${viewId}`
: `${base}/new_directory.php`;
// City: in the address block, the last "<br> CITY CA ZIP<br>" line.
let city: string | undefined;
const addrM = block.match(/<div id="directory_address">([\s\S]*?)<\/div>/);
if (addrM) {
const cityM = addrM[1].match(/<br>\s*([A-Za-z][A-Za-z .'\-]+?)\s+CA\s+\d{5}/);
if (cityM) city = cityM[1].replace(/\s+/g, ' ').trim();
}
// Website: last <a … class=directory_category target=_blank>.
let website: string | undefined;
const siteM = block.match(/<a\s+href="(http[^"]+)"\s+class=directory_category\s+target=_blank>/i);
if (siteM) {
let s = siteM[1].trim();
// Source has a known bug: many entries are stored as "http://https://..."
s = s.replace(/^http:\/\/(?=https?:\/\/)/i, '');
website = s;
}
rows.push({ name, url, city, website });
}
return { rows, total };
}
async function fetchAllWvwc(base: string): Promise<{ name: string; url: string; city?: string; website?: string }[]> {
const all: { name: string; url: string; city?: string; website?: string }[] = [];
// Bootstrap: read first page to learn total count.
const first = await fetchWvwcPage(base, 0);
all.push(...first.rows);
const total = first.total;
if (total <= 15) return dedupe(all);
for (let start = 15; start < total; start += 15) {
await new Promise((r) => setTimeout(r, 300));
const page = await fetchWvwcPage(base, start);
all.push(...page.rows);
}
return dedupe(all);
}
function dedupe<T extends { name: string; url: string }>(rows: T[]): T[] {
const seen = new Set<string>();
const out: T[] = [];
for (const r of rows) {
const k = r.url + '|' + r.name.toLowerCase();
if (seen.has(k)) continue;
seen.add(k);
out.push(r);
}
return out;
}
// ---------- Match + tag ----------
function normalizeName(s: string): string {
return s
.toLowerCase()
.replace(/\b(inc|llc|lp|llp|corp|corporation|co|company|the|of|and)\b\.?/g, '')
.replace(/[^a-z0-9 ]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
async function matchAndTag(member: ChamberMember): Promise<boolean> {
const norm = normalizeName(member.name);
if (norm.length < 4) return false;
const r = await query<{ id: number }>(
`
SELECT id FROM businesses
WHERE on_corridor
AND (
regexp_replace(lower(name), '[^a-z0-9]+', ' ', 'g') = $1
OR regexp_replace(lower(coalesce(raw->>'business_name','')), '[^a-z0-9]+', ' ', 'g') = $1
OR regexp_replace(lower(coalesce(raw->>'dba_name','')), '[^a-z0-9]+', ' ', 'g') = $1
)
LIMIT 5
`,
[norm]
);
if (!r.rowCount) return false;
for (const row of r.rows) {
// 1. business_contacts (legacy audit log)
await query(
`
INSERT INTO business_contacts (
business_id, role, person_name, source, source_url, confidence, notes
) VALUES ($1, 'chamber_member', $2, 'chamber_directory', $3, 0.85, $4)
ON CONFLICT DO NOTHING
`,
[row.id, member.name, member.url, member.chamber]
);
// 2. business_enrichment.chamber_memberships (JSONB merge — never overwrite
// other chambers already recorded for this business; same pattern as
// ad_signals.ts:118-127).
const blob: Record<string, { url: string; city?: string; website?: string; scraped_at: string }> = {};
blob[member.chamber] = {
url: member.url,
...(member.city ? { city: member.city } : {}),
...(member.website ? { website: member.website } : {}),
scraped_at: new Date().toISOString(),
};
await query(
`
INSERT INTO business_enrichment (business_id, chamber_memberships)
VALUES ($1, $2)
ON CONFLICT (business_id) DO UPDATE
SET chamber_memberships = COALESCE(business_enrichment.chamber_memberships, '{}'::jsonb) || EXCLUDED.chamber_memberships
`,
[row.id, JSON.stringify(blob)]
);
}
return true;
}
// ---------- Driver ----------
async function scrapeChamber(ch: Chamber): Promise<{ scraped: number; matched: number }> {
if (ch.platform === 'chambermaster') {
let scraped = 0,
matched = 0;
for (const letter of ALPHA) {
const members = await fetchAlpha(ch.base, letter);
scraped += members.length;
for (const m of members) {
if (await matchAndTag({ ...m, chamber: ch.name })) matched++;
}
await new Promise((r) => setTimeout(r, 300));
}
return { scraped, matched };
}
// wvwc_custom
const members = await fetchAllWvwc(ch.base);
let matched = 0;
for (const m of members) {
if (await matchAndTag({ ...m, chamber: ch.name })) matched++;
}
return { scraped: members.length, matched };
}
async function main() {
let totalScraped = 0;
let totalMatched = 0;
for (const ch of CHAMBERS) {
console.log(`[chambers] ${ch.name} · ${ch.platform} · ${ch.base}`);
try {
const { scraped, matched } = await scrapeChamber(ch);
console.log(`[chambers] ${ch.name}: ${scraped} members scraped, ${matched} matched corridor businesses`);
totalScraped += scraped;
totalMatched += matched;
} catch (e: any) {
console.error(`[chambers] ${ch.name}: ERROR ${e?.message || e}`);
}
}
console.log(`[chambers] done · ${totalScraped} total members scraped, ${totalMatched} matched`);
await pool.end();
}
main().catch((e) => {
console.error('[chambers]', e);
process.exit(1);
});