← back to Homesonspec
collectors/berkeley-building/src/index.ts
529 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Berkeley Building Co. adapter — the Boise, IDAHO (Treasure Valley) site-built
* spec builder in the Clayton Properties Group family (slug "berkeley-building";
* recon + built 2026-08-30, TK-10487). Same architecture class as the arbor +
* silverthorne + elite-homes adapters: Clayton has no unified site-built feed, so
* each brand gets its own adapter on its own domain, and all share the SAME
* React-SSR platform (react-helmet / `data-reactid` markup + per-home
* ["SingleFamilyResidence","Product"] JSON-LD + a HomeOverview spec list).
*
* SITE IDENTITY (confirmed — the CPG "Berkeley Building Co.", not a namesake):
* https://www.berkeleybuildingco.com — the per-home JSON-LD carries
* `brand.name === "Berkeley Building"`, and every captured home sits in the
* BOISE / Treasure Valley IDAHO metro (Meridian / Nampa / Caldwell / Eagle /
* Kuna / Star, all "…, ID 83xxx"). The homepage references Berkeley Building
* and the site is on the identical Clayton platform as the sister brands.
*
* HOST NOTE: robots.txt is FULLY OPEN ("User-agent: * Allow: /") and references
* the sitemap at https://www.berkeleybuildingco.com/sitemap.xml. The sitemap
* lists the `www.` form of every URL; we keep it as the record sourceUrl.
*
* WIRE FORMAT: berkeleybuildingco.com is the SAME React SSR platform as the
* sister brands, served GZIP'd (~380 KB decompressed per home; the shared
* LiveFetcher sends Accept-Encoding + auto-decompresses — a plain non-decoding
* GET returns binary garbage, the trap that bit sister-site recon; every recon
* curl here used --compressed). NO browser, NO bot-wall, NO XHR/WS for the facts.
* A plain honest-UA GET returns the complete page.
*
* MACHINE-READABLE SOURCE (preferred per the standing "own data over fragile
* DOM" lesson): each per-home page carries a per-home JSON-LD block typed
* ["SingleFamilyResidence","Product"] that is the authoritative carrier of the
* facts most prone to DOM rot —
* - address (streetAddress / addressLocality / addressRegion / postalCode),
* - geo (latitude / longitude),
* - price (offers[].price, a real integer, NOT a "$" DOM string),
* - community (containedIn.name, e.g. "Lavender Place"),
* - a STABLE per-home id (productId === sku, a 24-hex builder document id).
* We prefer JSON-LD for all of the above and fall back to the DOM only when a
* field is genuinely absent from it.
*
* SELECTOR DRIFT vs the elite-homes / arbor sister adapters (documented so a
* future edit doesn't "fix" it back to the elite selectors):
* - The PRIMARY spec list uses DOUBLE-underscore BEM here:
* `HomeOverview__iconListItem` (elite/arbor use SINGLE-underscore
* `HomeOverview_iconListItem`). The value lives in a
* `<span class="HomeOverview__iconListValue">` — NOT a `<b>` like elite. Baths
* render react-text-split ("2" + ".5") inside that span → posDec collapses to
* 2.5. The PRIMARY list is only the FIRST `HomeOverview__iconList` <ul>;
* sibling `HomeCard__iconList` lists (other homes in the community) are never
* read for the primary home's specs.
* - The primary list carries exactly Beds / Baths / SQ FT / Stories — NO Floor
* Plan item, NO Lot #, NO Garage. Floor plan is a separate
* `Floor Plan</span><a…>Violet</a>`. lotNumber and garageSpaces are honest
* nulls (the page publishes neither).
* - STATUS: there is NO `Carousel_h2Lead` / `HomeOverview_lead` status banner
* on the primary block (elite's authoritative source). Instead the primary
* home ALSO appears in the community's home grid as its OWN
* `<a href="{this-detail-path}"><div class="HomeCard__status">…</div>` card,
* and that self-card status IS the authoritative user-facing status
* ("Quick Move-In!", "Model Home", "Under Construction", "Available <Month
* Year>", "Pending"). We resolve status by matching the HomeCard whose href
* equals THIS page's own detail path (from the JSON-LD `url`) — never a
* sibling card. statusFromLabel maps the phrase to the enum; "Pending"
* (contract-pending, not a construction stage) and any unrecognized phrase
* resolve to an honest null, never a guess.
*
* Honest nulls (genuinely absent from the page, NOT fabricated):
* - garageSpaces: no garage field on the primary block → null.
* - lotNumber: no lot # on the primary block → null.
* - price: MODEL HOMES carry no offers[].price on this site → honest null price
* (4 of 12 captured homes are model homes with no price — a real, not padded,
* honest-null case; price coverage is genuinely < 100%).
* - estCompletionDate: no per-home ISO completion date on the page → null.
*
* Facts-only: images OMITTED (buildercloud photos exist but v1 stays images-off /
* mediaRights=NONE per HomesOnSpec policy). Plain HTTP; one detail page == one
* inventory_home.
*
* REAL INVENTORY (honest): the sitemap lists ~35 three-segment per-home detail
* pages across the six ID cities (plus a handful of /homes/undefined/… partials
* we skip). We capture a diverse 12-home fixture sample; the selftest floor is
* the REAL captured count (12), never padded.
*
* Batch control: BERKELEY_PAGE_LIMIT caps per-home pages/run (default 60).
* Optional BERKELEY_CITY filters the sitemap URL list by the first /homes/{city}
* slug (comma-list, e.g. "meridian,nampa").
*/
const BUILDER_SLUG = "berkeley-building";
const ORIGIN = "https://www.berkeleybuildingco.com";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const CITY_FILTER = (process.env.BERKELEY_CITY ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.BERKELEY_PAGE_LIMIT ?? "60");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite integer, or null. Never guesses; 0 / negative / NaN → null.
// Strips non-digits, so "1,595" → 1595.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths may be 2.5), or null. Tolerates the "2 .5" spacing.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/\s+/g, "").replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<!--[\s\S]*?-->/g, "")
.replace(/<[^>]+>/g, "")
.replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
interface BerkeleyHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
planName: string | null;
lotNumber: string | null; // not published on the primary block → always null (honest)
price: number | null; // model homes carry no offers price → honest null
beds: number | null;
bathsTotal: number | null; // decimal, half-baths folded (site publishes a single "2.5")
sqft: number | null;
stories: number | null;
garages: number | null; // not published → always null (honest)
lat: number | null;
lon: number | null;
phone: string | null;
status: "PLANNED" | "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
builderInventoryId: string | null;
}
/** Pull every parsed JSON-LD object out of the page (flattening arrays + @graph). */
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const top = Array.isArray(data) ? data : [data];
for (const it of top) {
if (it && typeof it === "object") {
const g = (it as Record<string, unknown>)["@graph"];
if (Array.isArray(g)) {
for (const node of g) if (node && typeof node === "object") out.push(node as Record<string, unknown>);
} else {
out.push(it as Record<string, unknown>);
}
}
}
}
return out;
}
function hasType(obj: Record<string, unknown>, token: string): boolean {
const t = obj["@type"];
const types = Array.isArray(t) ? t : [t];
return types.some((x) => x === token);
}
/** The per-home ["SingleFamilyResidence","Product"] JSON-LD object, or null. */
function homeJsonLd(html: string): Record<string, unknown> | null {
for (const o of jsonLdObjects(html)) {
if (hasType(o, "SingleFamilyResidence") || (hasType(o, "Product") && o["address"])) return o;
}
return null;
}
function asObj(v: unknown): Record<string, unknown> | null {
return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
}
function asNum(v: unknown): number | null {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) ? n : null;
}
/**
* Map a Berkeley HomeCard__status phrase (the self-card status) to the schema
* constructionStatus enum. Exported so the selftest can exercise every branch.
*
* - "Quick Move-In!" / "Move-In Ready" / "Model Home" / "Available Now" →
* MOVE_IN_READY (a model home is a completed, standing home).
* - "Under Construction" / "Being Built" / "Available <Month Year>" (a dated
* future-completion availability) → UNDER_CONSTRUCTION.
* - "Coming Soon" / "To Be Built" / "Pre-Sale" → PLANNED.
* - "Pending" (contract-pending — NOT a construction stage) and any
* unrecognized / absent phrase → honest null, never guessed.
*/
export function statusFromLabel(label: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
if (!label) return null;
const p = label.toLowerCase();
// "Pending" is a sale state, not a construction stage → honest null.
if (p.includes("pending")) return null;
// NOTE (TK-10487 Cody): "Model Home" -> MOVE_IN_READY is a CONSTRUCTION-status
// claim (the home is complete + standing), NOT a sales-availability claim — the
// 4 model homes in the sample carry price=null. A downstream consumer must not
// render "MOVE_IN_READY + no price" as "for sale now"; the field is
// constructionStatus, and price=null is the honest "not listed for sale" signal.
if (
p.includes("move-in") ||
p.includes("move in") ||
p.includes("quick move") ||
p.includes("model home") ||
p.includes("available now")
)
return "MOVE_IN_READY";
if (p.includes("under construction") || p.includes("being built") || /available\s+[a-z]+\s+\d{4}/.test(p))
return "UNDER_CONSTRUCTION";
if (p.includes("coming soon") || p.includes("to be built") || p.includes("pre-sale") || p.includes("presale"))
return "PLANNED";
return null;
}
// Minimal ZIP3-range guard for Berkeley's operating state (ID — Boise / Treasure
// Valley metro; ID ZIP3 spans 832-838). If a page ever publishes a ZIP whose ZIP3
// can't belong to the parsed state, drop it to an honest null rather than
// propagate a self-contradictory fact. Extensible.
const STATE_ZIP3: Record<string, [number, number][]> = {
ID: [[832, 838]],
OR: [[970, 979]],
WA: [[980, 994]],
};
function zipConsistentWithState(zip: string | null, state: string | null): boolean {
if (!zip || !state) return true;
const ranges = STATE_ZIP3[state.toUpperCase()];
if (!ranges) return true;
const z3 = Number(zip.slice(0, 3));
return ranges.some(([lo, hi]) => z3 >= lo && z3 <= hi);
}
/**
* The self-card status: the community home-grid HomeCard whose anchor href equals
* THIS page's own detail path carries the authoritative status in its
* <div class="HomeCard__status">. We match on the exact self-path so a sibling
* card can never leak its status onto the primary home. Returns the raw phrase.
*
* PLATFORM ASSUMPTION (TK-10487 Cody): this assumes HomeCard__status is the
* IMMEDIATE first child of the anchor (the current Clayton React-SSR output). If
* that platform ever wraps the status in an intermediate element, this drops to
* null on ALL homes — a LOUD failure the selftest's `cov.status === n` catches,
* not a silent mislabel. Isolation is verified by the same-page distinct-status
* test in selftest.test.ts.
*/
export function selfCardStatusLabel(html: string, selfPath: string): string | null {
if (!selfPath) return null;
// Escape the path for a literal regex; allow an optional trailing slash.
const esc = selfPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const re = new RegExp(
`href="${esc}/?"[^>]*>\\s*<div class="HomeCard__status[^"]*"[^>]*>([^<]*)<\\/div>`,
"i",
);
return clean(html.match(re)?.[1] ?? null);
}
/**
* Parse ONE Berkeley per-home page into a home, or null if it isn't a resolvable
* per-home detail page. Address/geo/price/community/id come from the per-home
* JSON-LD (authoritative); beds/baths/stories/sqft come from the PRIMARY home's
* HomeOverview__ list (never a sibling HomeCard); plan from the Floor Plan link;
* status from the matching self-card. Garage + lot# are not published → honest
* nulls.
*/
export function parseHome(html: string, pageUrl: string): BerkeleyHome | null {
const ld = homeJsonLd(html);
if (!ld) return null; // no per-home Product JSON-LD → not a resolvable detail page
// ---- authoritative address from JSON-LD PostalAddress ----
const addr = asObj(ld["address"]);
const street = clean(addr?.["streetAddress"]);
if (!street) return null; // a Product block with no street isn't a home listing
const city = clean(addr?.["addressLocality"]);
const state = normalizeStateCode(clean(addr?.["addressRegion"]));
const zipRaw = (clean(addr?.["postalCode"]) ?? "").match(/\d{5}/)?.[0] ?? null;
const zip = zipConsistentWithState(zipRaw, state) ? zipRaw : null;
// ---- authoritative geo from JSON-LD GeoCoordinates ----
const geo = asObj(ld["geo"]);
const lat = asNum(geo?.["latitude"]);
const lon = asNum(geo?.["longitude"]);
// ---- authoritative price from JSON-LD offers[].price (null on model homes) ----
const offers = ld["offers"];
const firstOffer = Array.isArray(offers) ? asObj(offers[0]) : asObj(offers);
const price = posInt(firstOffer?.["price"]);
// ---- authoritative community from JSON-LD containedIn.name ----
const containedIn = asObj(ld["containedIn"]);
const community = clean(containedIn?.["name"]);
// ---- authoritative stable per-home id from JSON-LD productId / sku ----
const builderInventoryId = clean(ld["productId"]) ?? clean(ld["sku"]);
// ---- self-path (from the JSON-LD url) for the status self-card match ----
const selfUrl = clean(ld["url"]) ?? pageUrl;
const selfPath = selfUrl.replace(/^https?:\/\/[^/]+/, "").replace(/[?#].*$/, "");
// ---- PRIMARY home specs: the FIRST HomeOverview__iconList <ul> (double-BEM;
// NOT sibling HomeCard__ lists). Value is in a HomeOverview__iconListValue
// span (baths react-text-split "2"+".5"); label in HomeOverview__iconListLabel.
// Carries only Beds / Baths / SQ FT / Stories — no garage, no lot #. ----
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
let stories: number | null = null;
const primaryUl = html.match(/<ul[^>]*class="[^"]*HomeOverview__iconList[^"]*"[^>]*>([\s\S]*?)<\/ul>/i);
if (primaryUl) {
for (const m of primaryUl[1]!.matchAll(/<li\s+class="HomeOverview__iconListItem[^"]*"[^>]*>([\s\S]*?)<\/li>/gi)) {
const item = m[1] ?? "";
const value = clean(item.match(/HomeOverview__iconListValue[^>]*>([\s\S]*?)<\/span>/i)?.[1] ?? null);
const label = (clean(item.match(/HomeOverview__iconListLabel[^>]*>([\s\S]*?)<\/span>/i)?.[1] ?? null) ?? "").toLowerCase();
if (/^beds?\b/.test(label)) beds = posInt(value);
else if (/^baths?\b/.test(label)) bathsTotal = posDec(value);
else if (/^stor(?:y|ies)\b/.test(label)) stories = posInt(value);
else if (/^sq\.?\s*ft\.?\b/.test(label)) sqft = posInt(value);
}
}
// ---- floor-plan name: the HomeOverview "Floor Plan" item's <a>Violet</a> ----
const planItem = html.match(/Floor Plan<\/span>\s*<a[^>]*>([^<]+)<\/a>/i);
const planName = clean(planItem?.[1] ?? null);
// ---- status: the matching self-card HomeCard__status is authoritative ----
const status = statusFromLabel(selfCardStatusLabel(html, selfPath));
// ---- sales phone: first tel: link on the page (consultant / office) ----
const phone = clean(html.match(/href="tel:(\+?[\d]+)"/i)?.[1] ?? null);
// garageSpaces + lotNumber: not published on Berkeley's primary block → honest null.
const garages = null;
const lotNumber = null;
return {
url: pageUrl,
street,
city,
state,
zip,
community,
planName,
lotNumber,
price,
beds,
bathsTotal,
sqft,
stories,
garages,
lat,
lon,
phone,
status,
builderInventoryId,
};
}
/**
* A per-home inventory DETAIL URL: /homes/{city}/{community}/{address} — exactly
* THREE path segments after /homes/ (same shape as elite-homes; NOTE arbor +
* silverthorne use FOUR — Berkeley's paths carry no state/metro segment). This
* guard rejects the 1-seg /homes index, the 2-seg city listing pages, and any
* /undefined/ partial (the sitemap carries several /homes/undefined/… entries).
*/
function isDetailUrl(u: string): boolean {
const tail = u.split("/homes/")[1];
if (!tail) return false;
const segs = tail.replace(/\/$/, "").split(/[?#]/)[0]!.split("/");
if (segs.length !== 3) return false;
return segs.every((s) => !!s && s !== "undefined");
}
/** The evidence snippet stamped on the constructionStatus field — the raw
* self-card status phrase, or null when no matching self-card is present. */
function statusEvidence(html: string, selfPath: string): string | null {
return selfCardStatusLabel(html, selfPath);
}
export const berkeleyBuildingAdapter: SourceAdapter = {
key: "berkeley-building-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter(isDetailUrl);
if (CITY_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const c = u.split("/homes/")[1]?.split("/")[0]?.toLowerCase();
return c ? CITY_FILTER.includes(c) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
// Recompute self-path for the status evidence snippet (mirrors parseHome).
const ld = homeJsonLd(html);
const selfUrl = clean(ld?.["url"]) ?? page.url;
const selfPath = selfUrl.replace(/^https?:\/\/[^/]+/, "").replace(/[?#].*$/, "");
const evid = statusEvidence(html, selfPath);
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
// The REAL street address (never a "Lot X" fallback) — from the per-home
// JSON-LD PostalAddress.streetAddress. `?? h.url` never fires because
// parseHome already returned null when street was absent, but it keeps
// the hint type-safe. Parenthesized per the standing rule.
address: (h.street ?? h.url),
builderInventoryId: h.builderInventoryId ?? h.url,
planName: h.planName ?? undefined,
lotNumber: h.lotNumber ?? undefined,
lat: h.lat ?? undefined,
lon: h.lon ?? undefined,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price}` : null,
page.url,
h.price != null ? `Offer price $${h.price}` : null,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url), // honest null (not published)
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Berkeley Building single-family inventory home"),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
evid,
page.url,
evid,
),
estCompletionDate: fv<string>(null, null, page.url), // no per-home ISO completion date on the page
lotNumber: fv(h.lotNumber, h.lotNumber, page.url), // honest null (not published)
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
planName: fv(h.planName, h.planName, page.url),
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};