← back to Homesonspec
collectors/beazer/src/index.ts
484 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Beazer Homes adapter — public JSON-API + SSR-flight source (recon 2026-07-28).
*
* beazer.com is a Next.js app over an Optimizely (Episerver) CMS. Inventory is
* collected in three layers, all no-auth, robots.txt = Allow / :
*
* 1. AREAS. The sitemap (https://www.beazer.com/sitemap.xml) enumerates the
* 17 metro landing pages as depth-0 URLs of the form /<area>-<ST>
* (e.g. /atlanta-GA, /houston-TX). We read the sitemap, keep those, and
* derive the 2-letter state from the slug suffix.
*
* 2. METRO GUID → COMMUNITIES. Each area landing page carries exactly one
* Optimizely `GuidValue` — its own MetroDetailPage node id — which is the
* `ancestorMainId` the communities API keys on:
* GET /api/get-communities-per-area
* ?skip=0&limit=100&orderBy=FeatureOrder:ASC&filters={}
* &ancestorMainId=<METRO_GUID>
* → { items:[ CommunityDetailPage… ], total }. Each community item is an
* AGGREGATE (MinimumPrice + Bed/SqFt/Baths RANGES, not a single home) but
* carries the community's Name, Url, Latitude, Longitude, and HasHomes.
* (limit is server-capped at 100; every metro fits in one page today, but
* we page defensively on `total`.)
*
* 3. PER-HOME QMI. For each community with HasHomes=true we GET its Url. The
* individual quick-move-in homes are server-rendered into the Next.js
* flight payload as `SpecDetailPage`-shaped objects, each a REAL single
* home — distinct StreetAddress, distinct HomePrice, own HomesiteNumber,
* Beds/Baths/SquareFoot, Availability, HomeType, and a ParentPlanContent
* floorplan name. These are the records we publish. We deliberately do NOT
* use the /<area>/<city>/<community>/<plan> "PlanDetailPage" URLs — those
* are FLOORPLAN pages listing many homes at many prices (the Dream Finders
* hollow-row trap); the per-community QMI objects are the true single-home
* facts and carry a per-home /<...>/<plan>/<homesite> Url.
*
* The community's Latitude/Longitude (US lon is NEGATIVE — signs preserved) is
* not on the home object, so the fetch layer stashes the community context
* (name, lat, lon, canonical url) in the RawPage URL fragment; extract() reads
* it back, keeping extraction pure (same bytes + url → same records).
*
* Facts-only: home & plan objects carry InteriorImages/ElevationImages URLs;
* those are intentionally dropped (images = []).
*
* Batch control: BEAZER_PAGE_LIMIT (max community PAGES fetched, default 10)
* Optional area: BEAZER_AREA (a single slug, e.g. houston-TX)
*/
const ORIGIN = "https://www.beazer.com";
const SITEMAP_URL = `${ORIGIN}/sitemap.xml`;
const COMMUNITIES_API = `${ORIGIN}/api/get-communities-per-area`;
const BUILDER_SLUG = "beazer";
const API_PAGE_SIZE = 100; // server max for get-communities-per-area
const PAGE_LIMIT = Number(process.env.BEAZER_PAGE_LIMIT ?? 10);
const AREA_FILTER = (process.env.BEAZER_AREA ?? "").trim().toLowerCase() || null;
// A depth-0 metro landing URL: /<slug>-<ST> with no further path segments.
// The slug may contain double-dashes (e.g. maryland--dc-MD, virginia--dc-VA),
// so allow hyphens freely; the trailing -<ST> two-letter state is the anchor.
const AREA_SLUG_RE = /^https:\/\/www\.beazer\.com\/([a-z0-9][a-z0-9-]*-([A-Z]{2}))$/;
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths may legitimately be small), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
// Availability is an ISO datetime ("2025-10-28T04:00:00Z"); publish wants a
// plain ISO date. Reject anything that isn't a real date in a plausible window.
const isoDate = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const d = new Date(s);
if (Number.isNaN(d.getTime())) return null;
const year = d.getUTCFullYear();
if (year < 2000 || year > 2100) return null;
return d.toISOString().slice(0, 10);
};
/** Beazer HomeType strings → our homeType enum. Unknown/blank → null (not guessed). */
function homeType(v: unknown): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" | "DUPLEX" | "OTHER" | null {
const s = (str(v) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("town")) return "TOWNHOME";
if (s.includes("condo")) return "CONDO";
if (s.includes("duplex")) return "DUPLEX";
if (s.includes("single family") || s.includes("single-family")) return "SINGLE_FAMILY";
return "OTHER";
}
// ---------------------------------------------------------------------------
// Community context passed from the fetch layer to extract() via URL fragment.
// ---------------------------------------------------------------------------
interface CommunityCtx {
name: string;
canonicalUrl: string;
lat: number | null;
lon: number | null;
}
function encodeCommunityUrl(canonicalUrl: string, ctx: CommunityCtx): string {
const frag = Buffer.from(
JSON.stringify({ n: ctx.name, u: ctx.canonicalUrl, la: ctx.lat, lo: ctx.lon }),
"utf8",
).toString("base64url");
return `${canonicalUrl}#beazer-community=${frag}`;
}
function decodeCommunityUrl(url: string): { canonicalUrl: string; ctx: CommunityCtx | null } {
const hashIdx = url.indexOf("#beazer-community=");
if (hashIdx < 0) return { canonicalUrl: url, ctx: null };
const canonicalUrl = url.slice(0, hashIdx);
try {
const raw = url.slice(hashIdx + "#beazer-community=".length);
const obj = JSON.parse(Buffer.from(raw, "base64url").toString("utf8")) as {
n?: string; u?: string; la?: number | null; lo?: number | null;
};
return {
canonicalUrl,
ctx: {
name: str(obj.n) ?? "",
canonicalUrl: str(obj.u) ?? canonicalUrl,
lat: typeof obj.la === "number" && Number.isFinite(obj.la) && obj.la !== 0 ? obj.la : null,
lon: typeof obj.lo === "number" && Number.isFinite(obj.lo) && obj.lo !== 0 ? obj.lo : null,
},
};
} catch {
return { canonicalUrl, ctx: null };
}
}
// ---------------------------------------------------------------------------
// Metro-GUID + communities API helpers.
// ---------------------------------------------------------------------------
/** Pull the single Optimizely metro GUID out of an area landing page's HTML. */
function metroGuidFromArea(html: string): string | null {
const dec = html.replace(/\\"/g, '"');
const m = dec.match(/"GuidValue":"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"/);
return m ? m[1]! : null;
}
interface CommunityItem {
Name?: string;
CommunityName?: string;
Url?: string;
Latitude?: number;
Longitude?: number;
HasHomes?: boolean;
}
function communitiesApiUrl(guid: string, skip: number): string {
const q = new URLSearchParams({
skip: String(skip),
limit: String(API_PAGE_SIZE),
orderBy: "FeatureOrder:ASC",
filters: "{}",
ancestorMainId: guid,
});
return `${COMMUNITIES_API}?${q.toString()}`;
}
function parseCommunities(body: string): { items: CommunityItem[]; total: number } {
try {
const j = JSON.parse(body) as { items?: CommunityItem[]; total?: number };
return { items: Array.isArray(j.items) ? j.items : [], total: Number(j.total ?? 0) };
} catch {
return { items: [], total: 0 };
}
}
// ---------------------------------------------------------------------------
// Per-home QMI extraction from a community page's SSR flight payload.
// The homes are proper JSON objects streamed inside self.__next_f chunks with
// escaped quotes; we un-escape then brace-match every object that has both a
// StreetAddress and a HomePrice (that pair uniquely identifies a QMI home).
// ---------------------------------------------------------------------------
interface BeazerHome {
StreetAddress?: string;
City?: string;
StateID?: string;
Zip?: string;
HomesiteNumber?: string;
Beds?: number;
Baths?: number;
SquareFoot?: number;
HomePrice?: number;
HomeType?: string;
Availability?: string;
IsModel?: boolean;
Url?: string;
ParentPlanContent?: {
PlanDetailPage?: { item?: { PlanName?: string } };
};
}
function extractHomes(html: string): BeazerHome[] {
const dec = html.replace(/\\"/g, '"').replace(/\\u0026/g, "&").replace(/\\\//g, "/");
const back = (i: number): number => {
let d = 0;
for (let k = i; k >= 0; k--) {
if (dec[k] === "}") d++;
else if (dec[k] === "{") {
if (d === 0) return k;
d--;
}
}
return -1;
};
const fwd = (s: number): number => {
let d = 0;
for (let k = s; k < dec.length; k++) {
if (dec[k] === "{") d++;
else if (dec[k] === "}") {
d--;
if (d === 0) return k;
}
}
return -1;
};
const homes: BeazerHome[] = [];
const seen = new Set<number>();
const re = /"StreetAddress":/g;
let m: RegExpExecArray | null;
while ((m = re.exec(dec))) {
const s = back(m.index);
if (s < 0 || seen.has(s)) continue;
seen.add(s);
const e = fwd(s);
if (e < 0) continue;
let obj: BeazerHome;
try {
obj = JSON.parse(dec.slice(s, e + 1)) as BeazerHome;
} catch {
continue;
}
// A home object always carries a numeric HomePrice AND a StreetAddress.
// Anything else that mentions "StreetAddress" (e.g. a sales-office block)
// lacks HomePrice and is skipped.
if (obj.StreetAddress && typeof obj.HomePrice === "number") homes.push(obj);
}
return homes;
}
export const beazerAdapter: SourceAdapter = {
key: "beazer-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// 1) Enumerate the metro areas from the sitemap.
let sitemap: RawPage;
try {
sitemap = await fetcher.fetch(SITEMAP_URL);
} catch (error) {
console.warn(` beazer sitemap: ${error instanceof Error ? error.message : String(error)}`);
return;
}
const locs = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)].map((x) => x[1]!);
const areas: { url: string; state: string }[] = [];
for (const loc of locs) {
const m = loc.match(AREA_SLUG_RE);
if (!m) continue;
if (AREA_FILTER && !m[1]!.toLowerCase().endsWith(AREA_FILTER) && m[1]!.toLowerCase() !== AREA_FILTER) continue;
areas.push({ url: loc, state: m[2]! });
}
let pagesYielded = 0;
const limit = Math.max(1, PAGE_LIMIT);
for (const area of areas) {
if (pagesYielded >= limit) return;
// 2) Area page → metro GUID.
let areaPage: RawPage;
try {
areaPage = await fetcher.fetch(area.url);
} catch (error) {
console.warn(` beazer area ${area.url}: ${error instanceof Error ? error.message : String(error)}`);
continue;
}
const guid = metroGuidFromArea(areaPage.body.toString("utf8"));
if (!guid) {
console.warn(` beazer area ${area.url}: no metro GUID found — skipped`);
continue;
}
// 2b) Page the communities API for this metro.
const communities: CommunityItem[] = [];
let skip = 0;
for (;;) {
let apiPage: RawPage;
try {
apiPage = await fetcher.fetch(communitiesApiUrl(guid, skip));
} catch (error) {
console.warn(` beazer communities ${area.url} skip=${skip}: ${error instanceof Error ? error.message : String(error)}`);
break;
}
const { items, total } = parseCommunities(apiPage.body.toString("utf8"));
communities.push(...items);
skip += API_PAGE_SIZE;
if (items.length < API_PAGE_SIZE || (total > 0 && skip >= total)) break;
}
// 3) Each community with homes → community page → QMI home objects.
for (const c of communities) {
if (pagesYielded >= limit) return;
const canonicalUrl = str(c.Url);
if (!canonicalUrl || c.HasHomes !== true) continue;
const name = str(c.CommunityName) ?? str(c.Name);
if (!name) continue;
const commCtx: CommunityCtx = {
name,
canonicalUrl,
lat: typeof c.Latitude === "number" && Number.isFinite(c.Latitude) && c.Latitude !== 0 ? c.Latitude : null,
lon: typeof c.Longitude === "number" && Number.isFinite(c.Longitude) && c.Longitude !== 0 ? c.Longitude : null,
};
let page: RawPage;
try {
page = await fetcher.fetch(canonicalUrl);
} catch (error) {
console.warn(` beazer community ${canonicalUrl}: ${error instanceof Error ? error.message : String(error)}`);
continue;
}
// Re-stamp the RawPage url with the encoded community context so extract()
// can attach the community's name + geo (which the home objects lack).
yield { ...page, url: encodeCommunityUrl(canonicalUrl, commCtx) };
pagesYielded++;
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const { canonicalUrl, ctx } = decodeCommunityUrl(page.url);
const homes = extractHomes(page.body.toString("utf8"));
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
// Community context: prefer the fetch-stamped fragment; without it (e.g. a
// raw fixture with no fragment) we can't attach homes to a community FK.
const community = ctx?.name ?? null;
const commLat = ctx?.lat ?? null;
const commLon = ctx?.lon ?? null;
if (homes.length === 0) {
// Not an error condition per se (some HasHomes communities gate their
// QMI list behind a VIP flow); report it honestly so it's visible.
return { records: [], errors: [{ url: canonicalUrl, reason: "no QMI home objects in community page" }] };
}
if (!community) {
return { records: [], errors: [{ url: canonicalUrl, reason: "community page missing community context — cannot attach homes" }] };
}
// Emit the community FIRST (publish creates the FK target the homes need),
// once per page. City/state/zip come from the homes (community item has no
// street address); we take them from the first home that carries them.
const firstWithGeo = homes.find((h) => str(h.City) || str(h.StateID) || zip5(h.Zip));
const commCity = str(firstWithGeo?.City);
const commState = normalizeStateCode(str(firstWithGeo?.StateID));
const commZip = zip5(firstWithGeo?.Zip);
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, canonicalUrl, "Beazer community name"),
street: fv<string>(null, null, canonicalUrl),
city: fv(commCity, commCity, canonicalUrl),
state: fv(commState, str(firstWithGeo?.StateID), canonicalUrl),
zip: fv(commZip, str(firstWithGeo?.Zip), canonicalUrl),
county: fv<string>(null, null, canonicalUrl),
metro: fv<string>(null, null, canonicalUrl),
lat: fv(commLat, commLat === null ? null : String(commLat), canonicalUrl, commLat === null ? null : "community Latitude"),
lon: fv(commLon, commLon === null ? null : String(commLon), canonicalUrl, commLon === null ? null : "community Longitude"),
hoaFeeMonthly: fv<number>(null, null, canonicalUrl),
schoolDistrict: fv<string>(null, null, canonicalUrl),
ageRestricted: fv<boolean>(null, null, canonicalUrl),
},
});
for (const h of homes) {
const address = str(h.StreetAddress);
if (!address) {
errors.push({ url: canonicalUrl, reason: "home missing StreetAddress — skipped" });
continue;
}
// Beazer flags model/display homes; those are not for-sale inventory.
if (h.IsModel === true) {
errors.push({ url: canonicalUrl, reason: `home ${address} is a model home — skipped` });
continue;
}
const homeUrl = str(h.Url) ?? canonicalUrl;
const city = str(h.City) ?? commCity;
const state = normalizeStateCode(str(h.StateID)) ?? commState;
const zip = zip5(h.Zip) ?? commZip;
const price = posNum(h.HomePrice);
const beds = posNum(h.Beds);
const baths = nonNegNum(h.Baths);
const sqft = posNum(h.SquareFoot);
const homesite = str(h.HomesiteNumber);
const plan = str(h.ParentPlanContent?.PlanDetailPage?.item?.PlanName);
const hType = homeType(h.HomeType);
const estCompletion = isoDate(h.Availability);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
lotNumber: homesite ?? undefined,
builderInventoryId: homesite ?? undefined,
lat: commLat ?? undefined,
lon: commLon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, homeUrl, "Beazer home StreetAddress"),
city: fv(city, city, homeUrl),
state: fv(state, str(h.StateID), homeUrl),
zip: fv(zip, str(h.Zip), homeUrl),
price: fv(price, price === null ? null : String(h.HomePrice), homeUrl, price === null ? null : `HomePrice ${h.HomePrice}`),
beds: fv(beds, beds === null ? null : String(h.Beds), homeUrl),
bathsTotal: fv(baths, baths === null ? null : String(h.Baths), homeUrl),
sqft: fv(sqft, sqft === null ? null : String(h.SquareFoot), homeUrl),
stories: fv<number>(null, null, homeUrl),
garageSpaces: fv<number>(null, null, homeUrl),
homeType: fv(hType, str(h.HomeType), homeUrl, hType === null ? null : `HomeType: ${str(h.HomeType)}`),
// Availability is a move-in / completion estimate; we treat any QMI
// home with a real Availability date as move-in-ready inventory.
constructionStatus: fv("MOVE_IN_READY" as const, str(h.Availability), homeUrl, "Beazer quick move-in home"),
estCompletionDate: fv(estCompletion, str(h.Availability), homeUrl),
lotNumber: fv(homesite, homesite, homeUrl, homesite === null ? null : "HomesiteNumber"),
builderInventoryId: fv(homesite, homesite, homeUrl),
// Community-level geo (homes don't carry their own lat/lon).
lat: fv(commLat, commLat === null ? null : String(commLat), homeUrl, commLat === null ? null : "community Latitude"),
lon: fv(commLon, commLon === null ? null : String(commLon), homeUrl, commLon === null ? null : "community Longitude"),
planName: fv(plan, plan, homeUrl),
// facts-only: InteriorImages/ElevationImages exist but are dropped.
images: fv<string[]>([], null, homeUrl),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};