← back to Homesonspec
collectors/brookfield/src/index.ts
422 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Brookfield Residential adapter — Sitecore Discover JSON-API source (recon 2026-07-28).
*
* brookfieldresidential.com is a Next.js SPA over a Sitecore Discover backend —
* the same Discover product family as Meritage/LGI, but bound with Brookfield's
* own real-estate index ("brp_real_estate_data"). Unlike LGI, Brookfield's
* Discover rows are NOT hollow SEO stubs: the per-LOT record carries full,
* varied per-home facts, so a single paginated POST yields everything — no SSR
* fallback needed (verified: 636 homes, 97% real distinct price, 100% beds/
* baths/sqft/street/geo).
*
* POST https://discover.sitecorecloud.io/discover/v2/227641806
* Content-Type: application/json
* body: { context:{ locale:{ country:"us", language:"en" } },
* widget:{ items:[{ rfk_id:"brp_real_estate_data", entity:"map",
* search:{ filter:{ type:"and", filters:[
* {type:eq name:type value:"Lot"}, // individual home
* {type:eq name:shouldcountasquickmovein value:true}, // a real QMI home
* {type:eq name:soldhome value:false}, // still for sale
* {type:eq name:community_country value:"USA"} // US-only (HomesOnSpec scope)
* ]},
* content:{}, limit:100, offset },
* sources:["1125798"] }] } }
*
* The response widget (type:"content_grid", entity:"map") carries
* { total_item, limit, offset, content:[ lot… ] }
* where each LOT is one physical, addressed inventory home:
* addressline1 (street), city, stateorprovince, ziporpostalcode,
* baseprice / minimumprice / maximumprice (per-lot; min==max on a spec home,
* 0/withheld when showbaseprice=false — treat as unpriced, never guessed),
* maximumresidencebedrooms (=min, one plan per lot), maximumtotalbaths +
* maximumresidencefullbaths (half = total - full), maximumsquarefootage,
* maximumstories, maximumresidencegarage, community_name, plan_name,
* community_geo ("lat,lon"), hometype, statusdescription/inferredlotstatus,
* datehomeisplannedtobecomplete, url.
*
* IMPORTANT distinctions verified during recon:
* • `type:Neighborhood` rows are COMMUNITY AGGREGATES (basicprice / min-max
* price band, 128 rows) — NOT individual homes; we deliberately query
* `type:Lot` so every record is one addressed home.
* • `displayprice` is a marketing-suppressed field (always 0 here); the real
* per-lot number is `baseprice` (== minimumprice == maximumprice on a spec).
* • `availableforsale` is NOT a usable server filter (returns 0); sold homes
* are excluded via `soldhome:false` instead. A soldhome row that still leaks
* through is dropped in extract().
* • Prices are USD (country filter = USA); Canadian (CAD) rows are excluded so
* the platform never mixes currencies.
*
* Verified: the `authorization` header is OPTIONAL (200 without it); the
* customer id (227641806) in the path + source id 1125798 are the only
* identifiers. robots.txt allows /new-homes (only App_config/temp/*default.aspx
* are disallowed); the discover host serves no robots.txt (fail-open).
* Facts-only: image_url exists on every record but is intentionally dropped.
*
* Batch control: BROOKFIELD_PAGE_LIMIT (pages of `limit` homes, default 10)
* Optional state: BROOKFIELD_STATE (2-letter, e.g. TX — server-side filter
* on the full stateorprovince name)
*/
const DISCOVER_URL = "https://discover.sitecorecloud.io/discover/v2/227641806";
const SOURCE_ID = "1125798";
const RFK_ID = "brp_real_estate_data";
const BUILDER_SLUG = "brookfield";
const ORIGIN = "https://www.brookfieldresidential.com";
const PAGE_SIZE = 100; // server max per content_grid page
const PAGE_LIMIT = Number(process.env.BROOKFIELD_PAGE_LIMIT ?? 10);
const STATE_FILTER = (process.env.BROOKFIELD_STATE ?? "").trim().toUpperCase() || null;
// The synthesized normalized-page URL carries the offset so snapshots stay distinct.
const PAGE_URL = (offset: number) => `${DISCOVER_URL}#content_grid&offset=${offset}`;
// 2-letter → full state/province name (Discover stores the full name, so a
// BROOKFIELD_STATE=TX must be mapped to "Texas" for the server-side filter).
const STATE_NAMES: Record<string, string> = {
AZ: "Arizona", CA: "California", CO: "Colorado", DE: "Delaware",
MD: "Maryland", NC: "North Carolina", SC: "South Carolina",
TX: "Texas", VA: "Virginia",
};
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages/stories may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
interface DiscoverLot {
id?: string;
type?: string;
soldhome?: boolean;
addressline1?: string;
city?: string;
stateorprovince?: string;
ziporpostalcode?: string | number;
community_country?: string;
baseprice?: number | string | null;
minimumprice?: number | string | null;
maximumprice?: number | string | null;
displayprice?: number | string | null;
showbaseprice?: boolean;
maximumresidencebedrooms?: number;
minimumresidencebedrooms?: number;
maximumtotalbaths?: number;
maximumresidencefullbaths?: number;
maximumsquarefootage?: number;
maximumstories?: number;
maximumresidencegarage?: number;
community_name?: string;
plan_name?: string;
community_geo?: string; // "lat,lon"
hometype?: string;
statusdescription?: string;
salesstatusdescription?: string;
inferredlotstatus?: string;
datehomeisplannedtobecomplete?: string; // ISO-ish local datetime
compositeid?: string;
marketinglotnumber?: string;
url?: string;
}
/** Build the content_grid POST body for a given offset. */
function buildBody(offset: number): unknown {
const filters: unknown[] = [
{ type: "eq", name: "type", value: "Lot" },
// NOTE: we deliberately do NOT filter shouldcountasquickmovein=true. That is Brookfield's
// sales-portal "quick-move-in" view filter, not a data-quality gate — it hides genuinely
// under-construction spec homes. type=Lot + soldhome=false already yields individual homes;
// dropping the QMI filter recovers ~26% more inventory (412 -> 521) and a realistic
// move-in-ready vs under-construction split. constructionStatus() classifies each result.
// [yolo iter-1 contrarian fix, verified 2026-07-28]
{ type: "eq", name: "soldhome", value: false },
{ type: "eq", name: "community_country", value: "USA" },
];
if (STATE_FILTER && STATE_NAMES[STATE_FILTER]) {
filters.push({ type: "eq", name: "stateorprovince", value: STATE_NAMES[STATE_FILTER] });
}
return {
context: { locale: { country: "us", language: "en" } },
widget: {
items: [
{
rfk_id: RFK_ID,
entity: "map",
search: { filter: { type: "and", filters }, content: {}, limit: PAGE_SIZE, offset },
sources: [SOURCE_ID],
},
],
},
};
}
/** Pull the content_grid widget (entity:"map") out of a Discover response body. */
function gridWidget(body: string): { total: number; offset: number; content: DiscoverLot[] } | null {
let parsed: unknown;
try {
parsed = JSON.parse(body);
} catch {
return null;
}
const widgets = (parsed as { widgets?: unknown[] })?.widgets;
if (!Array.isArray(widgets)) return null;
const w = widgets.find(
(x) => (x as { type?: string; rfk_id?: string })?.type === "content_grid" &&
(x as { rfk_id?: string })?.rfk_id === RFK_ID,
) as { total_item?: number; offset?: number; content?: DiscoverLot[] } | undefined;
if (!w) return null;
return {
total: Number(w.total_item ?? 0),
offset: Number(w.offset ?? 0),
content: Array.isArray(w.content) ? w.content : [],
};
}
/** "33.724, -117.739" → { lat, lon }. US lon is negative — signs preserved. */
function parseGeo(loc: unknown): { lat: number | null; lon: number | null } {
const s = str(loc);
if (!s) return { lat: null, lon: null };
const m = s.match(/(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)/);
if (!m) return { lat: null, lon: null };
const lat = Number(m[1]);
const lon = Number(m[2]);
return {
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
lon: Number.isFinite(lon) && lon !== 0 ? lon : null,
};
}
/** The real per-lot price. `baseprice` (== min == max on a spec home) is the
* authoritative number; when the builder suppresses it (showbaseprice=false)
* every price field is 0 — we return null rather than guess a price. */
function lotPrice(lot: DiscoverLot): { value: number | null; raw: string | null } {
const candidates = [lot.baseprice, lot.minimumprice, lot.maximumprice];
for (const c of candidates) {
const p = posNum(c);
if (p !== null) return { value: p, raw: String(c) };
}
return { value: null, raw: null };
}
/** Brookfield lot status → our construction-status enum. `statusdescription`
* is a sale/product phase ("Spec", "To Be Built", "Model"…) and
* `inferredlotstatus` a lifecycle label ("Available Home", "Reserved"…).
* A "Spec"/"inventory" home with a build-complete signal is move-in ready; a
* "To Be Built" / future-dated home is under construction. Anything the feed
* leaves ambiguous → null (never guessed). */
function constructionStatus(lot: DiscoverLot): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const status = (str(lot.statusdescription) ?? "").toLowerCase();
const inferred = (str(lot.inferredlotstatus) ?? "").toLowerCase();
const blob = `${status} ${inferred}`;
if (blob.includes("to be built") || blob.includes("presale") || blob.includes("pre-sale") || blob.includes("coming soon")) {
return "UNDER_CONSTRUCTION";
}
// A completed spec/inventory home that is available/quick-move-in ready.
if (blob.includes("move-in") || blob.includes("move in") || blob.includes("quick move") || blob.includes("completed") || blob.includes("available home")) {
return "MOVE_IN_READY";
}
// "Spec"/"Model"/"Reserved" without a completion signal → don't fabricate a stage.
return null;
}
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
/** Brookfield hometype label → our homeType enum. Observed values: "Single
* Family Home", "Townhome". Unknown labels fall back to OTHER (never guessed
* as single-family). The raw label is carried as evidence on the field. */
type HomeType = "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" | "DUPLEX" | "OTHER";
function homeType(v: unknown): HomeType {
const s = (str(v) ?? "").toLowerCase();
if (!s) return "SINGLE_FAMILY"; // Brookfield's default residential product
if (s.includes("town")) return "TOWNHOME";
if (s.includes("condo")) return "CONDO";
if (s.includes("duplex")) return "DUPLEX";
if (s.includes("single") || s.includes("detached")) return "SINGLE_FAMILY";
return "OTHER";
}
/** Brookfield completion date is a local ISO-ish datetime ("2027-01-19T00:00:00").
* Normalize to a YYYY-MM-DD date, clamped to a sane window so a bogus value
* never reaches the publisher's `new Date()`. */
const isoDate = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (!m) return null;
const year = Number(m[1]);
if (year < 2000 || year > 2100) return null;
return `${m[1]}-${m[2]}-${m[3]}`;
};
/** Make a relative Brookfield url absolute; fall back to the page URL. */
const homeUrl = (lot: DiscoverLot, pageUrl: string): string => {
const u = str(lot.url);
if (!u) return pageUrl;
return u.startsWith("http") ? u : `${ORIGIN}${u}`;
};
export const brookfieldAdapter: SourceAdapter = {
key: "brookfield-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let offset = 0;
for (let pageNo = 0; pageNo < Math.max(1, PAGE_LIMIT); pageNo++) {
let page: RawPage;
try {
page = await fetcher.postJson(PAGE_URL(offset), buildBody(offset));
} catch (error) {
console.warn(` brookfield offset=${offset}: ${error instanceof Error ? error.message : String(error)}`);
return; // a block/403 stops collection (source marked degraded upstream)
}
const grid = gridWidget(page.body.toString("utf8"));
yield page;
const got = grid?.content.length ?? 0;
offset += PAGE_SIZE;
// Stop when the server returns a short page or we've covered total_item.
if (!grid || got < PAGE_SIZE || (grid.total > 0 && offset >= grid.total)) return;
}
},
extract(page: RawPage): ExtractionOutput {
try {
const grid = gridWidget(page.body.toString("utf8"));
if (!grid) {
return { records: [], errors: [{ url: page.url, reason: "no brp_real_estate_data content_grid widget in Discover response" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
for (const lot of grid.content) {
// Defensive: only individual lots, never a leaked aggregate / sold home.
if (lot.type && lot.type !== "Lot") continue;
if (lot.soldhome === true) continue;
const state = normalizeStateCode(str(lot.stateorprovince));
const city = str(lot.city);
const zip = zip5(lot.ziporpostalcode);
const address = str(lot.addressline1);
const community = str(lot.community_name);
const plan = str(lot.plan_name);
const { lat, lon } = parseGeo(lot.community_geo);
const { value: price, raw: priceRaw } = lotPrice(lot);
const beds = posNum(lot.maximumresidencebedrooms);
const totalBaths = nonNegNum(lot.maximumtotalbaths);
const fullBaths = nonNegNum(lot.maximumresidencefullbaths);
// Brookfield reports total + full; derive half = total - full (>=0), then
// present bathsTotal as the reported total (already includes halves).
const bathsTotal = totalBaths;
const halfBaths = totalBaths !== null && fullBaths !== null ? Math.max(0, totalBaths - fullBaths) : null;
const sqft = posNum(lot.maximumsquarefootage);
const stories = posNum(lot.maximumstories);
const garages = nonNegNum(lot.maximumresidencegarage);
const homeId = str(lot.id) ?? str(lot.compositeid);
const url = homeUrl(lot, page.url);
const cStatus = constructionStatus(lot);
const estCompletion = isoDate(lot.datehomeisplannedtobecomplete);
const lotNumber = str(lot.marketinglotnumber);
if (!address) {
errors.push({ url, reason: `lot ${homeId ?? "?"} missing addressline1 — skipped` });
continue;
}
// The publisher requires an inventory home to hang off a community (FK).
if (!community) {
errors.push({ url, reason: `home ${address} has no community_name — cannot attach to a community, skipped` });
continue;
}
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "Discover community_name"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, str(lot.stateorprovince), url),
zip: fv(zip, str(lot.ziporpostalcode), url),
county: fv<string>(null, null, url),
metro: fv<string>(null, null, url),
lat: fv(lat, null, url, lat === null ? null : "Discover community_geo"),
lon: fv(lon, null, url, lon === null ? null : "Discover community_geo"),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
builderInventoryId: homeId ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, url, "Discover lot addressline1"),
city: fv(city, city, url),
state: fv(state, str(lot.stateorprovince), url),
zip: fv(zip, str(lot.ziporpostalcode), url),
price: fv(price, priceRaw, url, price === null ? null : `Discover baseprice ${priceRaw}`),
beds: fv(beds, beds === null ? null : String(lot.maximumresidencebedrooms), url),
bathsTotal: fv(bathsTotal, bathsTotal === null ? null : String(bathsTotal), url, bathsTotal === null ? null : `${fullBaths ?? "?"} full + ${halfBaths ?? 0} half`),
sqft: fv(sqft, sqft === null ? null : String(lot.maximumsquarefootage), url),
stories: fv(stories, stories === null ? null : String(lot.maximumstories), url),
garageSpaces: fv(garages, garages === null ? null : String(lot.maximumresidencegarage), url),
homeType: fv(homeType(lot.hometype), str(lot.hometype), url, str(lot.hometype) ? `Brookfield hometype: ${str(lot.hometype)}` : "Brookfield inventory home"),
constructionStatus: fv(cStatus, str(lot.statusdescription), url, cStatus === null ? null : `status: ${str(lot.statusdescription)} / ${str(lot.inferredlotstatus)}`),
estCompletionDate: fv(estCompletion, str(lot.datehomeisplannedtobecomplete), url),
lotNumber: fv(lotNumber, lotNumber, url),
builderInventoryId: fv(homeId, homeId, url),
lat: fv(lat, null, url, lat === null ? null : "Discover community_geo"),
lon: fv(lon, null, url, lon === null ? null : "Discover community_geo"),
planName: fv(plan, plan, url),
// facts-only: image_url exists in the feed but is intentionally dropped.
images: fv<string[]>([], null, url),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};