← back to Homesonspec
collectors/lgi-homes/src/index.ts
416 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* LGI Homes adapter — Sitecore Discover enumeration + SSR page facts (recon 2026-07-28).
*
* lgihomes.com is a Next.js SPA over a Sitecore XM Cloud / Sitecore Discover
* backend — the same Discover API family as Meritage, but bound DIFFERENTLY:
*
* 1. ENUMERATION (Discover POST). The home-search grid is server-rendered, so
* the Discover call never fires client-side on load. Replaying it directly
* works though — customer id 157298599 in the path, NO `sources` attribute
* (LGI's domain rejects the Meritage `xm_cloud_public_website` source; the
* request only validates when `sources` is omitted, letting the domain use
* its default), rfk_id "rfkid_7" (type:content_grid), entity "content"
* (LGI indexes under `content`, not `home` — the `home` entity is empty),
* context.page.uri "/search" (the bound page), filter type==community.
* This returns ~236 community records. Crucially the Discover CONTENT rows
* are thin SEO index objects (name/url/description only) with NO structured
* price/beds/baths/sqft — the facets exist but the record bodies are hollow.
* So Discover is used ONLY to enumerate the community URLs.
*
* 2. FACTS (SSR page). Each community page embeds a Next.js __NEXT_DATA__ blob
* carrying the real facts: the community's location (communityName,
* address1, city, state, zipCode, latitude, longitude) AND every floorplan
* offered there, each with numberofBedrooms / numberofBathrooms /
* numberofCarGarage / sizeSqft / numberofStories / listPrice (real sale
* price) / monthlyPayment / floorPlanName / floorPlanID / url. Crawling the
* ~236 community pages covers the full national floorplan inventory
* (~1340 floorplans) with 5–6× fewer fetches than crawling every floorplan.
*
* Each LGI "floorplan" is a model design sold at a community (there is no
* per-physical-home street address in the feed), so an inventory_home is keyed
* by community + plan; the community's sales-center address1 is carried as the
* community street. `listPrice` is the real sale price; `monthlyPayment` (a
* financed-payment figure) is intentionally NOT used as price.
*
* Verified: no auth header (Discover 200 without it); robots.txt allows /
* (lgihomes.com); the discover host serves no robots.txt (fail-open).
* Facts-only: image_url exists on every record but is intentionally dropped.
*
* Batch control: LGI_PAGE_LIMIT (max community pages to fetch, default 10)
*/
const DISCOVER_URL = "https://discover.sitecorecloud.io/discover/v2/157298599";
const ORIGIN = "https://www.lgihomes.com";
const BUILDER_SLUG = "lgi-homes";
const RFK_ID = "rfkid_7";
const BOUND_URI = "/search";
const ENUM_PAGE_SIZE = 100; // server max per content_grid page
const PAGE_LIMIT = Number(process.env.LGI_PAGE_LIMIT ?? 10);
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
/** Sitecore field-value unwrap: {value}, {jsonValue:{value}}, or a raw scalar. */
function fieldVal(v: unknown): unknown {
if (v == null || typeof v !== "object") return v;
const o = v as Record<string, unknown>;
if ("jsonValue" in o) {
const jv = o.jsonValue as Record<string, unknown> | null;
return jv && "value" in jv ? jv.value : null;
}
if ("value" in o) return o.value;
if ("path" in o) return o.path; // url objects
return v;
}
/** Unwrap a Sitecore key-value-pair reference object (sale/ad status), which is
* {jsonValue:{ name, displayName, fields:{ Value:{value} } }} — no scalar
* `value`, so fieldVal() returns null on it. Returns the human label or null. */
function kvpLabel(v: unknown): string | null {
if (v == null || typeof v !== "object") return str(v);
const jv = (v as Record<string, unknown>).jsonValue as Record<string, unknown> | undefined;
const node = jv ?? (v as Record<string, unknown>);
const fields = node.fields as Record<string, unknown> | undefined;
const val = fields?.Value as Record<string, unknown> | undefined;
return str(val?.value) ?? str(node.displayName) ?? str(node.name);
}
/** Build the Discover content_grid POST body enumerating community records. */
function enumBody(offset: number): unknown {
return {
context: { page: { uri: BOUND_URI } },
widget: {
items: [
{
rfk_id: RFK_ID,
search: {
content: {},
offset,
limit: ENUM_PAGE_SIZE,
filter: { type: "and", filters: [{ type: "anyOf", name: "type", values: ["community"] }] },
},
entity: "content",
},
],
},
};
}
interface DiscoverContentRow {
url?: string;
type?: string;
name?: string;
}
/** Pull the content_grid widget rows out of a Discover response body. */
function enumWidget(body: string): { total: number; content: DiscoverContentRow[] } | null {
let parsed: unknown;
try {
parsed = JSON.parse(body);
} catch {
return null;
}
const widgets = (parsed as { widgets?: unknown[] })?.widgets;
if (!Array.isArray(widgets)) return null;
const w = widgets.find(
(x) => (x as { type?: string })?.type === "content_grid",
) as { total_item?: number; content?: DiscoverContentRow[] } | undefined;
if (!w) return null;
return {
total: Number(w.total_item ?? 0),
content: Array.isArray(w.content) ? w.content : [],
};
}
/** Extract the __NEXT_DATA__ JSON object from a Next.js SSR HTML page. */
function nextData(html: string): unknown | null {
const m = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
if (!m) return null;
try {
return JSON.parse(m[1]!);
} catch {
return null;
}
}
/** Depth-first search for the first object satisfying `pred`. */
function deepFind(root: unknown, pred: (o: Record<string, unknown>) => boolean): Record<string, unknown> | null {
const stack: unknown[] = [root];
const seen = new Set<unknown>();
while (stack.length) {
const cur = stack.pop();
if (!cur || typeof cur !== "object" || seen.has(cur)) continue;
seen.add(cur);
const o = cur as Record<string, unknown>;
if (pred(o)) return o;
for (const k of Object.keys(o)) {
const v = o[k];
if (v && typeof v === "object") stack.push(v);
}
}
return null;
}
/** Collect every distinct floorplan-fact object embedded on a community page. */
function collectFloorplans(root: unknown): Record<string, unknown>[] {
const stack: unknown[] = [root];
const seen = new Set<unknown>();
const byUrl = new Map<string, Record<string, unknown>>();
const out: Record<string, unknown>[] = [];
while (stack.length) {
const cur = stack.pop();
if (!cur || typeof cur !== "object" || seen.has(cur)) continue;
seen.add(cur);
const o = cur as Record<string, unknown>;
// A floorplan-fact object: has a plan name AND structured beds + sqft.
if (o.floorPlanName !== undefined && o.sizeSqft !== undefined && o.numberofBedrooms !== undefined) {
const path = str(fieldVal(o.url));
const key = path ?? `${str(fieldVal(o.floorPlanID)) ?? ""}:${str(fieldVal(o.floorPlanName)) ?? ""}`;
if (key && !byUrl.has(key)) {
byUrl.set(key, o);
out.push(o);
}
}
for (const k of Object.keys(o)) {
const v = o[k];
if (v && typeof v === "object") stack.push(v);
}
}
return out;
}
/** LGI status → our construction-status enum.
*
* LGI's community `saleStatus` vocabulary is a MARKETING/sales phase, not a
* construction stage — observed values are "active", "reduced pricing", "new
* floor plans available", "last chance", "coming soon", "sold out". Only a few
* of these carry an unambiguous build/move-in signal; the rest (notably
* "active" / "reduced pricing", the bulk of the feed) say nothing about
* whether a given home is finished. Facts-only: we map ONLY the unambiguous
* ends and return null otherwise — never guessing a stage the feed doesn't
* state. The raw label is still carried as evidence on the field. */
function constructionStatus(status: unknown): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = (str(status) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("coming soon") || s.includes("pre-sale") || s.includes("presale") || s.includes("pre sale")) {
return "UNDER_CONSTRUCTION";
}
if (s.includes("move-in ready") || s.includes("move in ready") || s.includes("quick move") || s.includes("last chance") || s.includes("sold out")) {
return "MOVE_IN_READY";
}
// "active", "reduced pricing", "new floor plans available", etc. → no
// construction-stage signal; leave null rather than fabricate one.
return null;
}
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
const finiteOrNull = (v: unknown): number | null => {
const n = typeof v === "number" ? v : Number(v);
return Number.isFinite(n) && n !== 0 ? n : null;
};
export const lgiHomesAdapter: SourceAdapter = {
key: "lgi-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// 1) Enumerate community URLs via Discover (paginated). The Discover pages
// themselves are hollow (no facts), so they are NOT yielded for
// extraction — only mined for the community page URLs to crawl.
const communityUrls: string[] = [];
let offset = 0;
for (;;) {
let page: RawPage;
try {
page = await fetcher.postJson(`${DISCOVER_URL}#enum&offset=${offset}`, enumBody(offset));
} catch (error) {
console.warn(` lgi enum offset=${offset}: ${error instanceof Error ? error.message : String(error)}`);
break; // a block/403 stops enumeration (source marked degraded upstream)
}
const grid = enumWidget(page.body.toString("utf8"));
const rows = grid?.content ?? [];
for (const r of rows) {
const u = str(r.url);
if (u && (r.type === "community" || !r.type)) communityUrls.push(u);
}
offset += ENUM_PAGE_SIZE;
if (!grid || rows.length < ENUM_PAGE_SIZE || (grid.total > 0 && offset >= grid.total)) break;
}
// 2) Fetch each community page (bounded by LGI_PAGE_LIMIT). Each is a
// self-contained SSR page carrying the community + all its floorplans.
const limit = Math.max(1, PAGE_LIMIT);
for (const url of communityUrls.slice(0, limit)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` lgi community ${url}: ${error instanceof Error ? error.message : String(error)}`);
// one bad page doesn't stop the crawl; a 403 will throw BlockedError which
// the runner treats as degraded — but keep pulling the rest here.
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const nd = nextData(html);
if (!nd) {
return { records: [], errors: [{ url: page.url, reason: "no __NEXT_DATA__ on LGI community page" }] };
}
// The community location object: has a name, coordinates, and address parts.
const comm = deepFind(
nd,
(o) =>
o.communityName !== undefined &&
o.latitude !== undefined &&
o.city !== undefined &&
o.zipCode !== undefined,
);
if (!comm) {
return { records: [], errors: [{ url: page.url, reason: "no community location object in __NEXT_DATA__" }] };
}
const community = str(fieldVal(comm.communityName));
if (!community) {
return { records: [], errors: [{ url: page.url, reason: "community object missing communityName" }] };
}
const commStreet = str(fieldVal(comm.address1));
const city = str(fieldVal(comm.city));
const state = normalizeStateCode(str(fieldVal(comm.state)));
const zip = zip5(fieldVal(comm.zipCode));
const lat = finiteOrNull(fieldVal(comm.latitude));
const lon = finiteOrNull(fieldVal(comm.longitude)); // US lon negative — sign preserved
const commSaleStatus = kvpLabel(comm.saleStatus); // marketing phase (evidence only)
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
// Community FIRST — publish creates the FK target the home records need.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, page.url, "LGI communityName"),
street: fv(commStreet, commStreet, page.url, commStreet === null ? null : "LGI community address1"),
city: fv(city, city, page.url),
state: fv(state, str(fieldVal(comm.state)), page.url),
zip: fv(zip, str(fieldVal(comm.zipCode)), page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(lat, null, page.url, lat === null ? null : "LGI community latitude"),
lon: fv(lon, null, page.url, lon === null ? null : "LGI community longitude"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
// One inventory_home per floorplan offered at this community.
const floorplans = collectFloorplans(nd);
for (const f of floorplans) {
const plan = str(fieldVal(f.floorPlanName));
if (!plan) continue; // no plan name → cannot key a home; skip silently
const planId = str(fieldVal(f.floorPlanID));
const planPath = str(fieldVal(f.url));
const homeUrl = planPath ? (planPath.startsWith("http") ? planPath : `${ORIGIN}${planPath}`) : page.url;
const price = posNum(fieldVal(f.listPrice)); // real sale price (not monthlyPayment)
const beds = posNum(fieldVal(f.numberofBedrooms));
const bathsTotal = nonNegNum(fieldVal(f.numberofBathrooms));
const sqft = posNum(fieldVal(f.sizeSqft));
const stories = posNum(fieldVal(f.numberofStories));
const garages = nonNegNum(fieldVal(f.numberofCarGarage));
const adStatusLabel = kvpLabel(f.adStatus) ?? commSaleStatus;
const cStatus = constructionStatus(adStatusLabel);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
// No per-home street address exists (floorplans are model designs at
// a community); key the home by its plan path / community+plan.
address: homeUrl,
builderInventoryId: planId ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan,
},
fields: {
// The community's sales-center address is the closest real street.
street: fv(commStreet, commStreet, homeUrl, commStreet === null ? null : "LGI community address1"),
city: fv(city, city, homeUrl),
state: fv(state, str(fieldVal(comm.state)), homeUrl),
zip: fv(zip, str(fieldVal(comm.zipCode)), homeUrl),
price: fv(price, price === null ? null : String(fieldVal(f.listPrice)), homeUrl, price === null ? null : `LGI listPrice ${fieldVal(f.listPrice)}`),
beds: fv(beds, beds === null ? null : String(fieldVal(f.numberofBedrooms)), homeUrl),
bathsTotal: fv(bathsTotal, bathsTotal === null ? null : String(fieldVal(f.numberofBathrooms)), homeUrl),
sqft: fv(sqft, sqft === null ? null : String(fieldVal(f.sizeSqft)), homeUrl),
stories: fv(stories, stories === null ? null : String(fieldVal(f.numberofStories)), homeUrl),
garageSpaces: fv(garages, garages === null ? null : String(fieldVal(f.numberofCarGarage)), homeUrl),
homeType: fv("SINGLE_FAMILY" as const, null, homeUrl, "LGI single-family floorplan"),
constructionStatus: fv(cStatus, adStatusLabel, homeUrl, cStatus === null ? null : `status: ${adStatusLabel}`),
estCompletionDate: fv<string>(null, null, homeUrl),
lotNumber: fv<string>(null, null, homeUrl),
builderInventoryId: fv(planId, planId, homeUrl),
lat: fv(lat, null, homeUrl, lat === null ? null : "LGI community latitude"),
lon: fv(lon, null, homeUrl, lon === null ? null : "LGI community longitude"),
planName: fv(plan, plan, homeUrl),
// facts-only: image_url exists in the feed but is intentionally dropped.
images: fv<string[]>([], null, homeUrl),
},
});
}
if (floorplans.length === 0) {
errors.push({ url: page.url, reason: `community ${community} had no floorplan-fact objects` });
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};