← back to Homesonspec
collectors/kb-home/src/index.ts
178 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* KB Home adapter — SINGLE_FEED source (recon 2026-07-23).
* AEM/Handlebars, server-rendered. The whole national move-in-ready inventory
* ships in ONE page (/move-in-ready) as `var allMIRs = JSON.parse("...")` — a
* double-encoded JSON string. One fetch yields every home. DEFAULT = national
* (all states, each home tagged by its own r.State); set KBHOME_STATE=XX to filter
* to one state. Run this ONCE per sweep — per-state iteration is a no-op after the
* first state (page-dedup on the identical feed). Plain HTTP, no anti-bot.
*/
const FEED_URL = "https://www.kbhome.com/move-in-ready";
const ORIGIN = "https://www.kbhome.com";
const BUILDER_SLUG = "kb-home";
// KB is a SINGLE national feed (one fetch = every home). Per-state iteration is dead-on-arrival:
// page-level contentHash dedup skips the identical 5.8MB feed for every state after the first, so
// only one state ever gets extracted per feed-change. Default = national extraction (all states);
// an explicit KBHOME_STATE still filters (back-compat). Each home is tagged by its own r.State.
const STATE_FILTER = process.env.KBHOME_STATE?.toUpperCase();
const ALL_STATES = !STATE_FILTER || STATE_FILTER === "ALL";
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
/** Grab the double-encoded JSON string argument of `var allMIRs = JSON.parse("…")`. */
function extractQuotedArg(html: string, marker: string): string | null {
const at = html.indexOf(marker);
if (at < 0) return null;
const start = html.indexOf('"', at);
if (start < 0) return null;
let esc = false;
for (let i = start + 1; i < html.length; i++) {
const c = html[i]!;
if (esc) esc = false;
else if (c === "\\") esc = true;
else if (c === '"') return html.slice(start, i + 1);
}
return null;
}
function statusOf(moveInCopy: string | null, moveInDate: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
const s = (moveInCopy ?? "").toLowerCase();
if (/available now|move.?in ready|ready now|complete/.test(s)) return "MOVE_IN_READY";
if (/coming soon|planned/.test(s)) return "PLANNED";
if (moveInDate && new Date(moveInDate) <= new Date()) return "MOVE_IN_READY";
return "UNDER_CONSTRUCTION";
}
export const kbHomeAdapter: SourceAdapter = {
key: "kb-home-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
yield await fetcher.fetch(FEED_URL); // one 5.8MB page carries all inventory
},
extract(page: RawPage): ExtractionOutput {
const errors: { url: string; reason: string }[] = [];
try {
const html = page.body.toString("utf8");
const quoted = extractQuotedArg(html, "var allMIRs = JSON.parse(");
let all: Record<string, unknown>[] = [];
if (quoted) {
try { all = JSON.parse(JSON.parse(quoted)); } catch { /* fall through */ }
}
if (!Array.isArray(all) || all.length === 0) {
return { records: [], errors: [{ url: page.url, reason: "allMIRs not found/empty" }] };
}
const rows = ALL_STATES ? all : all.filter((r) => str(r.State)?.toUpperCase() === STATE_FILTER);
const records: ExtractedRecord[] = [];
// Emit each unique community first (publish needs the FK target before homes).
const seen = new Set<string>();
for (const r of rows) {
const communityName = str(r.CommunityName);
// key on name+state: in national mode, same-named communities in different states
// must NOT collide into one (Cody-gated) — else the 2nd state's community is dropped.
const ckey = `${communityName}|${str(r.State)?.toUpperCase() ?? ""}`;
if (!communityName || seen.has(ckey)) continue;
seen.add(ckey);
const phone = str(r.CommunityOfficePhone);
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
fields: {
name: fv(communityName, communityName, page.url),
street: fv<string>(null, null, page.url),
city: fv(str(r.City), str(r.City), page.url),
state: fv(str(r.State)?.toUpperCase() ?? null, str(r.State), page.url),
zip: fv(str(r.ZipCode), str(r.ZipCode), page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(num(r.Latitude), null, page.url),
lon: fv(num(r.Longitude), null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(phone ? phone.replace(/[^0-9+]/g, "") : null, phone, page.url, "community sales phone"),
},
});
}
for (const r of rows) {
const address = str(r.Address);
const communityName = str(r.CommunityName);
if (!address || !communityName) continue;
// KB's ThumbnailImage is an OBJECT {Image,Caption,…} and GalleryPhotos an ARRAY
// of such objects — the URL lives under `.Image` (a site-relative /globalassets
// path), NOT the object itself. The old str()/split() read both as null, so KB
// had 0 photos. Build thumb-first + gallery, absolutize, drop "coming soon"
// placeholders (worse than the clean gradient fallback). [recon 2026-07-28]
const imgUrl = (o: unknown): string | null => str((o as Record<string, unknown> | undefined)?.Image);
const gallery = Array.isArray(r.GalleryPhotos) ? (r.GalleryPhotos as unknown[]) : [];
const rawImgs = [imgUrl(r.ThumbnailImage), ...gallery.map(imgUrl)].filter((u): u is string => !!u);
const images = [...new Set(rawImgs)]
.filter((u) => !/photo-coming-soon|\/images\/default\//i.test(u))
.map((u) => (u.startsWith("http") ? u : ORIGIN + u))
.slice(0, 8);
const lot = str(r.Homesite) ?? str(r.MLSNumber);
const homeType = /town|condo|attached/i.test(str(r.HomeType) ?? "") ? "TOWNHOME" : "SINGLE_FAMILY";
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName,
address,
builderInventoryId: lot,
planName: str(r.FloorPlanName),
},
fields: {
street: fv(address, address, page.url),
city: fv(str(r.City), str(r.City), page.url),
state: fv(str(r.State)?.toUpperCase() ?? null, str(r.State), page.url),
zip: fv(str(r.ZipCode), str(r.ZipCode), page.url),
price: fv(num(r.Price), str(r.Price), page.url, r.Price ? `price ${String(r.Price)}` : null),
beds: fv(num(r.Bedrooms), str(r.Bedrooms), page.url),
bathsTotal: fv(num(r.Bathrooms), str(r.Bathrooms), page.url),
sqft: fv(num(r.Size), str(r.Size), page.url),
stories: fv(num(r.Stories), null, page.url),
garageSpaces: fv(num(r.Garages), null, page.url),
homeType: fv(homeType as never, str(r.HomeType), page.url, "KB Home inventory home"),
constructionStatus: fv(statusOf(str(r.MoveInDateCopy), str(r.MoveInDate)), str(r.MoveInDateCopy), page.url),
estCompletionDate: fv(str(r.MoveInDate), str(r.MoveInDate), page.url),
lotNumber: fv(str(r.Homesite), str(r.Homesite), page.url),
builderInventoryId: fv(lot, lot, page.url),
planName: fv(str(r.FloorPlanName), str(r.FloorPlanName), page.url),
availabilityStatus: fv(str(r.MoveInDateCopy), str(r.MoveInDateCopy), page.url),
images: fv<string[]>(images, null, page.url, images.length ? "builder listing photo" : null),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};