← back to Homesonspec
collectors/khovnanian/src/index.ts
223 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* K. Hovnanian (khov.com) adapter — Next.js App-Router RSC source (recon 2026-07-29).
*
* khov.com serves per-community pages at /new-construction-homes/{state}/{city}/{community}/…
* that server-render their quick-move-in inventory into the streamed RSC (`self.__next_f`)
* as escaped JSON. We unescape, string-aware brace-match every object carrying a real
* `homeSite` + street `address`, and read price + a `specifications` array
* (["2,856 Sq ft","3 Stories","4 Beds","3.5 Baths","2 Cars"]) + absolute CloudFront image.
* Plain HTTP, no anti-bot, robots-permitted. State-scoped via KHOVNANIAN_STATE (full slug,
* e.g. "delaware", "california"). Facts-only; images enabled per Steve's hotlink approval.
*/
const SITEMAP = "https://www.khov.com/sitemap.xml";
const BUILDER_SLUG = "khovnanian";
const STATE_SEG = (process.env.KHOVNANIAN_STATE ?? "").toLowerCase().trim(); // URL slug, e.g. "delaware"
const PAGE_LIMIT = Number(process.env.KHOVNANIAN_PAGE_LIMIT ?? 200);
// Full state slug → 2-letter code (URL carries the slug; homes carry a 2-letter `state`,
// but this is the fallback and lets a home inherit the sweep's state).
const STATE_CODE: Record<string, string> = {
alabama: "AL", arizona: "AZ", california: "CA", colorado: "CO", delaware: "DE",
florida: "FL", georgia: "GA", illinois: "IL", maryland: "MD", "new-jersey": "NJ",
"north-carolina": "NC", ohio: "OH", pennsylvania: "PA", "south-carolina": "SC",
texas: "TX", virginia: "VA", "west-virginia": "WV", minnesota: "MN",
};
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
const titleCase = (slug: string): string =>
slug.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()).trim();
/** khov's totalPrice is {label,value:"$$704,990",lastValue:"$$724,040"} (the $$ is an RSC
* artifact); also accepts a bare number/string. Returns the current total price. */
function priceOf(v: unknown): number | null {
if (typeof v === "number" || typeof v === "string") return num(v);
if (v && typeof v === "object") { const o = v as Record<string, unknown>; return num(o.value) ?? num(o.textContent); }
return null;
}
/** Pull beds/baths/sqft/stories/garages out of the specifications string array. */
function specs(arr: unknown): { beds: number | null; baths: number | null; sqft: number | null; stories: number | null; garages: number | null } {
const out = { beds: null as number | null, baths: null as number | null, sqft: null as number | null, stories: null as number | null, garages: null as number | null };
if (!Array.isArray(arr)) return out;
for (const raw of arr) {
const s = str(raw);
if (!s) continue;
if (/sq\s*\.?\s*ft/i.test(s)) out.sqft = num(s);
else if (/stor/i.test(s)) out.stories = num(s);
else if (/bed/i.test(s)) out.beds = num(s);
else if (/bath/i.test(s)) out.baths = num(s);
else if (/car|garage/i.test(s)) out.garages = num(s);
}
return out;
}
const MONTHS: Record<string, string> = {
january: "01", february: "02", march: "03", april: "04", may: "05", june: "06",
july: "07", august: "08", september: "09", october: "10", november: "11", december: "12",
};
/** tags like "Available September 2026" or "Available Now" / "Move-in Ready". */
function statusOf(tags: unknown): { status: "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED"; estDate: string | null } {
const t = (Array.isArray(tags) ? tags.map((x) => str(x) ?? "").join(" ") : "").toLowerCase();
if (/available now|move.?in|ready to move|quick move/.test(t)) return { status: "MOVE_IN_READY", estDate: null };
const m = t.match(/available\s+([a-z]+)\s+(\d{4})/);
if (m && MONTHS[m[1]!]) return { status: "UNDER_CONSTRUCTION", estDate: `${m[2]}-${MONTHS[m[1]!]}-01` };
if (/coming soon|planned|future/.test(t)) return { status: "PLANNED", estDate: null };
return { status: "UNDER_CONSTRUCTION", estDate: null };
}
function homeTypeOf(name: string | null): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" {
const s = (name ?? "").toLowerCase();
if (/condo/.test(s)) return "CONDO";
if (/town/.test(s)) return "TOWNHOME";
return "SINGLE_FAMILY";
}
/** String-aware brace match: the enclosing {...} object around index i. */
function enclosingObject(s: string, i: number): Record<string, unknown> | null {
let start = -1, depth = 0;
for (let j = i; j >= 0; j--) {
const c = s[j]!;
if (c === "}") depth++;
else if (c === "{") { if (depth === 0) { start = j; break; } depth--; }
}
if (start < 0) return null;
depth = 0; let inStr = false, esc = false;
for (let j = start; j < s.length; j++) {
const c = s[j]!;
if (inStr) { if (esc) esc = false; else if (c === "\\") esc = true; else if (c === '"') inStr = false; }
else if (c === '"') inStr = true;
else if (c === "{") depth++;
else if (c === "}") { depth--; if (depth === 0) { try { return JSON.parse(s.slice(start, j + 1)); } catch { return null; } } }
}
return null;
}
export const khovnanianAdapter: SourceAdapter = {
key: "khovnanian-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") { yield* fetchFixtures(ctx); return; }
const fetcher = new LiveFetcher(ctx.registry);
const sm = await fetcher.fetch(SITEMAP);
// community pages: /new-construction-homes/{state}/{city}/{community}/… ; drop blog + non-state.
const urls = [...sm.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.filter((u) => /\/new-construction-homes\//i.test(u) && !/\/blog\//i.test(u))
.filter((u) => !STATE_SEG || u.toLowerCase().includes(`/new-construction-homes/${STATE_SEG}/`));
urls.sort();
for (const url of urls.slice(0, PAGE_LIMIT)) {
try { yield await fetcher.fetch(url); }
catch (error) { console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`); }
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const u = html.replace(/\\"/g, '"').replace(/\\\\/g, "\\");
// community slug from the page URL: /new-construction-homes/{state}/{city}/{community}/…
const segs = page.url.split("?")[0]!.split("/").filter(Boolean);
const nci = segs.indexOf("new-construction-homes");
const communityName = nci >= 0 && segs[nci + 3] ? titleCase(segs[nci + 3]!) : null;
// QMI homes carry a real `homeSite`; brace-match each enclosing object, dedup by address.
const seen = new Set<string>();
const homes: Record<string, unknown>[] = [];
const re = /"homeSite"\s*:\s*"[^"]+"/g;
let m: RegExpExecArray | null;
while ((m = re.exec(u))) {
const obj = enclosingObject(u, m.index);
const address = obj ? str(obj.address) : null;
// a real QMI home has a street address starting with a number
if (obj && address && /^\d/.test(address) && !seen.has(address)) { seen.add(address); homes.push(obj); }
}
if (homes.length === 0) return { records: [], errors: [] };
const first = homes[0]!;
const cname = communityName ?? str(first.communityName) ?? "Unknown";
const records: ExtractedRecord[] = [];
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(str(first.city), str(first.city), page.url),
state: fv(str(first.state)?.toUpperCase() ?? STATE_CODE[STATE_SEG] ?? null, str(first.state), page.url),
zip: fv(str(first.zipCode), str(first.zipCode), page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv<string>(null, null, page.url),
},
});
for (const h of homes) {
const address = str(h.address);
if (!address) continue;
const sp = specs(h.specifications);
const st = statusOf(h.tags);
const img = str((h.image as Record<string, unknown> | undefined)?.src);
const images = img && img.startsWith("http") ? [img] : [];
const invId = str(h.id) ?? str(h.homeSite);
const planName = str((h.name as Record<string, unknown> | undefined)?.textContent);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
address,
builderInventoryId: invId,
planName,
},
fields: {
street: fv(address, address, page.url),
city: fv(str(h.city), str(h.city), page.url),
state: fv(str(h.state)?.toUpperCase() ?? STATE_CODE[STATE_SEG] ?? null, str(h.state), page.url),
zip: fv(str(h.zipCode), str(h.zipCode), page.url),
price: fv(priceOf(h.totalPrice), null, page.url, h.totalPrice ? `totalPrice ${JSON.stringify(h.totalPrice).slice(0, 30)}` : null),
beds: fv(sp.beds, null, page.url),
bathsTotal: fv(sp.baths, null, page.url),
sqft: fv(sp.sqft, null, page.url),
stories: fv(sp.stories, null, page.url),
garageSpaces: fv(sp.garages, null, page.url),
homeType: fv(homeTypeOf(planName) as never, planName, page.url, "K. Hovnanian inventory home"),
constructionStatus: fv(st.status, str((h.tags as unknown[])?.[0]) ?? null, page.url),
estCompletionDate: fv(st.estDate, null, page.url),
lotNumber: fv(str(h.homeSite), str(h.homeSite), page.url),
builderInventoryId: fv(invId, invId, page.url),
planName: fv(planName, planName, page.url),
availabilityStatus: fv(str((h.tags as unknown[])?.[0]) ?? null, null, page.url),
images: fv<string[]>(images, null, page.url, images.length ? "builder listing photo" : null),
},
});
}
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};