← back to Homesonspec
collectors/taylor-morrison/src/index.ts
229 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Taylor Morrison adapter — Sitecore "fedmodel" source (recon-verified 2026-07-23).
* Two-level sitemap: sitemap-index → per-state sitemap → 4-segment community URLs
* (/ca/{metro}/{city}/{community}). Per community we fetch `{url}/available-homes`;
* the page embeds 13 `data-fed-ref="fedmodel"` blocks — the FIRST is an outdated-
* browser decoy, so we scan for the block carrying `availableHomesList` and walk
* `availableHomesList.sections[*].homes[]`. Per-community phone via JSON-LD
* telephone. No per-home geo. Plain HTTP, no anti-bot.
*/
const SITEMAP_INDEX = "https://www.taylormorrison.com/sitemap-index.xml";
const BUILDER_SLUG = "taylor-morrison";
const STATE_SEG = (process.env.TAYLORMORRISON_STATE ?? "ca").toLowerCase(); // URL + sitemap segment
const PAGE_LIMIT = Number(process.env.TAYLORMORRISON_PAGE_LIMIT ?? 300);
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
/** "California" → "CA"; already-2-letter passes through. */
const US_STATE: Record<string, string> = {
california: "CA", texas: "TX", arizona: "AZ", colorado: "CO", florida: "FL",
nevada: "NV", oregon: "OR", washington: "WA", georgia: "GA", "north carolina": "NC",
"south carolina": "SC",
};
function stateCode(v: unknown): string | null {
const s = str(v);
if (!s) return null;
if (/^[A-Za-z]{2}$/.test(s)) return s.toUpperCase();
return US_STATE[s.toLowerCase()] ?? null;
}
/** "9/30/2026" → "2026-09-30" (ISO date). Returns null if unparseable. */
function isoDate(v: unknown): string | null {
const s = str(v);
if (!s) return null;
const m = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
if (!m) return null;
const [, mm, dd, yyyy] = m;
return `${yyyy}-${mm!.padStart(2, "0")}-${dd!.padStart(2, "0")}`;
}
/**
* Pull the fedmodel JSON object that contains `availableHomesList`. There are
* ~13 fedmodel <script> blocks per page; only one carries the inventory.
*/
function extractHomesModel(html: string): Record<string, unknown> | null {
const re = /data-fed-ref="fedmodel"[^>]*>/g;
let m: RegExpExecArray | null;
while ((m = re.exec(html))) {
const start = m.index + m[0].length;
const end = html.indexOf("</script>", start);
if (end < 0) continue;
const blob = html.slice(start, end);
if (!blob.includes("availableHomesList")) continue;
try {
return JSON.parse(blob) as Record<string, unknown>;
} catch {
/* keep scanning — a different block may parse */
}
}
return null;
}
/** Depth-first search for the first value under `key` anywhere in the object tree. */
function deepFind(o: unknown, key: string, depth = 0): unknown {
if (depth > 10 || o == null) return null;
if (Array.isArray(o)) {
for (const v of o) {
const r = deepFind(v, key, depth + 1);
if (r != null) return r;
}
} else if (typeof o === "object") {
const rec = o as Record<string, unknown>;
if (key in rec && rec[key] != null) return rec[key];
for (const v of Object.values(rec)) {
const r = deepFind(v, key, depth + 1);
if (r != null) return r;
}
}
return null;
}
function homeTypeOf(h: Record<string, unknown>, community: string | null): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" {
const s = `${str(h.floorPlan) ?? ""} ${str(h.community_Name) ?? community ?? ""}`.toLowerCase();
if (/condo/.test(s)) return "CONDO";
if (/town(home|house)?/.test(s)) return "TOWNHOME";
return "SINGLE_FAMILY";
}
function statusOf(h: Record<string, unknown>): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
if (h.isComingSoon === true) return "PLANNED";
const ready = isoDate(h.readyDate);
if (ready && new Date(ready) <= new Date()) return "MOVE_IN_READY";
return "UNDER_CONSTRUCTION";
}
export const taylorMorrisonAdapter: SourceAdapter = {
key: "taylor-morrison-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") { yield* fetchFixtures(ctx); return; }
const fetcher = new LiveFetcher(ctx.registry);
// Level 1: sitemap-index → the target state's sitemap.
const idx = await fetcher.fetch(SITEMAP_INDEX);
const stateSitemap = [...idx.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.find((u) => new RegExp(`/${STATE_SEG}/sitemap\\.xml$`, "i").test(u));
if (!stateSitemap) return;
// Level 2: state sitemap → 4-segment community URLs (drop group-page rollups).
const sm = await fetcher.fetch(stateSitemap);
const communityRe = new RegExp(`taylormorrison\\.com/${STATE_SEG}/[^/]+/[^/]+/[^/]+/?$`, "i");
const urls = [...sm.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.filter((u) => communityRe.test(u) && !/community-group-page/i.test(u));
urls.sort();
// Level 3: per-community available-homes page.
for (const url of urls.slice(0, PAGE_LIMIT)) {
const homesUrl = url.replace(/\/$/, "") + "/available-homes";
try { yield await fetcher.fetch(homesUrl); }
catch (error) { console.warn(` skip ${homesUrl}: ${error instanceof Error ? error.message : String(error)}`); }
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const model = extractHomesModel(html);
if (!model) return { records: [], errors: [] }; // community with no available-homes model
const ahl = deepFind(model, "availableHomesList") as Record<string, unknown> | null;
const sections = (ahl?.sections as Record<string, unknown>[] | undefined) ?? [];
const homes: Record<string, unknown>[] = [];
for (const s of sections) {
const hs = (s.homes as Record<string, unknown>[] | undefined) ?? [];
for (const h of hs) if (str(h.address)) homes.push(h);
}
if (homes.length === 0) return { records: [], errors: [] };
const first = homes[0]!;
const communityName = str(first.community_Name);
if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community name" }] };
const phoneRaw = html.match(/"telephone":\s*"([^"]+)"/)?.[1] ?? null;
const phone = phoneRaw ? phoneRaw.replace(/[^0-9+]/g, "") : null;
const records: ExtractedRecord[] = [];
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
fields: {
name: fv(communityName, communityName, page.url),
street: fv<string>(null, null, page.url),
city: fv(str(first.city), str(first.city), page.url),
state: fv(stateCode(first.state), str(first.state), page.url),
zip: fv(str(first.zip), str(first.zip), page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv<number>(null, null, page.url), // TM feed carries no community geo
lon: fv<number>(null, null, page.url),
hoaFeeMonthly: fv(num(first.hoaDues), null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv(first.is_FiftyFivePlus === true ? true : null, null, page.url),
salesPhone: fv(phone && phone.length >= 10 ? phone : null, phoneRaw, page.url, "community sales phone"),
},
});
for (const h of homes) {
const address = str(h.address);
if (!address) continue;
// TM's feed returns site-relative image paths (e.g. "/-/media/…"); absolutize
// them against the TM origin so downstream <img> tags don't 404 when rendered
// on homesonspec.com. Already-absolute (http…) srcs pass through unchanged.
const photoRaw = str((h.photo as Record<string, unknown> | undefined)?.Src);
const photo = photoRaw && photoRaw.startsWith("/") ? `https://www.taylormorrison.com${photoRaw}` : photoRaw;
const lot = str(h.homeSite);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG, communityName, address,
lotNumber: lot,
builderInventoryId: str(h.ItemId),
planName: str(h.floorPlan),
},
fields: {
street: fv(address, address, page.url),
city: fv(str(h.city), str(h.city), page.url),
state: fv(stateCode(h.state), str(h.state), page.url),
zip: fv(str(h.zip), str(h.zip), page.url),
price: fv(num(h.price), h.price != null ? String(h.price) : null, page.url, h.price != null ? `price ${String(h.price)}` : null),
beds: fv(num(h.bed), null, page.url),
bathsTotal: fv(num(h.totalBath), null, page.url),
sqft: fv(num(h.sqft), null, page.url),
stories: fv<number>(null, null, page.url),
garageSpaces: fv(num(h.garages), null, page.url),
homeType: fv(homeTypeOf(h, communityName) as never, str(h.floorPlan), page.url, "Taylor Morrison inventory home"),
constructionStatus: fv(statusOf(h), h.isComingSoon === true ? "coming soon" : str(h.readyDate), page.url),
estCompletionDate: fv(isoDate(h.readyDate), str(h.readyDate), page.url),
lotNumber: fv(lot, lot, page.url),
builderInventoryId: fv(str(h.ItemId), str(h.ItemId), page.url),
planName: fv(str(h.floorPlan), str(h.floorPlan), page.url),
availabilityStatus: fv(h.availabilityStatus === "1" ? "available" : str(h.availabilityStatus), str(h.availabilityStatus), page.url),
images: fv<string[]>(photo ? [photo] : [], null, page.url, photo ? "builder listing photo" : null),
},
});
}
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};