← back to Homesonspec
feat(collectors): add Taylor Morrison adapter (Sitecore fedmodel, CA)
86433ff451a95258cdf6a69fa7683a313732e642 · 2026-07-23 01:13:04 -0700 · Steve Abrams
Two-level sitemap → per-community /available-homes. Scans all 13 fedmodel
blocks for the one carrying availableHomesList (first is a browser-warning
decoy), walks sections[].homes[]. Normalizes California→CA, M/D/YYYY→ISO,
JSON-LD telephone, photo.Src → images. Verified against live fixture:
3 records, 0 errors, all fields correct. Builder+source rows already seeded.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M apps/workers/src/cli.tsA collectors/taylor-morrison/package.jsonA collectors/taylor-morrison/src/index.tsM pnpm-lock.yaml
Diff
commit 86433ff451a95258cdf6a69fa7683a313732e642
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jul 23 01:13:04 2026 -0700
feat(collectors): add Taylor Morrison adapter (Sitecore fedmodel, CA)
Two-level sitemap → per-community /available-homes. Scans all 13 fedmodel
blocks for the one carrying availableHomesList (first is a browser-warning
decoy), walks sections[].homes[]. Normalizes California→CA, M/D/YYYY→ISO,
JSON-LD telephone, photo.Src → images. Verified against live fixture:
3 records, 0 errors, all fields correct. Builder+source rows already seeded.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
apps/workers/src/cli.ts | 2 +
collectors/taylor-morrison/package.json | 15 +++
collectors/taylor-morrison/src/index.ts | 224 ++++++++++++++++++++++++++++++++
pnpm-lock.yaml | 22 ++++
4 files changed, 263 insertions(+)
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index 9def79e..88a7963 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -6,6 +6,7 @@ import { drHortonAdapter } from "@spechomes/collector-dr-horton";
import { kbHomeAdapter } from "@spechomes/collector-kb-home";
import { pulteAdapter } from "@spechomes/collector-pulte";
import { triPointeAdapter } from "@spechomes/collector-tri-pointe";
+import { taylorMorrisonAdapter } from "@spechomes/collector-taylor-morrison";
import { runPipeline } from "./pipeline";
import { recordSourceRun } from "./verify";
@@ -21,6 +22,7 @@ const ADAPTERS = {
[kbHomeAdapter.key]: kbHomeAdapter,
[pulteAdapter.key]: pulteAdapter,
[triPointeAdapter.key]: triPointeAdapter,
+ [taylorMorrisonAdapter.key]: taylorMorrisonAdapter,
};
async function main() {
diff --git a/collectors/taylor-morrison/package.json b/collectors/taylor-morrison/package.json
new file mode 100644
index 0000000..6545497
--- /dev/null
+++ b/collectors/taylor-morrison/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@spechomes/collector-taylor-morrison",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@spechomes/collectors-common": "workspace:*",
+ "@spechomes/schemas": "workspace:*",
+ "@spechomes/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/taylor-morrison/src/index.ts b/collectors/taylor-morrison/src/index.ts
new file mode 100644
index 0000000..4624688
--- /dev/null
+++ b/collectors/taylor-morrison/src/index.ts
@@ -0,0 +1,224 @@
+import type { ExtractedRecord, FieldValue } from "@spechomes/schemas";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@spechomes/collectors-common";
+
+/**
+ * Taylor Morrison adapter — Sitecore "fedmodel" source (recon-verified 2026-07-23).
+ * Two-level sitemap: sitemap-index → per-state sitemap → 4-segment community URLs
+ * (/ca/{metro}/{city}/{community}). Per community we fetch `{url}/available-homes`;
+ * the page embeds 13 `data-fed-ref="fedmodel"` blocks — the FIRST is an outdated-
+ * browser decoy, so we scan for the block carrying `availableHomesList` and walk
+ * `availableHomesList.sections[*].homes[]`. Per-community phone via JSON-LD
+ * telephone. No per-home geo. Plain HTTP, no anti-bot.
+ */
+const SITEMAP_INDEX = "https://www.taylormorrison.com/sitemap-index.xml";
+const BUILDER_SLUG = "taylor-morrison";
+const STATE_SEG = (process.env.TAYLORMORRISON_STATE ?? "ca").toLowerCase(); // URL + sitemap segment
+const PAGE_LIMIT = Number(process.env.TAYLORMORRISON_PAGE_LIMIT ?? 300);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+const num = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
+
+/** "California" → "CA"; already-2-letter passes through. */
+const US_STATE: Record<string, string> = {
+ california: "CA", texas: "TX", arizona: "AZ", colorado: "CO", florida: "FL",
+ nevada: "NV", oregon: "OR", washington: "WA", georgia: "GA", "north carolina": "NC",
+ "south carolina": "SC",
+};
+function stateCode(v: unknown): string | null {
+ const s = str(v);
+ if (!s) return null;
+ if (/^[A-Za-z]{2}$/.test(s)) return s.toUpperCase();
+ return US_STATE[s.toLowerCase()] ?? null;
+}
+
+/** "9/30/2026" → "2026-09-30" (ISO date). Returns null if unparseable. */
+function isoDate(v: unknown): string | null {
+ const s = str(v);
+ if (!s) return null;
+ const m = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
+ if (!m) return null;
+ const [, mm, dd, yyyy] = m;
+ return `${yyyy}-${mm!.padStart(2, "0")}-${dd!.padStart(2, "0")}`;
+}
+
+/**
+ * Pull the fedmodel JSON object that contains `availableHomesList`. There are
+ * ~13 fedmodel <script> blocks per page; only one carries the inventory.
+ */
+function extractHomesModel(html: string): Record<string, unknown> | null {
+ const re = /data-fed-ref="fedmodel"[^>]*>/g;
+ let m: RegExpExecArray | null;
+ while ((m = re.exec(html))) {
+ const start = m.index + m[0].length;
+ const end = html.indexOf("</script>", start);
+ if (end < 0) continue;
+ const blob = html.slice(start, end);
+ if (!blob.includes("availableHomesList")) continue;
+ try {
+ return JSON.parse(blob) as Record<string, unknown>;
+ } catch {
+ /* keep scanning — a different block may parse */
+ }
+ }
+ return null;
+}
+
+/** Depth-first search for the first value under `key` anywhere in the object tree. */
+function deepFind(o: unknown, key: string, depth = 0): unknown {
+ if (depth > 10 || o == null) return null;
+ if (Array.isArray(o)) {
+ for (const v of o) {
+ const r = deepFind(v, key, depth + 1);
+ if (r != null) return r;
+ }
+ } else if (typeof o === "object") {
+ const rec = o as Record<string, unknown>;
+ if (key in rec && rec[key] != null) return rec[key];
+ for (const v of Object.values(rec)) {
+ const r = deepFind(v, key, depth + 1);
+ if (r != null) return r;
+ }
+ }
+ return null;
+}
+
+function homeTypeOf(h: Record<string, unknown>, community: string | null): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" {
+ const s = `${str(h.floorPlan) ?? ""} ${str(h.community_Name) ?? community ?? ""}`.toLowerCase();
+ if (/condo/.test(s)) return "CONDO";
+ if (/town(home|house)?/.test(s)) return "TOWNHOME";
+ return "SINGLE_FAMILY";
+}
+function statusOf(h: Record<string, unknown>): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
+ if (h.isComingSoon === true) return "PLANNED";
+ const ready = isoDate(h.readyDate);
+ if (ready && new Date(ready) <= new Date()) return "MOVE_IN_READY";
+ return "UNDER_CONSTRUCTION";
+}
+
+export const taylorMorrisonAdapter: SourceAdapter = {
+ key: "taylor-morrison-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") { yield* fetchFixtures(ctx); return; }
+ const fetcher = new LiveFetcher(ctx.registry);
+
+ // Level 1: sitemap-index → the target state's sitemap.
+ const idx = await fetcher.fetch(SITEMAP_INDEX);
+ const stateSitemap = [...idx.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+ .map((m) => m[1]!)
+ .find((u) => new RegExp(`/${STATE_SEG}/sitemap\\.xml$`, "i").test(u));
+ if (!stateSitemap) return;
+
+ // Level 2: state sitemap → 4-segment community URLs (drop group-page rollups).
+ const sm = await fetcher.fetch(stateSitemap);
+ const communityRe = new RegExp(`taylormorrison\\.com/${STATE_SEG}/[^/]+/[^/]+/[^/]+/?$`, "i");
+ const urls = [...sm.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+ .map((m) => m[1]!)
+ .filter((u) => communityRe.test(u) && !/community-group-page/i.test(u));
+ urls.sort();
+
+ // Level 3: per-community available-homes page.
+ for (const url of urls.slice(0, PAGE_LIMIT)) {
+ const homesUrl = url.replace(/\/$/, "") + "/available-homes";
+ try { yield await fetcher.fetch(homesUrl); }
+ catch (error) { console.warn(` skip ${homesUrl}: ${error instanceof Error ? error.message : String(error)}`); }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const html = page.body.toString("utf8");
+ const model = extractHomesModel(html);
+ if (!model) return { records: [], errors: [] }; // community with no available-homes model
+
+ const ahl = deepFind(model, "availableHomesList") as Record<string, unknown> | null;
+ const sections = (ahl?.sections as Record<string, unknown>[] | undefined) ?? [];
+ const homes: Record<string, unknown>[] = [];
+ for (const s of sections) {
+ const hs = (s.homes as Record<string, unknown>[] | undefined) ?? [];
+ for (const h of hs) if (str(h.address)) homes.push(h);
+ }
+ if (homes.length === 0) return { records: [], errors: [] };
+
+ const first = homes[0]!;
+ const communityName = str(first.community_Name);
+ if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community name" }] };
+ const phoneRaw = html.match(/"telephone":\s*"([^"]+)"/)?.[1] ?? null;
+ const phone = phoneRaw ? phoneRaw.replace(/[^0-9+]/g, "") : null;
+
+ const records: ExtractedRecord[] = [];
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
+ fields: {
+ name: fv(communityName, communityName, page.url),
+ street: fv<string>(null, null, page.url),
+ city: fv(str(first.city), str(first.city), page.url),
+ state: fv(stateCode(first.state), str(first.state), page.url),
+ zip: fv(str(first.zip), str(first.zip), page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv<string>(null, null, page.url),
+ lat: fv<number>(null, null, page.url), // TM feed carries no community geo
+ lon: fv<number>(null, null, page.url),
+ hoaFeeMonthly: fv(num(first.hoaDues), null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv(first.is_FiftyFivePlus === true ? true : null, null, page.url),
+ salesPhone: fv(phone && phone.length >= 10 ? phone : null, phoneRaw, page.url, "community sales phone"),
+ },
+ });
+
+ for (const h of homes) {
+ const address = str(h.address);
+ if (!address) continue;
+ const photo = str((h.photo as Record<string, unknown> | undefined)?.Src);
+ const lot = str(h.homeSite);
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG, communityName, address,
+ lotNumber: lot,
+ builderInventoryId: str(h.ItemId),
+ planName: str(h.floorPlan),
+ },
+ fields: {
+ street: fv(address, address, page.url),
+ city: fv(str(h.city), str(h.city), page.url),
+ state: fv(stateCode(h.state), str(h.state), page.url),
+ zip: fv(str(h.zip), str(h.zip), page.url),
+ price: fv(num(h.price), h.price != null ? String(h.price) : null, page.url, h.price != null ? `price ${String(h.price)}` : null),
+ beds: fv(num(h.bed), null, page.url),
+ bathsTotal: fv(num(h.totalBath), null, page.url),
+ sqft: fv(num(h.sqft), null, page.url),
+ stories: fv<number>(null, null, page.url),
+ garageSpaces: fv(num(h.garages), null, page.url),
+ homeType: fv(homeTypeOf(h, communityName) as never, str(h.floorPlan), page.url, "Taylor Morrison inventory home"),
+ constructionStatus: fv(statusOf(h), h.isComingSoon === true ? "coming soon" : str(h.readyDate), page.url),
+ estCompletionDate: fv(isoDate(h.readyDate), str(h.readyDate), page.url),
+ lotNumber: fv(lot, lot, page.url),
+ builderInventoryId: fv(str(h.ItemId), str(h.ItemId), page.url),
+ planName: fv(str(h.floorPlan), str(h.floorPlan), page.url),
+ availabilityStatus: fv(h.availabilityStatus === "1" ? "available" : str(h.availabilityStatus), str(h.availabilityStatus), page.url),
+ images: fv<string[]>(photo ? [photo] : [], null, page.url, photo ? "builder listing photo" : null),
+ },
+ });
+ }
+ return { records, errors: [] };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4937ca1..e5bc0e2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -355,6 +355,28 @@ importers:
specifier: ^4.0.0
version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+ collectors/taylor-morrison:
+ dependencies:
+ '@spechomes/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@spechomes/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@spechomes/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
collectors/toll-brothers:
dependencies:
'@spechomes/collectors-common':
← 08c6d17 fix(collectors): raise DRH+Lennar default PAGE_LIMIT 8→300 f
·
back to Homesonspec
·
fix(workers): declare taylor-morrison collector dep so pnpm 2570ea0 →