← back to Homesonspec
Add D.R. Horton adapter (2nd builder) — Sitecore var-model blob, CA-scoped via DRHORTON_STATE, captures homes+community+salesPhone+images; verified 3 live Manteca homes; registered in pipeline
d27cab5cd074852719b1062809f84a065b4ad874 · 2026-07-22 23:34:10 -0700 · Steve
Files touched
M apps/workers/package.jsonM apps/workers/src/cli.tsA collectors/dr-horton/package.jsonA collectors/dr-horton/src/index.tsA collectors/dr-horton/tsconfig.jsonM pnpm-lock.yaml
Diff
commit d27cab5cd074852719b1062809f84a065b4ad874
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jul 22 23:34:10 2026 -0700
Add D.R. Horton adapter (2nd builder) — Sitecore var-model blob, CA-scoped via DRHORTON_STATE, captures homes+community+salesPhone+images; verified 3 live Manteca homes; registered in pipeline
---
apps/workers/package.json | 3 +-
apps/workers/src/cli.ts | 2 +
collectors/dr-horton/package.json | 15 +++
collectors/dr-horton/src/index.ts | 204 +++++++++++++++++++++++++++++++++++++
collectors/dr-horton/tsconfig.json | 1 +
pnpm-lock.yaml | 25 +++++
6 files changed, 249 insertions(+), 1 deletion(-)
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 6c8201b..0d5d5c4 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -21,7 +21,8 @@
"pg-boss": "^10.1.5",
"@spechomes/publisher": "workspace:*",
"@spechomes/collector-toll-brothers": "workspace:*",
- "@spechomes/collector-lennar": "workspace:*"
+ "@spechomes/collector-lennar": "workspace:*",
+ "@spechomes/collector-dr-horton": "workspace:*"
},
"devDependencies": {
"tsx": "^4.19.2",
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index 8e76607..1537e87 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -2,6 +2,7 @@ import { prisma } from "@spechomes/database";
import { meridianHomesAdapter } from "@spechomes/collector-meridian-homes";
import { tollBrothersAdapter } from "@spechomes/collector-toll-brothers";
import { lennarAdapter } from "@spechomes/collector-lennar";
+import { drHortonAdapter } from "@spechomes/collector-dr-horton";
import { runPipeline } from "./pipeline";
import { recordSourceRun } from "./verify";
@@ -13,6 +14,7 @@ const ADAPTERS = {
[meridianHomesAdapter.key]: meridianHomesAdapter,
[tollBrothersAdapter.key]: tollBrothersAdapter,
[lennarAdapter.key]: lennarAdapter,
+ [drHortonAdapter.key]: drHortonAdapter,
};
async function main() {
diff --git a/collectors/dr-horton/package.json b/collectors/dr-horton/package.json
new file mode 100644
index 0000000..ecade98
--- /dev/null
+++ b/collectors/dr-horton/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@spechomes/collector-dr-horton",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@spechomes/collectors-common": "workspace:*",
+ "@spechomes/schemas": "workspace:*",
+ "@spechomes/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/dr-horton/src/index.ts b/collectors/dr-horton/src/index.ts
new file mode 100644
index 0000000..2b112cf
--- /dev/null
+++ b/collectors/dr-horton/src/index.ts
@@ -0,0 +1,204 @@
+import type { ExtractedRecord, FieldValue } from "@spechomes/schemas";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@spechomes/collectors-common";
+
+/**
+ * D.R. Horton adapter — EMBEDDED_JSON source (recon 2026-07-23).
+ * Sitecore/ASP.NET, server-rendered. Each community page embeds the full
+ * inventory as `var model = {...};` whose Items[] carry price/beds/baths/sqft/
+ * address/status/plan/lot + a Thumbnail. Per-community sales phone is a tel:
+ * link. Plain HTTP (real UA) — no browser, no anti-bot on page reads.
+ *
+ * Scope-limited to one state via DRHORTON_STATE (default california) and
+ * DRHORTON_PAGE_LIMIT community pages per run.
+ */
+const SITEMAP_URL = "https://www.drhorton.com/sitemaps/website.xml";
+const ORIGIN = "https://www.drhorton.com";
+const BUILDER_SLUG = "dr-horton";
+const STATE = (process.env.DRHORTON_STATE ?? "california").toLowerCase();
+const PAGE_LIMIT = Number(process.env.DRHORTON_PAGE_LIMIT ?? 8);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+function titleCase(slug: string): string {
+ return slug.split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
+}
+
+/** A community-detail URL is /{state}/{metro}/{city}/{community} — exactly 4 segments,
+ * none of them the floor-plans / qmis child pages. */
+function isCommunityUrl(url: string): boolean {
+ const path = url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "");
+ const seg = path.split("/").filter(Boolean);
+ return seg.length === 4 && seg[0] === STATE && !["floor-plans", "qmis", "qmi", "quick-move-in"].includes(seg[3]!);
+}
+
+/** Robustly extract an assigned JSON object literal (`var name = {...};`) via brace matching. */
+function extractAssignedJson(html: string, varName: string): Record<string, unknown> | null {
+ const marker = `var ${varName} = `;
+ const at = html.indexOf(marker);
+ if (at < 0) return null;
+ const start = html.indexOf("{", at);
+ if (start < 0) return null;
+ let depth = 0, inStr = false, esc = false;
+ for (let j = start; j < html.length; j++) {
+ const c = html[j]!;
+ if (inStr) {
+ if (esc) esc = false;
+ else if (c === "\\") esc = true;
+ else if (c === '"') inStr = false;
+ } else if (c === '"') inStr = true;
+ else if (c === "{") depth++;
+ else if (c === "}") {
+ depth--;
+ if (depth === 0) {
+ try { return JSON.parse(html.slice(start, j + 1)); } catch { return null; }
+ }
+ }
+ }
+ return null;
+}
+
+/** "Manteca CA, 95337" → { city, state, zip } */
+function parseCityStateZip(v: string | null): { city: string | null; state: string | null; zip: string | null } {
+ if (!v) return { city: null, state: null, zip: null };
+ const m = v.match(/^(.*?)[,\s]+([A-Z]{2})[,\s]+(\d{5})/);
+ if (!m) return { city: null, state: null, zip: null };
+ return { city: m[1]!.trim(), state: m[2]!, zip: m[3]! };
+}
+
+function statusToEnum(status: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
+ const s = (status ?? "").toLowerCase();
+ if (/ready|complete|move.?in/.test(s)) return "MOVE_IN_READY";
+ if (/coming.?soon|planned|future/.test(s)) return "PLANNED";
+ return "UNDER_CONSTRUCTION"; // "Available" spec homes still building → conservative
+}
+
+const num = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);
+
+export const drHortonAdapter: SourceAdapter = {
+ key: "dr-horton-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ const sitemap = await fetcher.fetch(SITEMAP_URL);
+ const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+ .map((m) => m[1]!.replace(/^http:\/\//, "https://"))
+ .filter(isCommunityUrl);
+ urls.sort();
+ for (const url of urls.slice(0, PAGE_LIMIT)) {
+ try {
+ yield await fetcher.fetch(url);
+ } catch (error) {
+ console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ const errors: { url: string; reason: string }[] = [];
+ try {
+ const html = page.body.toString("utf8");
+ const model = extractAssignedJson(html, "model");
+ const items = (model?.Items as Record<string, unknown>[] | undefined) ?? [];
+
+ const seg = page.url.replace(/^https?:\/\/[^/]+/, "").split("/").filter(Boolean);
+ const communityName = titleCase(seg[3] ?? "");
+ if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community in url" }] };
+
+ // Community geography from the first item (Items carry CityStateZip).
+ const first = items[0] ?? {};
+ const geo = parseCityStateZip(str(first.CityStateZip));
+ const urlCity = seg[2] ? titleCase(seg[2]) : null;
+ const salesPhoneRaw = html.match(/tel:([+0-9()\s.\-]{7,})/)?.[1] ?? null;
+ const salesPhone = salesPhoneRaw ? salesPhoneRaw.replace(/[^0-9+]/g, "") : null;
+
+ const records: ExtractedRecord[] = [];
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
+ fields: {
+ name: fv(communityName, communityName, page.url),
+ street: fv<string>(null, null, page.url),
+ city: fv(geo.city ?? urlCity, geo.city, page.url),
+ state: fv("CA" as const, geo.state, page.url),
+ zip: fv(geo.zip, geo.zip, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv(seg[1] ? titleCase(seg[1]) : null, seg[1] ?? null, page.url),
+ lat: fv<number>(null, null, page.url),
+ lon: fv<number>(null, null, page.url),
+ hoaFeeMonthly: fv<number>(null, null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ salesPhone: fv(salesPhone && salesPhone.length >= 10 ? salesPhone : null, salesPhoneRaw, page.url, "community sales phone"),
+ },
+ });
+
+ for (const it of items) {
+ const address = str(it.Address);
+ if (!address) continue;
+ const g = parseCityStateZip(str(it.CityStateZip));
+ const halfB = num(it.NumberOfHalfBathrooms) ?? 0;
+ const fullB = num(it.NumberOfBathrooms);
+ const baths = fullB !== null ? fullB + halfB * 0.5 : null;
+ const status = str(it.Status);
+ const thumb = str(it.Thumbnail);
+ const image = thumb ? (thumb.startsWith("http") ? thumb : ORIGIN + thumb) : null;
+ const lot = str(it.LotNumber) ?? str(it.ItemId) ?? str(it.Id);
+ const planName = str(it.PlanName);
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName,
+ address,
+ builderInventoryId: lot,
+ planName,
+ },
+ fields: {
+ street: fv(address, address, page.url),
+ city: fv(g.city ?? geo.city ?? urlCity, g.city, page.url),
+ state: fv("CA" as const, g.state, page.url),
+ zip: fv(g.zip ?? geo.zip, g.zip, page.url),
+ price: fv(num(it.Price), it.Price === undefined ? null : String(it.Price), page.url,
+ it.Price ? `price ${String(it.Price)}` : null),
+ beds: fv(num(it.NumberOfBedrooms), it.NumberOfBedrooms === undefined ? null : String(it.NumberOfBedrooms), page.url),
+ bathsTotal: fv(baths, it.NumberOfBathrooms === undefined ? null : `${String(it.NumberOfBathrooms)} full + ${String(halfB)} half`, page.url),
+ sqft: fv(num(it.SquareFootage), it.SquareFootage === undefined ? null : String(it.SquareFootage), page.url),
+ stories: fv(num(it.NumberOfStories), null, page.url),
+ garageSpaces: fv(num(it.NumberOfGarages), null, page.url),
+ homeType: fv("SINGLE_FAMILY" as const, null, page.url, "D.R. Horton inventory home"),
+ constructionStatus: fv(statusToEnum(status), status, page.url),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv(str(it.LotNumber), str(it.LotNumber), page.url),
+ builderInventoryId: fv(lot, lot, page.url),
+ planName: fv(planName, planName, page.url),
+ availabilityStatus: fv(status, status, page.url),
+ images: fv<string[]>(image ? [image] : [], null, page.url, image ? "builder listing photo" : null),
+ },
+ });
+ }
+
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/dr-horton/tsconfig.json b/collectors/dr-horton/tsconfig.json
new file mode 100644
index 0000000..e9bfc48
--- /dev/null
+++ b/collectors/dr-horton/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 741066e..2ac6061 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -142,6 +142,9 @@ importers:
apps/workers:
dependencies:
+ '@spechomes/collector-dr-horton':
+ specifier: workspace:*
+ version: link:../../collectors/dr-horton
'@spechomes/collector-lennar':
specifier: workspace:*
version: link:../../collectors/lennar
@@ -208,6 +211,28 @@ importers:
specifier: ^4.0.0
version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+ collectors/dr-horton:
+ dependencies:
+ '@spechomes/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@spechomes/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@spechomes/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
collectors/dream-finders:
dependencies:
'@spechomes/collectors-common':
← 0d7bc77 Listings contact: capture per-community sales-office phone f
·
back to Homesonspec
·
Add KB Home adapter (3rd builder) — single-feed (var allMIRs 65775ba →