← back to Homesonspec

collectors/dr-horton/src/index.ts

228 lines

import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
  fetchFixtures,
  LiveFetcher,
  type ExtractionOutput,
  type FetchContext,
  type RawPage,
  type SourceAdapter,
} from "@homesonspec/collectors-common";

/**
 * D.R. Horton adapter — EMBEDDED_JSON source (recon 2026-07-23).
 * Sitecore/ASP.NET, server-rendered. Each community page embeds the full
 * inventory as `var model = {...};` whose Items[] carry price/beds/baths/sqft/
 * address/status/plan/lot + a Thumbnail. Per-community sales phone is a tel:
 * link. Plain HTTP (real UA) — no browser, no anti-bot on page reads.
 *
 * Scope-limited to one state via DRHORTON_STATE (default california) and
 * DRHORTON_PAGE_LIMIT community pages per run.
 */
const SITEMAP_URL = "https://www.drhorton.com/sitemaps/website.xml";
const ORIGIN = "https://www.drhorton.com";
const BUILDER_SLUG = "dr-horton";
const STATE = (process.env.DRHORTON_STATE ?? "california").toLowerCase();
// Statewide default: DRH's CA sitemap alone has ~226 communities. A low cap
// silently surfaces <4% of inventory (no cursor exists to advance past the
// head), so the default must cover a whole state in one run. contentHash
// snapshot-dedup keeps unchanged communities cheap on recurring runs.
const PAGE_LIMIT = Number(process.env.DRHORTON_PAGE_LIMIT ?? 300);

function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}

function titleCase(slug: string): string {
  return slug.split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
}

/** A community-detail URL is /{state}/{metro}/{city}/{community} — exactly 4 segments,
 *  none of them the floor-plans / qmis child pages. */
function isCommunityUrl(url: string): boolean {
  const path = url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "");
  const seg = path.split("/").filter(Boolean);
  return seg.length === 4 && seg[0] === STATE && !["floor-plans", "qmis", "qmi", "quick-move-in"].includes(seg[3]!);
}

/** Robustly extract an assigned JSON object literal (`var name = {...};`) via brace matching. */
function extractAssignedJson(html: string, varName: string): Record<string, unknown> | null {
  const marker = `var ${varName} = `;
  const at = html.indexOf(marker);
  if (at < 0) return null;
  const start = html.indexOf("{", at);
  if (start < 0) return null;
  let depth = 0, inStr = false, esc = false;
  for (let j = start; j < html.length; j++) {
    const c = html[j]!;
    if (inStr) {
      if (esc) esc = false;
      else if (c === "\\") esc = true;
      else if (c === '"') inStr = false;
    } else if (c === '"') inStr = true;
    else if (c === "{") depth++;
    else if (c === "}") {
      depth--;
      if (depth === 0) {
        try { return JSON.parse(html.slice(start, j + 1)); } catch { return null; }
      }
    }
  }
  return null;
}

/** "Manteca CA, 95337" → { city, state, zip } */
function parseCityStateZip(v: string | null): { city: string | null; state: string | null; zip: string | null } {
  if (!v) return { city: null, state: null, zip: null };
  const m = v.match(/^(.*?)[,\s]+([A-Z]{2})[,\s]+(\d{5})/);
  if (!m) return { city: null, state: null, zip: null };
  return { city: m[1]!.trim(), state: m[2]!, zip: m[3]! };
}

function statusToEnum(status: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
  const s = (status ?? "").toLowerCase();
  if (/ready|complete|move.?in/.test(s)) return "MOVE_IN_READY";
  if (/coming.?soon|planned|future/.test(s)) return "PLANNED";
  return "UNDER_CONSTRUCTION"; // "Available" spec homes still building → conservative
}

const num = (v: unknown): number | null => {
  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
  return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() ? v.trim() : null);

// Community coordinate from the page's schema.org ld+json block (lowercase "latitude"/"longitude" —
// the community's own geo; the UPPERCASE "Latitude" entries are a nearby-communities list, ignored).
// DRH already fetches these pages, so this is a $0 parse-only geo source. State-gated: rejected if the
// ld+json addressregion doesn't match the community's parsed state, or the coord fails a coarse US
// bbox (a wrong pin is worse than null — same principle as the Census geocoder). [yolo iter-4]
function communityGeo(html: string, state: string | null): { lat: number; lon: number } | null {
  const m = html.match(/"latitude"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"longitude"\s*:\s*(-?\d+(?:\.\d+)?)/);
  if (!m) return null;
  const lat = Number(m[1]), lon = Number(m[2]);
  if (!Number.isFinite(lat) || !Number.isFinite(lon) || lat === 0 || lon === 0) return null;
  if (lat < 15 || lat > 72 || lon < -180 || lon > -60) return null; // coarse US bbox (lon must be west)
  const region = html.match(/"addressregion"\s*:\s*"([A-Za-z]{2})"/i)?.[1]?.toUpperCase();
  if (state && region && region !== String(state).toUpperCase()) return null; // wrong-state -> reject
  return { lat, lon };
}

export const drHortonAdapter: SourceAdapter = {
  key: "dr-horton-site",
  version: "1.0.0",

  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") {
      yield* fetchFixtures(ctx);
      return;
    }
    const fetcher = new LiveFetcher(ctx.registry);
    const sitemap = await fetcher.fetch(SITEMAP_URL);
    const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
      .map((m) => m[1]!.replace(/^http:\/\//, "https://"))
      .filter(isCommunityUrl);
    urls.sort();
    for (const url of urls.slice(0, PAGE_LIMIT)) {
      try {
        yield await fetcher.fetch(url);
      } catch (error) {
        console.warn(`  skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
      }
    }
  },

  extract(page: RawPage): ExtractionOutput {
    const errors: { url: string; reason: string }[] = [];
    try {
      const html = page.body.toString("utf8");
      const model = extractAssignedJson(html, "model");
      const items = (model?.Items as Record<string, unknown>[] | undefined) ?? [];

      const seg = page.url.replace(/^https?:\/\/[^/]+/, "").split("/").filter(Boolean);
      const communityName = titleCase(seg[3] ?? "");
      if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community in url" }] };

      // Community geography from the first item (Items carry CityStateZip).
      const first = items[0] ?? {};
      const geo = parseCityStateZip(str(first.CityStateZip));
      const cgeo = communityGeo(html, geo.state); // $0 in-feed community coord (state-gated) — [yolo iter-4]
      const urlCity = seg[2] ? titleCase(seg[2]) : null;
      const salesPhoneRaw = html.match(/tel:([+0-9()\s.\-]{7,})/)?.[1] ?? null;
      const salesPhone = salesPhoneRaw ? salesPhoneRaw.replace(/[^0-9+]/g, "") : null;

      const records: ExtractedRecord[] = [];
      records.push({
        entityType: "community",
        canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
        fields: {
          name: fv(communityName, communityName, page.url),
          street: fv<string>(null, null, page.url),
          city: fv(geo.city ?? urlCity, geo.city, page.url),
          state: fv(geo.state, geo.state, page.url), // real parsed state, not hardcoded CA
          zip: fv(geo.zip, geo.zip, page.url),
          county: fv<string>(null, null, page.url),
          metro: fv(seg[1] ? titleCase(seg[1]) : null, seg[1] ?? null, page.url),
          lat: fv(cgeo?.lat ?? null, cgeo ? String(cgeo.lat) : null, page.url, cgeo ? "community ld+json geo" : null),
          lon: fv(cgeo?.lon ?? null, cgeo ? String(cgeo.lon) : null, page.url, cgeo ? "community ld+json geo" : null),
          hoaFeeMonthly: fv<number>(null, null, page.url),
          schoolDistrict: fv<string>(null, null, page.url),
          ageRestricted: fv<boolean>(null, null, page.url),
          salesPhone: fv(salesPhone && salesPhone.length >= 10 ? salesPhone : null, salesPhoneRaw, page.url, "community sales phone"),
        },
      });

      for (const it of items) {
        const address = str(it.Address);
        if (!address) continue;
        const g = parseCityStateZip(str(it.CityStateZip));
        const halfB = num(it.NumberOfHalfBathrooms) ?? 0;
        const fullB = num(it.NumberOfBathrooms);
        const baths = fullB !== null ? fullB + halfB * 0.5 : null;
        const status = str(it.Status);
        const thumb = str(it.Thumbnail);
        const image = thumb ? (thumb.startsWith("http") ? thumb : ORIGIN + thumb) : null;
        const lot = str(it.LotNumber) ?? str(it.ItemId) ?? str(it.Id);
        const planName = str(it.PlanName);

        records.push({
          entityType: "inventory_home",
          canonicalHints: {
            builderSlug: BUILDER_SLUG,
            communityName,
            address,
            builderInventoryId: lot,
            planName,
          },
          fields: {
            street: fv(address, address, page.url),
            city: fv(g.city ?? geo.city ?? urlCity, g.city, page.url),
            state: fv(g.state, g.state, page.url), // real parsed state, not hardcoded CA
            zip: fv(g.zip ?? geo.zip, g.zip, page.url),
            price: fv(num(it.Price), it.Price === undefined ? null : String(it.Price), page.url,
              it.Price ? `price ${String(it.Price)}` : null),
            beds: fv(num(it.NumberOfBedrooms), it.NumberOfBedrooms === undefined ? null : String(it.NumberOfBedrooms), page.url),
            bathsTotal: fv(baths, it.NumberOfBathrooms === undefined ? null : `${String(it.NumberOfBathrooms)} full + ${String(halfB)} half`, page.url),
            sqft: fv(num(it.SquareFootage), it.SquareFootage === undefined ? null : String(it.SquareFootage), page.url),
            stories: fv(num(it.NumberOfStories), null, page.url),
            garageSpaces: fv(num(it.NumberOfGarages), null, page.url),
            lat: fv(cgeo?.lat ?? null, cgeo ? String(cgeo.lat) : null, page.url, cgeo ? "community ld+json geo (home inherits community coord)" : null),
            lon: fv(cgeo?.lon ?? null, cgeo ? String(cgeo.lon) : null, page.url, cgeo ? "community ld+json geo (home inherits community coord)" : null),
            homeType: fv("SINGLE_FAMILY" as const, null, page.url, "D.R. Horton inventory home"),
            constructionStatus: fv(statusToEnum(status), status, page.url),
            estCompletionDate: fv<string>(null, null, page.url),
            lotNumber: fv(str(it.LotNumber), str(it.LotNumber), page.url),
            builderInventoryId: fv(lot, lot, page.url),
            planName: fv(planName, planName, page.url),
            availabilityStatus: fv(status, status, page.url),
            images: fv<string[]>(image ? [image] : [], null, page.url, image ? "builder listing photo" : null),
          },
        });
      }

      return { records, errors };
    } catch (error) {
      return { records: [], errors: [{ url: page.url, reason: String(error) }] };
    }
  },
};