`collectors/toll-brothers/src/index.ts`

124 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
  fetchFixtures,
  LiveFetcher,
  type ExtractionOutput,
  type FetchContext,
  type RawPage,
  type SourceAdapter,
} from "@homesonspec/collectors-common";

/**
 * Toll Brothers adapter — FEED source (recon 2026-07-22).
 * Their sitemap itself lists /api/v1/search/homesearchV5.json: a public,
 * unauthenticated community feed (609 communities). Facts only; no media.
 * robots.txt: /api/ is not disallowed; we avoid the disallowed paths
 * (/siteplans, /nse, /compare, /new-homes, /sitesearch) entirely.
 *
 * Wave 1 ingests COMMUNITIES from the feed. Per-home QMI ingestion (the
 * ~11k sitemap-listed quick-move-in pages) is wave 2, gated on volume
 * controls in the workers scheduler.
 */

const FEED_URL = "https://www.tollbrothers.com/api/v1/search/homesearchV5.json";
const BUILDER_SLUG = "toll-brothers";

function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}

interface TbCommunity {
  communityId?: number | string;
  name?: string;
  url?: string;
  state?: string;
  city?: string;
  county?: string;
  metroName?: string;
  lat?: number | string;
  lon?: number | string;
  pricedFrom?: number | string;
  isFuture?: boolean;
}

function toNumber(value: unknown): number | null {
  if (value === null || value === undefined || value === "") return null;
  const n = Number(value);
  return Number.isFinite(n) ? n : null;
}

function extractCommunity(community: TbCommunity, sourceUrl: string): ExtractedRecord | null {
  const name = community.name?.trim() || null;
  if (!name) return null;
  const stateCode = normalizeStateCode(community.state ?? null);
  const lat = toNumber(community.lat);
  const lon = toNumber(community.lon);
  const evidence = (field: string, raw: unknown) =>
    raw === null || raw === undefined ? null : `${field}: ${String(raw)} (homesearchV5 community ${community.communityId ?? "?"})`;

  return {
    entityType: "community",
    canonicalHints: {
      builderSlug: BUILDER_SLUG,
      communityName: name,
      builderInventoryId: community.communityId ? String(community.communityId) : null,
      lat,
      lon,
    },
    fields: {
      name: fv(name, name, sourceUrl),
      street: fv<string>(null, null, sourceUrl), // feed does not carry a street address
      city: fv(community.city?.trim() || null, community.city ?? null, sourceUrl, evidence("city", community.city)),
      state: fv(stateCode, community.state ?? null, sourceUrl, evidence("state", community.state)),
      zip: fv<string>(null, null, sourceUrl), // not in feed — null, never guessed
      county: fv(community.county?.trim() || null, community.county ?? null, sourceUrl),
      metro: fv(community.metroName?.trim() || null, community.metroName ?? null, sourceUrl),
      lat: fv(lat, community.lat === undefined ? null : String(community.lat), sourceUrl),
      lon: fv(lon, community.lon === undefined ? null : String(community.lon), sourceUrl),
      hoaFeeMonthly: fv<number>(null, null, sourceUrl),
      schoolDistrict: fv<string>(null, null, sourceUrl),
      ageRestricted: fv<boolean>(null, null, sourceUrl),
      // Extra evidence-bearing facts (not in the strict community schema, harmless):
      sourcePageUrl: fv(community.url ?? null, community.url ?? null, sourceUrl),
      pricedFrom: fv(toNumber(community.pricedFrom), community.pricedFrom === undefined ? null : String(community.pricedFrom), sourceUrl),
    },
  };
}

export const tollBrothersAdapter: SourceAdapter = {
  key: "toll-brothers-site",
  version: "1.0.0",

  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") {
      yield* fetchFixtures(ctx);
      return;
    }
    const fetcher = new LiveFetcher(ctx.registry);
    yield await fetcher.fetch(FEED_URL);
  },

  extract(page: RawPage): ExtractionOutput {
    try {
      const data = JSON.parse(page.body.toString("utf8")) as {
        perfect?: TbCommunity[];
        partial?: TbCommunity[];
        count?: number;
      };
      const communities = [...(data.perfect ?? []), ...(data.partial ?? [])];
      const records: ExtractedRecord[] = [];
      const errors: { url: string; reason: string }[] = [];
      for (const community of communities) {
        if (community.isFuture) continue; // future communities have no purchasable inventory yet
        const record = extractCommunity(community, page.url);
        if (record) records.push(record);
        else errors.push({ url: page.url, reason: `community without a name (id ${community.communityId ?? "?"})` });
      }
      return { records, errors };
    } catch (error) {
      return { records: [], errors: [{ url: page.url, reason: `feed parse: ${String(error)}` }] };
    }
  },
};