`collectors/lennar/src/index.ts`

260 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
  fetchFixtures,
  LiveFetcher,
  type ExtractionOutput,
  type FetchContext,
  type RawPage,
  type SourceAdapter,
} from "@homesonspec/collectors-common";

/**
 * Lennar adapter — EMBEDDED_JSON source (recon 2026-07-22, difficulty 2).
 * Community pages under /new-homes/ ship a fully-hydrated Apollo state in
 * __NEXT_DATA__: CommunityType (name, cityRef, stateCode, zipCode, hoa,
 * lat/lon) + per-home HomesiteType records (price, beds, baths, sqft,
 * status, address, lat/lon, planRef). robots.txt permits the /new-homes
 * tree (only fragment paths disallowed).
 *
 * Statewide batching: LENNAR_PAGE_LIMIT community pages per run
 * (default 300 — enough to cover a whole state in one pass; there is no
 * cursor to advance past the head, so a low cap silently strands the tail)
 * at the registry rate limit; change detection makes re-visits cheap
 * (unchanged bytes stage nothing).
 */

const SITEMAP_URL = "https://www.lennar.com/api/images/sitemapfe.xml";
const BUILDER_SLUG = "lennar";
const PAGE_LIMIT = Number(process.env.LENNAR_PAGE_LIMIT ?? 300);

function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}

/** Community URL = /new-homes/{state}/{metro}/{city}/{community} exactly. */
function isCommunityUrl(url: string): boolean {
  const path = url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "");
  const segments = path.split("/").filter(Boolean);
  return segments[0] === "new-homes" && segments.length === 5;
}

function titleCase(slug: string): string {
  return slug.split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
}

type Apollo = Record<string, Record<string, unknown> | undefined>;

/** Resolve an Apollo `{__ref}` pointer (or return the value if plain). */
function deref(apollo: Apollo, value: unknown): Record<string, unknown> | null {
  if (value && typeof value === "object") {
    const ref = (value as { __ref?: string }).__ref;
    if (ref) return apollo[ref] ?? null;
    return value as Record<string, unknown>;
  }
  return null;
}

function str(value: unknown): string | null {
  return typeof value === "string" && value.trim() ? value.trim() : null;
}

/**
 * Pull a clean https image URL from a Lennar Apollo image node. Handles both
 * ImageType ({ url }) and TitleImageType ({ image: { url } }), deref-ing refs,
 * and drops the ?d=…&w=… CDN sizing query so the largest asset is stored.
 */
function imgUrl(apollo: Apollo, node: unknown): string | null {
  const n = deref(apollo, node) ?? (node as Record<string, unknown> | null);
  if (!n || typeof n !== "object") return null;
  const inner = (deref(apollo, n.image) ?? n.image ?? n) as Record<string, unknown>;
  const u = inner?.url;
  return typeof u === "string" && /^https:\/\//.test(u) ? u.split("?")[0]! : null;
}

/**
 * Builder listing photos for a home, best-effort and deduped: the homesite's own
 * elevation photo first, then its plan's hero + elevation, then the community
 * hero + gallery as fallback. Facts-only sources yield []; never guessed.
 */
function collectImages(
  apollo: Apollo,
  home: Record<string, unknown>,
  plan: Record<string, unknown> | null,
  rollup: Record<string, unknown> | undefined,
): string[] {
  const out: string[] = [];
  const push = (v: unknown) => {
    const u = imgUrl(apollo, v);
    if (u) out.push(u);
  };
  push(home.elevationImage);
  if (plan) {
    push(plan.heroImage);
    const el = plan['elevationImages({"first":1})'];
    if (Array.isArray(el)) el.forEach(push);
  }
  push(rollup?.heroImage);
  const gallery = rollup?.heroGalleryImages;
  if (Array.isArray(gallery)) gallery.slice(0, 4).forEach(push);
  return [...new Set(out)].slice(0, 6);
}

function numOrNull(value: unknown): number | null {
  const n = Number(value);
  return Number.isFinite(n) && n !== 0 ? n : null; // Lennar uses 0 for "no value"
}

function statusToEnum(status: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
  if (!status) return null;
  if (/under.?construction/i.test(status)) return "UNDER_CONSTRUCTION";
  if (/move.?in.?ready|completed|ready/i.test(status)) return "MOVE_IN_READY";
  if (/coming.?soon|planned|future/i.test(status)) return "PLANNED";
  return null; // UNDEFINED etc — recorded raw in availabilityStatus, never guessed
}

export const lennarAdapter: SourceAdapter = {
  key: "lennar-site",
  version: "1.1.0",

  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") {
      yield* fetchFixtures(ctx);
      return;
    }
    const fetcher = new LiveFetcher(ctx.registry);
    const sitemap = await fetcher.fetch(SITEMAP_URL);
    let urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
      .map((m) => m[1]!)
      .filter(isCommunityUrl);
    // Drop obvious non-community length-5 URLs (promo/event pages waste fetches).
    urls = urls.filter((u) => !/\/(promo|event)\//i.test(u));
    // LENNAR_STATE_FILTER=texas restricts the sweep to one state so coverage can
    // be filled targeted + fast instead of grinding the whole sitemap alphabetically.
    const stateFilter = process.env.LENNAR_STATE_FILTER?.trim().toLowerCase();
    if (stateFilter) urls = urls.filter((u) => u.toLowerCase().includes(`/new-homes/${stateFilter}/`));
    urls.sort();
    for (const url of urls.slice(0, PAGE_LIMIT)) {
      // A dead/stale sitemap URL (404) or a transient blip must SKIP, never abort
      // the whole sweep — large builder sitemaps always carry some rot.
      try {
        yield await fetcher.fetch(url);
      } catch (error) {
        console.warn(`  skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
      }
    }
  },

  extract(page: RawPage): ExtractionOutput {
    const errors: { url: string; reason: string }[] = [];
    try {
      const html = page.body.toString("utf8");
      // Community sales-office phone — the "broker" contact shown on listings.
      const phoneRaw = html.match(/"phoneNumber"\s*:\s*"([^"]+)"/)?.[1] ?? null;
      const salesPhone = phoneRaw && /\d{3}[-.\s]?\d{3}[-.\s]?\d{4}/.test(phoneRaw) ? phoneRaw : null;
      const match = html.match(/<script id="__NEXT_DATA__" type="application\/json"[^>]*>([\s\S]*?)<\/script>/);
      if (!match) return { records: [], errors: [{ url: page.url, reason: "no __NEXT_DATA__" }] };
      const nextData = JSON.parse(match[1]!) as {
        props?: { pageProps?: { initialApolloState?: Apollo } };
      };
      const apollo: Apollo = nextData.props?.pageProps?.initialApolloState ?? {};

      const segments = page.url.replace(/^https?:\/\/[^/]+/, "").split("/").filter(Boolean);
      const urlState = normalizeStateCode(segments[1] ?? null);
      const urlCity = segments[3] ? titleCase(segments[3]) : null;

      const rollup = Object.entries(apollo).find(([key]) => key.startsWith("CommunityType:"))?.[1];
      const communityName = str(rollup?.name) ?? (segments[4] ? titleCase(segments[4]) : null);
      if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community name" }] };

      const cityNode = deref(apollo, rollup?.city);
      const cityName = str(cityNode?.name) ?? urlCity;
      const stateCode = normalizeStateCode(str(rollup?.stateCode)) ?? urlState;
      const zipRaw = str(rollup?.zipCode);
      const zip = zipRaw && /^\d{5}$/.test(zipRaw) ? zipRaw : null;
      const hoaMonthly = numOrNull(rollup?.monthlyHoaFee);

      const records: ExtractedRecord[] = [];

      // Community first — publish creates the FK target before homes validate.
      records.push({
        entityType: "community",
        canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
        fields: {
          name: fv(communityName, str(rollup?.name), page.url),
          street: fv(str(rollup?.address), str(rollup?.address), page.url),
          city: fv(cityName, str(cityNode?.name) ?? urlCity, page.url),
          state: fv(stateCode, str(rollup?.stateCode) ?? segments[1] ?? null, page.url),
          zip: fv(zip, zipRaw, page.url),
          county: fv<string>(null, null, page.url),
          metro: fv(segments[2] ? titleCase(segments[2]) : null, segments[2] ?? null, page.url),
          lat: fv(numOrNull(rollup?.latitude), rollup?.latitude === undefined ? null : String(rollup?.latitude), page.url),
          lon: fv(numOrNull(rollup?.longitude), rollup?.longitude === undefined ? null : String(rollup?.longitude), page.url),
          hoaFeeMonthly: fv(hoaMonthly, rollup?.monthlyHoaFee === undefined ? null : String(rollup?.monthlyHoaFee), page.url),
          schoolDistrict: fv<string>(null, null, page.url),
          ageRestricted: fv<boolean>(null, null, page.url),
          salesPhone: fv(salesPhone, salesPhone, page.url, salesPhone ? "community sales phone" : null),
        },
      });

      for (const [key, entry] of Object.entries(apollo)) {
        if (!key.startsWith("HomesiteType:") || !entry) continue;
        const home = entry;
        const status = str(home.status);
        if (status && /sold|closed/i.test(status)) continue; // not available inventory
        const address = str(home.address);
        if (!address) continue; // no address → no canonical identity for a per-home record

        const bathsFull = numOrNull(home.baths) ?? (home.baths === 0 ? null : null);
        const halfBaths = Number(home.halfBaths ?? 0);
        const baths = bathsFull !== null ? bathsFull + (Number.isFinite(halfBaths) ? halfBaths * 0.5 : 0) : null;

        const plan = deref(apollo, home.plan);
        const planName = str(plan?.name);
        const garages = numOrNull(plan?.garages);
        const lotid = str(home.lotid) ?? str(home.id);

        records.push({
          entityType: "inventory_home",
          canonicalHints: {
            builderSlug: BUILDER_SLUG,
            communityName,
            address,
            builderInventoryId: lotid,
            lat: numOrNull(home.latitude),
            lon: numOrNull(home.longitude),
            planName,
          },
          fields: {
            street: fv(address, str(home.address), page.url),
            city: fv(cityName, str(cityNode?.name) ?? urlCity, page.url),
            state: fv(stateCode, str(rollup?.stateCode) ?? segments[1] ?? null, page.url),
            zip: fv(zip, zipRaw, page.url),
            price: fv(numOrNull(home.price), home.price === undefined ? null : String(home.price), page.url,
              home.price ? `price: ${String(home.price)} (${key})` : null),
            beds: fv(numOrNull(home.beds), home.beds === undefined ? null : String(home.beds), page.url),
            bathsTotal: fv(baths, home.baths === undefined ? null : `${String(home.baths)} + ${String(home.halfBaths ?? 0)} half`, page.url),
            sqft: fv(numOrNull(home.sqft), home.sqft === undefined ? null : String(home.sqft), page.url),
            stories: fv<number>(null, null, page.url),
            garageSpaces: fv(garages, plan?.garages === undefined ? null : String(plan?.garages), page.url),
            homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Lennar homesite listing"),
            constructionStatus: fv(statusToEnum(status), status, page.url),
            estCompletionDate: fv<string>(null, null, page.url),
            lotNumber: fv(str(home.number), str(home.number), page.url),
            builderInventoryId: fv(lotid, lotid, page.url),
            lat: fv(numOrNull(home.latitude), home.latitude === undefined ? null : String(home.latitude), page.url),
            lon: fv(numOrNull(home.longitude), home.longitude === undefined ? null : String(home.longitude), page.url),
            planName: fv(planName, planName, page.url),
            availabilityStatus: fv(status, status, page.url),
            images: fv<string[]>(collectImages(apollo, home, plan, rollup), null, page.url, "builder listing photos"),
          },
        });
      }

      return { records, errors };
    } catch (error) {
      return { records: [], errors: [{ url: page.url, reason: String(error) }] };
    }
  },
};