`collectors/dream-finders/src/index.ts`

218 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
  fetchFixtures,
  LiveFetcher,
  type ExtractionOutput,
  type FetchContext,
  type RawPage,
  type SourceAdapter,
} from "@homesonspec/collectors-common";

/**
 * Dream Finders adapter — EMBEDDED_JSON source (recon 2026-07-22, difficulty 1).
 * sitemap-specs.xml lists every spec/inventory home with daily lastmod —
 * effectively a native inventory feed. Each spec page carries ld+json
 * Product+Offer blocks (price, availability). robots.txt allows the
 * inventory tree (only login/favorites/sitesearch.json disallowed).
 *
 * URL shape: /new-homes/{state}/{city}/{community}/{plan}/{address-slug}/
 * Batch control: DREAMFINDERS_PAGE_LIMIT (default 10).
 */

const SITEMAP_URL = "https://dreamfindershomes.com/sitemap-specs.xml";
const BUILDER_SLUG = "dream-finders";
const PAGE_LIMIT = Number(process.env.DREAMFINDERS_PAGE_LIMIT ?? 10);

function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}

function titleCase(slug: string): string {
  return slug.split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
}

interface UrlParts {
  state: string | null;
  city: string | null;
  community: string | null;
  plan: string | null;
  address: string | null;
}

/** /new-homes/co/longmont/mountain-brook/ridgeline/2848-bear-springs-cir/ */
function parseSpecUrl(url: string): UrlParts | null {
  const segments = url.replace(/^https?:\/\/[^/]+/, "").split("/").filter(Boolean);
  if (segments[0] !== "new-homes" || segments.length < 6) return null;
  return {
    state: normalizeStateCode(segments[1] ?? null),
    city: segments[2] ? titleCase(segments[2]) : null,
    community: segments[3] ? titleCase(segments[3]) : null,
    plan: segments[4] ? titleCase(segments[4]) : null,
    address: segments[5] ? titleCase(segments[5]) : null,
  };
}

function extractLdJsonBlocks(html: string): unknown[] {
  const blocks: unknown[] = [];
  // NB: Dream Finders emits type='application/ld+json' with SINGLE quotes — the original regex
  // required double quotes and matched ZERO blocks, which was the true root of the hollow-data
  // failure (no ld+json ever parsed). Accept either quote style. [2026-07-29]
  for (const match of html.matchAll(/<script[^>]*type=['"]application\/ld\+json['"][^>]*>([\s\S]*?)<\/script>/g)) {
    try {
      blocks.push(JSON.parse(match[1]!));
    } catch {
      // malformed block — skip, never guess
    }
  }
  return blocks;
}

interface OwnHome {
  price: number | null; priceRaw: string | null;
  beds: number | null; baths: number | null; sqft: number | null;
  lat: number | null; lon: number | null;
}
const posNum = (v: unknown): number | null => {
  const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
  return Number.isFinite(n) && n > 0 ? n : null;
};

/**
 * Find THIS page's own home in the ld+json. A DF spec page carries a Product/SingleFamilyResidence
 * block PER home (its own + siblings in the community); the page's own home is the block whose `url`
 * ends with the page's address slug. The original findOwnOffer matched offers.url === pageUrl exactly
 * and got 0 (siblings' urls differ, trailing-slash drift), producing hollow 0%-price rows — the fix is
 * a slug match, and the same block also yields beds/baths/sqft + GeoCoordinates ($0 geo). [2026-07-29]
 */
function findOwnHome(blocks: unknown[], addressSlug: string): OwnHome {
  const empty: OwnHome = { price: null, priceRaw: null, beds: null, baths: null, sqft: null, lat: null, lon: null };
  const products: Record<string, unknown>[] = [];
  const walk = (node: unknown) => {
    if (!node || typeof node !== "object") return;
    if (Array.isArray(node)) { node.forEach(walk); return; }
    const obj = node as Record<string, unknown>;
    const t = JSON.stringify(obj["@type"] ?? "");
    if (/Product|SingleFamilyResidence|House/.test(t) && (obj.url || obj.offers)) products.push(obj);
    for (const v of Object.values(obj)) walk(v);
  };
  blocks.forEach(walk);
  const slug = addressSlug.replace(/\/$/, "").toLowerCase();
  // own home = the product whose url ends with the page's address slug. CRITICAL: no products[0]
  // fallback — some DF pages carry only SIBLING homes in ld+json (not their own), and blindly taking
  // the first product assigns one home's price/beds to every sibling page = the hollow-data trap that
  // originally sank this adapter. If the page's own home isn't in ld+json, return null (honest) rather
  // than a wrong home's facts. [2026-07-29]
  const own = products.find((p) => String((p.url ?? (p.offers as Record<string, unknown>)?.url) ?? "").replace(/\/$/, "").toLowerCase().endsWith(slug));
  if (!own) return empty;
  const offers = (own.offers as Record<string, unknown>) ?? {};
  const geo = (own.geo as Record<string, unknown>) ?? {};
  const bathsTotal = posNum(own.numberOfBathroomsTotal)
    ?? (posNum(own.numberOfFullBathrooms) !== null
        ? posNum(own.numberOfFullBathrooms)! + (posNum(own.numberOfPartialBathrooms) ?? 0) * 0.5
        : null);
  const sqftNode = own.floorSize as Record<string, unknown> | undefined;
  const lat = posNum(geo.latitude), lonRaw = geo.longitude;
  const lon = lonRaw != null && Number.isFinite(Number(lonRaw)) && Number(lonRaw) !== 0 ? Number(lonRaw) : null;
  return {
    price: posNum(offers.price), priceRaw: offers.price != null ? String(offers.price) : null,
    beds: posNum(own.numberOfBedrooms),
    baths: bathsTotal,
    sqft: posNum(sqftNode?.value ?? own.floorSize),
    lat: lat, // US bbox-gated below via coord()
    lon: lon !== null && lon < 0 && lon > -180 ? lon : null,
  };
}

export const dreamFindersAdapter: SourceAdapter = {
  key: "dream-finders-site",
  version: "1.0.0",

  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") {
      yield* fetchFixtures(ctx);
      return;
    }
    const fetcher = new LiveFetcher(ctx.registry);
    const sitemap = await fetcher.fetch(SITEMAP_URL);
    const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
      .map((m) => m[1]!)
      .filter((url) => parseSpecUrl(url) !== null);
    urls.sort();
    for (const url of urls.slice(0, PAGE_LIMIT)) {
      try {
        yield await fetcher.fetch(url);
      } catch (error) {
        console.warn(`  skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
      }
    }
  },

  extract(page: RawPage): ExtractionOutput {
    try {
      const parts = parseSpecUrl(page.url);
      if (!parts?.community || !parts.address) {
        return { records: [], errors: [{ url: page.url, reason: "unparseable spec URL" }] };
      }
      const html = page.body.toString("utf8");
      const blocks = extractLdJsonBlocks(html);
      const addressSlug = page.url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "").split("/").filter(Boolean).pop() ?? "";
      const own = findOwnHome(blocks, addressSlug);

      const records: ExtractedRecord[] = [
        // Community first — publish creates/refreshes the FK target.
        {
          entityType: "community",
          canonicalHints: { builderSlug: BUILDER_SLUG, communityName: parts.community },
          fields: {
            name: fv(parts.community, parts.community, page.url, "community segment of spec URL"),
            street: fv<string>(null, null, page.url),
            city: fv(parts.city, parts.city, page.url),
            state: fv(parts.state, parts.state, page.url),
            zip: fv<string>(null, null, page.url),
            county: fv<string>(null, null, page.url),
            metro: fv<string>(null, null, page.url),
            lat: fv(own.lat, own.lat === null ? null : String(own.lat), page.url, own.lat === null ? null : "ld+json GeoCoordinates"),
            lon: fv(own.lon, own.lon === null ? null : String(own.lon), page.url, own.lon === null ? null : "ld+json GeoCoordinates"),
            hoaFeeMonthly: fv<number>(null, null, page.url),
            schoolDistrict: fv<string>(null, null, page.url),
            ageRestricted: fv<boolean>(null, null, page.url),
          },
        },
        {
          entityType: "inventory_home",
          canonicalHints: {
            builderSlug: BUILDER_SLUG,
            communityName: parts.community,
            address: parts.address,
            planName: parts.plan,
          },
          fields: {
            street: fv(parts.address, parts.address, page.url, "address segment of spec URL"),
            city: fv(parts.city, parts.city, page.url),
            state: fv(parts.state, parts.state, page.url),
            zip: fv<string>(null, null, page.url),
            price: fv(own.price, own.priceRaw, page.url, own.priceRaw ? `ld+json Offer price ${own.priceRaw}` : null),
            beds: fv(own.beds, own.beds === null ? null : String(own.beds), page.url, own.beds === null ? null : "ld+json numberOfBedrooms"),
            bathsTotal: fv(own.baths, own.baths === null ? null : String(own.baths), page.url, own.baths === null ? null : "ld+json numberOfBathrooms"),
            sqft: fv(own.sqft, own.sqft === null ? null : String(own.sqft), page.url, own.sqft === null ? null : "ld+json floorSize"),
            stories: fv<number>(null, null, page.url),
            garageSpaces: fv<number>(null, null, page.url),
            homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Dream Finders spec home listing"),
            constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(null, null, page.url),
            estCompletionDate: fv<string>(null, null, page.url),
            lotNumber: fv<string>(null, null, page.url),
            builderInventoryId: fv<string>(null, null, page.url),
            lat: fv(own.lat, own.lat === null ? null : String(own.lat), page.url, own.lat === null ? null : "ld+json GeoCoordinates"),
            lon: fv(own.lon, own.lon === null ? null : String(own.lon), page.url, own.lon === null ? null : "ld+json GeoCoordinates"),
            planName: fv(parts.plan, parts.plan, page.url),
          },
        },
      ];
      return { records, errors: [] };
    } catch (error) {
      return { records: [], errors: [{ url: page.url, reason: String(error) }] };
    }
  },
};