`collectors/discovery/src/index.ts`

207 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
  fetchFixtures,
  LiveFetcher,
  sha256,
  type ExtractionOutput,
  type FetchContext,
  type RawPage,
  type SourceAdapter,
} from "@homesonspec/collectors-common";

/**
 * Discovery Homes adapter — first REGIONAL builder (recon 2026-07-27).
 * discoveryhomes.com hosts several Seeno-family brands on one JSON API:
 *   GET /api/communities  → JSON array (name/city/state/zip/lat/lon/builder/seo_name/url)
 *   GET /api/homes        → {renderedList:"<qmi-card html>"} — a SINGLE site-wide feed
 *                           (the community_id param is IGNORED by the server).
 * So we fetch /api/homes ONCE and join each card back to its community by the
 * href slug (/ST/city/communities/<slug>/...), which yields the home's true
 * builder + location. The join happens in fetch(); extract() stays pure by
 * reading a normalized JSON page we synthesize. Plain HTTP, robots allows /api/.
 * Facts-only (images intentionally omitted). Sold/unpriced homes are skipped.
 */
const ORIGIN = "https://www.discoveryhomes.com";
const COMMUNITIES_URL = `${ORIGIN}/api/communities`;
const HOMES_URL = `${ORIGIN}/api/homes`;
const NORMALIZED_URL = `${HOMES_URL}#normalized`;
// feed builder code -> our Builder.slug (all must exist as Builder rows)
const BUILDER_MAP: Record<string, string> = {
  discovery: "discovery-homes",
  jmscc: "jmc-homes",
  seeno: "seeno-homes",
  sterling: "sterling-homes",
};

function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
  return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (v == null ? null : String(v).trim() ? String(v).trim() : null);
// lat/lon: preserve sign (US longitudes are negative) — never strip "-" or reject <0.
const coord = (v: unknown): number | null => {
  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
  return Number.isFinite(n) && n !== 0 ? n : null;
};
const slugify = (s: string) => (s || "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");

interface Community {
  id: number; name?: string; builder?: string; city?: string; state?: string; zip?: unknown;
  county?: string; latitude?: unknown; longitude?: unknown; phone?: string; seo_name?: string; url?: string;
}
interface NormHome {
  builderSlug: string; communityName: string; street: string; city: string | null; state: string | null;
  zip: string | null; price: number; priceRaw: string; beds: number | null; baths: number | null;
  sqft: number | null; plan: string | null; homeId: string;
}

/** index communities by every slug a home href might reference */
function buildIndex(comms: Community[]): Map<string, Community> {
  const idx = new Map<string, Community>();
  for (const c of comms) {
    const keys = [c.seo_name, typeof c.url === "string" ? c.url.replace(/\/+$/, "").split("/").pop() : null, slugify(c.name ?? "")];
    for (const k of keys) if (k) idx.set(String(k).toLowerCase(), c);
  }
  return idx;
}

function parseHomes(html: string, idx: Map<string, Community>): NormHome[] {
  const out: NormHome[] = [];
  for (const card of html.split("qmi-card__container").slice(1)) {
    const href = card.match(/href="\/([A-Z]{2})\/([^/]+)\/communities\/([^/]+)\/[^"]*?-(\d{5,})/i);
    if (!href) continue;
    const hrefState = href[1]!.toUpperCase();
    const commSlug = href[3]!.toLowerCase();
    const homeId = href[4]!;
    const community = idx.get(commSlug);
    if (!community) continue;
    const builderSlug = BUILDER_MAP[(community.builder ?? "").toLowerCase()];
    if (!builderSlug) continue; // builder not registered / out of scope

    const priceRaw = (card.match(/qmi-card__price[^>]*>\s*([^<]+)/i)?.[1] ?? "").trim();
    const price = /\$[\d,]/.test(priceRaw) ? num(priceRaw) : null;
    if (!price) continue; // sold / unpriced → not current for-sale inventory

    const beds = num(card.match(/class="beds"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
    const baths = num(card.match(/class="baths"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
    const sqft = num(card.match(/class="sqft"[^>]*>\s*([\d,]+)/i)?.[1] ?? null);
    const plan = card.match(/qmi-card__name[^>]*>[\s\S]*?<a[^>]*>\s*([^<]+?)\s*</i)?.[1]?.trim() ?? null;
    const addrBlock = (card.match(/qmi-card__address[^>]*>\s*([^<]+)/i)?.[1] ?? "").replace(/\s+/g, " ").trim();

    // location: state from href (authoritative), zip from address block, city from the matched community
    const zip = addrBlock.match(/\b(\d{5})(?:-\d{4})?\b/)?.[1] ?? str(community.zip);
    const city = str(community.city);
    const state = hrefState || (str(community.state) ?? "")?.toUpperCase() || null;
    let street = addrBlock.replace(/,?\s*[A-Z]{2}\s+\d{5}(?:-\d{4})?\s*$/, "").trim();
    if (city) street = street.replace(new RegExp("\\s*" + city.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\s*$", "i"), "").trim();
    if (!street) continue;

    out.push({
      builderSlug, communityName: str(community.name) ?? commSlug, street, city, state, zip,
      price, priceRaw, beds, baths, sqft, plan, homeId,
    });
  }
  return out;
}

export const discoveryAdapter: SourceAdapter = {
  key: "discovery-homes-site",
  version: "1.1.0",

  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") {
      yield* fetchFixtures(ctx);
      return;
    }
    const fetcher = new LiveFetcher(ctx.registry);
    const commPage = await fetcher.fetch(COMMUNITIES_URL);
    yield commPage; // → community records

    let comms: Community[] = [];
    try { comms = JSON.parse(commPage.body.toString("utf8")); } catch { comms = []; }
    const idx = buildIndex(comms);
    const homesPage = await fetcher.fetch(HOMES_URL);
    let html = "";
    try { html = JSON.parse(homesPage.body.toString("utf8")).renderedList ?? ""; } catch { html = ""; }
    const normalized = parseHomes(html, idx);
    const body = Buffer.from(JSON.stringify(normalized), "utf8");
    yield { url: NORMALIZED_URL, retrievedAt: new Date().toISOString(), contentType: "application/json", body, contentHash: sha256(body) };
  },

  extract(page: RawPage): ExtractionOutput {
    try {
      if (page.url.includes("#normalized")) return extractNormalizedHomes(page);
      if (page.url.includes("/api/communities")) return extractCommunities(page);
      return { records: [], errors: [] };
    } catch (error) {
      return { records: [], errors: [{ url: page.url, reason: String(error) }] };
    }
  },
};

function extractCommunities(page: RawPage): ExtractionOutput {
  let all: Community[] = [];
  try { all = JSON.parse(page.body.toString("utf8")); } catch { return { records: [], errors: [{ url: page.url, reason: "communities JSON parse failed" }] }; }
  const records: ExtractedRecord[] = [];
  for (const c of all) {
    const builderSlug = BUILDER_MAP[(c.builder ?? "").toLowerCase()];
    if (!builderSlug) continue;
    const name = str(c.name);
    if (!name) continue;
    const phone = str(c.phone);
    records.push({
      entityType: "community",
      canonicalHints: { builderSlug, communityName: name },
      fields: {
        name: fv(name, name, page.url),
        street: fv<string>(null, null, page.url),
        city: fv(str(c.city), str(c.city), page.url),
        state: fv((str(c.state) ?? "").toUpperCase() || null, str(c.state), page.url),
        zip: fv(str(c.zip), str(c.zip), page.url),
        county: fv(str(c.county), str(c.county), page.url),
        metro: fv<string>(null, null, page.url),
        lat: fv(coord(c.latitude), str(c.latitude), page.url),
        lon: fv(coord(c.longitude), str(c.longitude), page.url),
        hoaFeeMonthly: fv<number>(null, null, page.url),
        schoolDistrict: fv<string>(null, null, page.url),
        ageRestricted: fv<boolean>(null, null, page.url),
        salesPhone: fv(phone ? phone.replace(/[^0-9+]/g, "") : null, phone, page.url, "community sales phone"),
      },
    });
  }
  return { records, errors: [] };
}

function extractNormalizedHomes(page: RawPage): ExtractionOutput {
  let homes: NormHome[] = [];
  try { homes = JSON.parse(page.body.toString("utf8")); } catch { return { records: [], errors: [{ url: page.url, reason: "normalized homes parse failed" }] }; }
  const records: ExtractedRecord[] = homes.map((h) => ({
    entityType: "inventory_home" as const,
    canonicalHints: { builderSlug: h.builderSlug, communityName: h.communityName, address: h.street, builderInventoryId: h.homeId, planName: h.plan },
    fields: {
      street: fv(h.street, h.street, page.url),
      city: fv(h.city, h.city, page.url),
      state: fv(h.state, h.state, page.url),
      zip: fv(h.zip, h.zip, page.url),
      price: fv(h.price, h.priceRaw, page.url, `price ${h.priceRaw}`),
      beds: fv(h.beds, null, page.url),
      bathsTotal: fv(h.baths, null, page.url),
      sqft: fv(h.sqft, null, page.url),
      stories: fv<number>(null, null, page.url),
      garageSpaces: fv<number>(null, null, page.url),
      homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Discovery Homes quick move-in"),
      constructionStatus: fv("MOVE_IN_READY" as never, h.priceRaw, page.url),
      estCompletionDate: fv<string>(null, null, page.url),
      lotNumber: fv<string>(null, null, page.url),
      builderInventoryId: fv(h.homeId, h.homeId, page.url),
      planName: fv(h.plan, h.plan, page.url),
      availabilityStatus: fv("Quick Move-In", null, page.url),
      images: fv<string[]>([], null, page.url), // facts-only — mediaRights=NONE
    },
  }));
  return { records, errors: [] };
}