`collectors/highland-homes-fl/src/index.ts`

126 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import { fetchFixtures, LiveFetcher, type ExtractionOutput, type FetchContext, type RawPage, type SourceAdapter } from "@homesonspec/collectors-common";

/**
 * Florida Highland Homes (highlandhomes.org), deliberately separate from the
 * existing Texas highlandhomes.com collector. Highland's own About page says it
 * joined Clayton Properties Group in 2019; its current brochure calls it "A
 * Clayton Company" and its Organization JSON-LD anchors it at Lakeland, FL.
 *
 * Source: ordinary GET of the public sitemap and per-home pages. robots.txt
 * allows these paths (only /team-portal, /my-favorites and /ajax are disallowed).
 * Facts come from the page's first-party JSON-LD graph: RealEstateListing,
 * SingleFamilyResidence, Offer, Organization and BreadcrumbList. Images are
 * intentionally omitted (mediaRights=NONE).
 */
const ORIGIN = "https://www.highlandhomes.org";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const BUILDER_SLUG = "highland-homes-fl";
const PAGE_LIMIT = Number(process.env.HIGHLAND_FL_PAGE_LIMIT ?? "60");

function fv<T>(value: T | null, raw: string | null, sourceUrl: string): FieldValue<T> {
  return { value, raw, evidenceText: raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const clean = (v: unknown): string | null => v == null ? null : String(v).replace(/<[^>]+>/g, "").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim() || null;
const num = (v: unknown): number | null => { const n = typeof v === "number" ? v : Number(String(v ?? "").replace(/[^0-9.]/g, "")); return Number.isFinite(n) && n > 0 ? n : null; };
const obj = (v: unknown): Record<string, unknown> | null => v && typeof v === "object" && !Array.isArray(v) ? v as Record<string, unknown> : null;
const types = (o: Record<string, unknown>, t: string) => (Array.isArray(o["@type"]) ? o["@type"] : [o["@type"]]).includes(t);

function jsonLd(html: string): Record<string, unknown>[] {
  const out: Record<string, unknown>[] = [];
  for (const m of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
    try {
      const parsed = JSON.parse(m[1]!.trim());
      for (const top of (Array.isArray(parsed) ? parsed : [parsed])) {
        const graph = obj(top)?.["@graph"];
        for (const node of (Array.isArray(graph) ? graph : [top])) if (obj(node)) out.push(node as Record<string, unknown>);
      }
    } catch { /* malformed JSON-LD is not evidence */ }
  }
  return out;
}

export type HighlandFlHome = {
  street: string; city: string | null; state: string | null; zip: string | null;
  community: string | null; planName: string | null; lotNumber: string | null;
  price: number | null; beds: number | null; baths: number | null; sqft: number | null;
  stories: number | null; garages: number | null; lat: number | null; lon: number | null;
  status: "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null; id: string;
};

export function statusFromAvailability(value: unknown): HighlandFlHome["status"] {
  const s = clean(value)?.toLowerCase() ?? "";
  if (s.includes("instock") || s.includes("move-in ready") || s.includes("move in ready")) return "MOVE_IN_READY";
  if (s.includes("under construction") || s.includes("preorder")) return "UNDER_CONSTRUCTION";
  if (s.includes("coming soon") || s.includes("pre-sale")) return "PLANNED";
  return null;
}

function prop(listing: Record<string, unknown>, name: string): number | null {
  const props = Array.isArray(listing.additionalProperty) ? listing.additionalProperty : [];
  const hit = props.map(obj).find((p) => clean(p?.name)?.toLowerCase() === name.toLowerCase());
  return num(hit?.value);
}

export function parseHome(html: string, pageUrl: string): HighlandFlHome | null {
  const nodes = jsonLd(html);
  const listing = nodes.find((o) => types(o, "RealEstateListing"));
  const residence = nodes.find((o) => types(o, "SingleFamilyResidence"));
  if (!listing || !residence) return null;
  const address = obj(residence.address);
  const street = clean(address?.streetAddress);
  if (!street) return null;
  const state = normalizeStateCode(clean(address?.addressRegion));
  if (state !== "FL") return null; // collision guard: this collector is Florida-only
  const zip = clean(address?.postalCode)?.match(/\b3\d{4}\b/)?.[0] ?? null;
  const offerRef = obj(listing.offers)?.["@id"];
  const offer = nodes.find((o) => o["@id"] === offerRef) ?? obj(listing.offers);
  const geo = obj(residence.geo);
  const contained = obj(residence.containedInPlace);
  const communityUrl = clean(contained?.url);
  const community = communityUrl ? communityUrl.split("/").filter(Boolean).at(-1)!.split("-").map((x) => x[0]!.toUpperCase() + x.slice(1)).join(" ") : null;
  const name = clean(residence.name);
  const planName = name?.match(/^(.+?)\s+at\s+/i)?.[1] ?? null;
  const id = clean(residence["@id"])?.replace(/#inventory$/, "").split("/").filter(Boolean).at(-1) ?? pageUrl.split("/").filter(Boolean).at(-1)!;
  return {
    street, city: clean(address?.addressLocality), state, zip, community, planName, lotNumber: id,
    price: num(offer?.price), beds: prop(listing, "Bedrooms"), baths: prop(listing, "Bathrooms"),
    sqft: prop(listing, "Total Square Feet") ?? num(obj(residence.floorSize)?.value),
    stories: prop(listing, "Stories") ?? prop(residence, "Number of Stories"),
    garages: prop(listing, "Garage Spaces") ?? prop(residence, "Garage Spaces"),
    lat: num(geo?.latitude), lon: geo?.longitude == null ? null : Number(geo.longitude),
    status: statusFromAvailability(offer?.availability ?? listing.description), id,
  };
}

function isDetailUrl(url: string): boolean {
  const path = new URL(url).pathname.split("/").filter(Boolean);
  return path.length === 7 && path[0] === "new-homes" && path[1] === "florida" && !path.includes("brochure");
}

export const highlandHomesFlAdapter: SourceAdapter = {
  key: "highland-homes-fl-site", version: "0.1.0",
  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
    if (ctx.mode === "fixture") { yield* fetchFixtures(ctx); return; }
    const fetcher = new LiveFetcher(ctx.registry);
    const sitemap = await fetcher.fetch(SITEMAP);
    const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>\s*([^<]+)\s*<\/loc>/g)].map((m) => m[1]!).filter(isDetailUrl);
    for (const url of urls.slice(0, PAGE_LIMIT)) try { yield await fetcher.fetch(url); } catch (error) { console.warn(`skip ${url}: ${String(error)}`); }
  },
  extract(page: RawPage): ExtractionOutput {
    try {
      const h = parseHome(page.body.toString("utf8"), page.url);
      if (!h) return { records: [], errors: [] };
      const cname = h.community ?? "Unknown";
      const community: ExtractedRecord = { entityType: "community", canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname }, fields: {
        name: fv(cname,cname,page.url), street:fv<string>(null,null,page.url), city:fv(h.city,h.city,page.url), state:fv(h.state,h.state,page.url), zip:fv(h.zip,h.zip,page.url), county:fv<string>(null,null,page.url), metro:fv<string>(null,null,page.url), lat:fv(h.lat,h.lat==null?null:String(h.lat),page.url), lon:fv(h.lon,h.lon==null?null:String(h.lon),page.url), hoaFeeMonthly:fv<number>(null,null,page.url), schoolDistrict:fv<string>(null,null,page.url), ageRestricted:fv<boolean>(null,null,page.url), salesPhone:fv<string>(null,null,page.url)
      }};
      const home: ExtractedRecord = { entityType: "inventory_home", canonicalHints: { builderSlug: BUILDER_SLUG, communityName:cname, address:h.street, builderInventoryId:h.id, planName:h.planName??undefined, lotNumber:h.lotNumber??undefined, lat:h.lat??undefined, lon:h.lon??undefined }, fields: {
        street:fv(h.street,h.street,page.url), city:fv(h.city,h.city,page.url), state:fv(h.state,h.state,page.url), zip:fv(h.zip,h.zip,page.url), price:fv(h.price,h.price==null?null:String(h.price),page.url), beds:fv(h.beds,h.beds==null?null:String(h.beds),page.url), bathsTotal:fv(h.baths,h.baths==null?null:String(h.baths),page.url), sqft:fv(h.sqft,h.sqft==null?null:String(h.sqft),page.url), stories:fv(h.stories,h.stories==null?null:String(h.stories),page.url), garageSpaces:fv(h.garages,h.garages==null?null:String(h.garages),page.url), homeType:fv("SINGLE_FAMILY" as const,"SingleFamilyResidence",page.url), constructionStatus:fv(h.status,h.status,page.url), estCompletionDate:fv<string>(null,null,page.url), lotNumber:fv(h.lotNumber,h.lotNumber,page.url), builderInventoryId:fv(h.id,h.id,page.url), lat:fv(h.lat,h.lat==null?null:String(h.lat),page.url), lon:fv(h.lon,h.lon==null?null:String(h.lon),page.url), planName:fv(h.planName,h.planName,page.url), images:fv<string[]>([],null,page.url)
      }};
      return { records:[community,home], errors:[] };
    } catch (error) { return { records:[], errors:[{url:page.url,reason:String(error)}] }; }
  }
};