← back to Homesonspec
collectors/legacy-al/src/index.ts
409 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Legacy Homes Alabama adapter — the North Alabama site-built spec builder in
* Clayton Properties Group (slug "legacy-al"; recon +
* built 2026-08-30, TK-10487). Same architecture class as the goodall-homes and
* harris-doyle adapters: Clayton has no unified site-built feed, so each brand
* gets its own adapter on its own domain.
*
* SITE IDENTITY: https://www.legacyhomesal.com. The sitemap inventory spans
* Huntsville-area cities including Harvest, Meridianville, Athens, Hazel Green,
* New Market, and Huntsville. This is the Alabama CPG Legacy brand.
*
* HOST NOTE: the bare host redirects to the canonical www host. robots.txt is
* fully open and references the public sitemap.
*
* WIRE FORMAT: BuilderCloud React SSR. A normal GET returns complete product
* JSON-LD plus the primary home's DetailOverview markup; no browser is needed.
*
* MACHINE-READABLE SOURCE (preferred per the standing "own data over fragile
* DOM" lesson): each per-home page carries a per-home JSON-LD block typed
* ["SingleFamilyResidence","Product"] that is the authoritative carrier of the
* facts most prone to DOM rot —
* - address (streetAddress / addressLocality / addressRegion / postalCode),
* - geo (latitude / longitude),
* - price (offers[].price, a real integer, NOT a "$" DOM string),
* - community (containedIn.name, e.g. "Silver Stream" — NOT a slug-titlecased
* URL segment, which is the trap that bit a sibling adapter),
* - a STABLE per-home id (productId === sku, a 24-hex builder document id).
* We prefer JSON-LD for all of the above and fall back to the DOM only when a
* field is genuinely absent from it.
*
* DOM (only for facts JSON-LD does not carry): the PRIMARY home renders beds,
* full baths, half baths, stories and sqft in DetailOverview_listItem nodes,
* plus the floor plan in DetailOverview_sectionItem. We deliberately ignore
* sibling cards. The visible "Active" value is a sales state, not construction
* evidence, so constructionStatus remains null.
*
* Honest nulls (genuinely absent from the page, NOT fabricated):
* - garageSpaces: Legacy's per-home page publishes no garage field → null.
* - estCompletionDate: no per-home completion date on the page → null.
*
* Facts-only: images OMITTED (buildercloud photos exist but v1 stays images-off
* / mediaRights=NONE per HomesOnSpec policy). Plain HTTP; one detail page ==
* one inventory_home.
*
* Batch control: LEGACY_AL_PAGE_LIMIT caps per-home pages/run (default 40). Optional
* LEGACY_AL_AREA filters the sitemap URL list by the first /homes/{city-al} slug
* (comma-list, e.g. "harvest-al,athens-al").
*/
const BUILDER_SLUG = "legacy-al";
const ORIGIN = "https://www.legacyhomesal.com";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const AREA_FILTER = (process.env.LEGACY_AL_AREA ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.LEGACY_AL_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite integer, or null. Never guesses; 0 / negative / NaN → null.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths may be 2.5), or null. Tolerates Legacy's "2 .5" spacing.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/\s+/g, "").replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, "")
.replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
interface LegacyHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
planName: string | null;
lotNumber: string | null;
price: number | null;
beds: number | null;
bathsTotal: number | null; // decimal, half-baths folded (site publishes a single "2.5")
sqft: number | null;
stories: number | null;
garages: number | null; // no garage field on the page → null (honest)
lat: number | null;
lon: number | null;
phone: string | null;
status: "PLANNED" | "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
builderInventoryId: string | null;
}
/** Pull every parsed JSON-LD object out of the page (flattening arrays + @graph). */
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const top = Array.isArray(data) ? data : [data];
for (const it of top) {
if (it && typeof it === "object") {
const g = (it as Record<string, unknown>)["@graph"];
if (Array.isArray(g)) {
for (const node of g) if (node && typeof node === "object") out.push(node as Record<string, unknown>);
} else {
out.push(it as Record<string, unknown>);
}
}
}
}
return out;
}
function hasType(obj: Record<string, unknown>, token: string): boolean {
const t = obj["@type"];
const types = Array.isArray(t) ? t : [t];
return types.some((x) => x === token);
}
/** The per-home ["SingleFamilyResidence","Product"] JSON-LD object, or null. */
function homeJsonLd(html: string): Record<string, unknown> | null {
for (const o of jsonLdObjects(html)) {
if (hasType(o, "SingleFamilyResidence") || (hasType(o, "Product") && o["address"])) return o;
}
return null;
}
function asObj(v: unknown): Record<string, unknown> | null {
return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
}
function asNum(v: unknown): number | null {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) ? n : null;
}
// Minimal ZIP3-range guard for Alabama. If a page
// ever publishes a ZIP whose ZIP3 can't belong to the parsed state, drop it to
// an honest null rather than propagate a self-contradictory fact. Extensible.
const STATE_ZIP3: Record<string, [number, number][]> = {
AL: [[350, 369]],
};
function zipConsistentWithState(zip: string | null, state: string | null): boolean {
if (!zip || !state) return true;
const ranges = STATE_ZIP3[state.toUpperCase()];
if (!ranges) return true;
const z3 = Number(zip.slice(0, 3));
return ranges.some(([lo, hi]) => z3 >= lo && z3 <= hi);
}
/**
* Parse ONE Legacy per-home page into a home, or null if it isn't a resolvable
* per-home detail page. Address/geo/price/community/id come from the per-home
* JSON-LD (authoritative); beds/baths/stories/sqft/lot/plan come from the
* PRIMARY home's DetailOverview list (never a sibling card).
*/
export function parseHome(html: string, pageUrl: string): LegacyHome | null {
const ld = homeJsonLd(html);
if (!ld) return null; // no per-home Product JSON-LD → not a resolvable detail page
// ---- authoritative address from JSON-LD PostalAddress ----
const addr = asObj(ld["address"]);
const street = clean(addr?.["streetAddress"]);
if (!street) return null; // a Product block with no street isn't a home listing
const city = clean(addr?.["addressLocality"]);
const state = normalizeStateCode(clean(addr?.["addressRegion"]));
const zipRaw = (clean(addr?.["postalCode"]) ?? "").match(/\d{5}/)?.[0] ?? null;
const zip = zipConsistentWithState(zipRaw, state) ? zipRaw : null;
// ---- authoritative geo from JSON-LD GeoCoordinates ----
const geo = asObj(ld["geo"]);
const lat = asNum(geo?.["latitude"]);
const lon = asNum(geo?.["longitude"]);
// ---- authoritative price from JSON-LD offers[].price ----
const offers = ld["offers"];
const firstOffer = Array.isArray(offers) ? asObj(offers[0]) : asObj(offers);
const price = posInt(firstOffer?.["price"]);
// ---- authoritative community from JSON-LD containedIn.name ----
const containedIn = asObj(ld["containedIn"]);
const community = clean(containedIn?.["name"]);
// ---- authoritative stable per-home id from JSON-LD productId / sku ----
const builderInventoryId = clean(ld["productId"]) ?? clean(ld["sku"]);
// ---- PRIMARY home specs: the DetailOverview list (NOT sibling HomeCards) ----
// Each item contains dedicated value + label spans. Full and half baths are
// separate, so combine them only after parsing both explicit values.
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
let stories: number | null = null;
let lotNumber: string | null = null;
let fullBaths: number | null = null;
let halfBaths: number | null = null;
for (const m of html.matchAll(/<li\s+class="DetailOverview_listItem"[^>]*>([\s\S]*?)<\/li>/gi)) {
const item = m[1] ?? "";
const valueMatch = item.match(/class="DetailOverview_listItemValue"[^>]*>([\s\S]*?)<\/span>/i);
const value = clean(valueMatch?.[1] ?? null);
const label = (clean(item.match(/class="DetailOverview_listItemLabel"[^>]*>([\s\S]*?)<\/span>/i)?.[1] ?? null) ?? "").toLowerCase();
if (/^beds?\b/.test(label)) beds = posInt(value);
else if (/^full baths?\b/.test(label)) fullBaths = posDec(value);
else if (/^half baths?\b/.test(label)) halfBaths = posDec(value);
else if (/^stor(?:y|ies)\b/.test(label)) stories = posInt(value);
else if (/^sq\.?\s*ft\.?\b/.test(label)) sqft = posInt(value);
else if (/^lot\s*#/.test(label)) lotNumber = value;
}
if (fullBaths != null || halfBaths != null) bathsTotal = (fullBaths ?? 0) + (halfBaths ?? 0) * 0.5;
// ---- floor-plan name: the DetailOverview "Floor Plan" anchor ----
const planItem = html.match(/Floor Plan:\s*(?:<!--[\s\S]*?-->)?\s*<a[^>]*>([^<]+)<\/a>/i);
const planName = clean(planItem?.[1] ?? null);
// ---- status: the DetailHeader lead span under the h1 ----
// "Active" in the detail table is a sales listing state, not a construction
// state. No supported construction evidence is published on these pages.
const status = null;
// ---- sales phone: first tel: link on the page (consultant / office) ----
const phone = clean(html.match(/href="tel:(\+?[\d]+)"/i)?.[1] ?? null);
// garageSpaces: Legacy's per-home page publishes no garage field → honest null.
const garages = null;
return {
url: pageUrl,
street,
city,
state,
zip,
community,
planName,
lotNumber,
price,
beds,
bathsTotal,
sqft,
stories,
garages,
lat,
lon,
phone,
status,
builderInventoryId,
};
}
/**
* A per-home inventory DETAIL URL: /homes/{city-al}/{community}/{address}
* — exactly THREE path segments after /homes/. Rejects city/community listing
* pages and any /undefined/ partial.
*/
function isDetailUrl(u: string): boolean {
const tail = u.split("/homes/")[1];
if (!tail) return false;
const segs = tail.replace(/\/$/, "").split(/[?#]/)[0]!.split("/");
if (segs.length !== 3) return false;
return segs.every((s) => !!s && s !== "undefined");
}
export const legacyAlAdapter: SourceAdapter = {
key: "legacy-al-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter(isDetailUrl);
if (AREA_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const area = u.split("/homes/")[1]?.split("/")[0]?.toLowerCase();
return area ? AREA_FILTER.includes(area) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
// The REAL street address (never a "Lot X" fallback) — from the
// per-home JSON-LD PostalAddress.streetAddress. `?? h.url` never fires
// because parseHome already returned null when street was absent, but
// it keeps the hint type-safe. Parenthesized per the standing rule.
address: (h.street ?? h.url),
builderInventoryId: h.builderInventoryId ?? h.url,
planName: h.planName ?? undefined,
lotNumber: h.lotNumber ?? undefined,
lat: h.lat ?? undefined,
lon: h.lon ?? undefined,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price}` : null,
page.url,
h.price != null ? `Offer price $${h.price}` : null,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url), // honest null
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Legacy Homes Alabama single-family inventory home"),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
null,
page.url,
null,
),
estCompletionDate: fv<string>(null, null, page.url), // no per-home completion date on the page
lotNumber: fv(h.lotNumber, h.lotNumber, page.url),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
planName: fv(h.planName, h.planName, page.url),
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};