← back to Homesonspec
collectors/arbor/src/index.ts
443 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Arbor Homes adapter — the Greater-Indianapolis (IN + KY + OH) site-built spec
* builder that Clayton Properties Group acquired in 2018 (slug "arbor"; recon +
* built 2026-08-30, TK-10487). Same architecture class as the goodall-homes and
* harris-doyle adapters: Clayton has no unified site-built feed, so each brand
* gets its own adapter on its own domain.
*
* SITE IDENTITY (confirmed, not the unrelated Oregon arborhomes.com):
* https://www.yourarborhome.com — the Organization JSON-LD reads "Arbor Homes …
* built more than 22,000 new homes in Indiana, Kentucky and Ohio … greater
* Indianapolis, IN, Louisville, KY and Columbus, OH regions", HQ 9225 Harrison
* Park Court, Indianapolis IN 46250. Metros in the sitemap: indianapolis-metro,
* louisvillesouthern-indiana, cincinnatidayton, columbus-oh. This is the CPG
* Arbor.
*
* HOST NOTE: the CANONICAL host is the BARE host — https://www.yourarborhome.com
* 301-redirects to https://yourarborhome.com (the reverse of the harris-doyle
* convention). robots.txt (fetched via the bare host) is FULLY OPEN
* ("User-agent: * Allow: /") and references the sitemap. The sitemap itself
* lists the `www.` form of every URL, and those `www.` URLs resolve fine
* (301→bare, transparently followed by the shared LiveFetcher). We keep the
* sitemap's `www.` URL as the record sourceUrl for traceability.
*
* WIRE FORMAT: yourarborhome.com is a React SSR site (react-helmet /
* `data-reactid` markup) served gzip'd (~32 KB decompressed per home; the shared
* LiveFetcher sends Accept-Encoding + auto-decompresses). NO browser, NO
* bot-wall, NO XHR/WS for the facts. A plain honest-UA GET returns the complete
* page.
*
* MACHINE-READABLE SOURCE (preferred per the standing "own data over fragile
* DOM" lesson): each per-home page carries a per-home JSON-LD block typed
* ["SingleFamilyResidence","Product"] that is the authoritative carrier of the
* facts most prone to DOM rot —
* - address (streetAddress / addressLocality / addressRegion / postalCode),
* - geo (latitude / longitude),
* - price (offers[].price, a real integer, NOT a "$" DOM string),
* - community (containedIn.name, e.g. "Silver Stream" — NOT a slug-titlecased
* URL segment, which is the trap that bit a sibling adapter),
* - a STABLE per-home id (productId === sku, a 24-hex builder document id).
* We prefer JSON-LD for all of the above and fall back to the DOM only when a
* field is genuinely absent from it.
*
* DOM (only for facts the JSON-LD does NOT carry): the PRIMARY home renders its
* beds / baths / stories / sqft / lot# / floor-plan in a
* <ul>…<li class="HomeOverview_iconListItem"><b>VALUE</b>Label</li>…</ul>. The
* page ALSO renders sibling "similar / available" homes as
* <div class="HomeCard_spec"><span class="HomeCard_specNumber">N</span>… — a
* DIFFERENT class we deliberately never read, so we can't mistake another lot's
* beds for this home's. Status text is the <span class="DetailHeader_h2Lead">
* lead under the h1 ("Quick Move-In Home Available Now!" → MOVE_IN_READY).
*
* Honest nulls (genuinely absent from the page, NOT fabricated):
* - garageSpaces: Arbor's per-home page publishes no garage field → null.
* - estCompletionDate: no per-home completion date on the page → null.
*
* Facts-only: images OMITTED (buildercloud photos exist but v1 stays images-off
* / mediaRights=NONE per HomesOnSpec policy). Plain HTTP; one detail page ==
* one inventory_home.
*
* Batch control: ARBOR_PAGE_LIMIT caps per-home pages/run (default 40). Optional
* ARBOR_METRO filters the sitemap URL list by the first /homes/{metro} slug
* (comma-list, e.g. "indianapolis-metro,columbus-oh").
*/
const BUILDER_SLUG = "arbor";
const ORIGIN = "https://www.yourarborhome.com";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const METRO_FILTER = (process.env.ARBOR_METRO ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.ARBOR_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite integer, or null. Never guesses; 0 / negative / NaN → null.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths may be 2.5), or null. Tolerates Arbor's "2 .5" spacing.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/\s+/g, "").replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, "")
.replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
interface ArborHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
planName: string | null;
lotNumber: string | null;
price: number | null;
beds: number | null;
bathsTotal: number | null; // decimal, half-baths folded (site publishes a single "2.5")
sqft: number | null;
stories: number | null;
garages: number | null; // no garage field on the page → null (honest)
lat: number | null;
lon: number | null;
phone: string | null;
status: "PLANNED" | "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
builderInventoryId: string | null;
}
/** Pull every parsed JSON-LD object out of the page (flattening arrays + @graph). */
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const top = Array.isArray(data) ? data : [data];
for (const it of top) {
if (it && typeof it === "object") {
const g = (it as Record<string, unknown>)["@graph"];
if (Array.isArray(g)) {
for (const node of g) if (node && typeof node === "object") out.push(node as Record<string, unknown>);
} else {
out.push(it as Record<string, unknown>);
}
}
}
}
return out;
}
function hasType(obj: Record<string, unknown>, token: string): boolean {
const t = obj["@type"];
const types = Array.isArray(t) ? t : [t];
return types.some((x) => x === token);
}
/** The per-home ["SingleFamilyResidence","Product"] JSON-LD object, or null. */
function homeJsonLd(html: string): Record<string, unknown> | null {
for (const o of jsonLdObjects(html)) {
if (hasType(o, "SingleFamilyResidence") || (hasType(o, "Product") && o["address"])) return o;
}
return null;
}
function asObj(v: unknown): Record<string, unknown> | null {
return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
}
function asNum(v: unknown): number | null {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) ? n : null;
}
/**
* Map Arbor's status lead text to the schema constructionStatus enum. Exported
* so the selftest can exercise the PLANNED / UNDER_CONSTRUCTION branches, which
* no captured fixture triggers (all current live homes are QMI or Model).
*/
export function statusFromLead(lead: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
if (!lead) return null;
const p = lead.toLowerCase();
if (p.includes("move-in") || p.includes("move in") || p.includes("quick move") || p.includes("available now"))
return "MOVE_IN_READY";
if (p.includes("under construction") || p.includes("being built")) return "UNDER_CONSTRUCTION";
if (p.includes("coming soon") || p.includes("to be built") || p.includes("pre-sale") || p.includes("presale"))
return "PLANNED";
return null;
}
// Minimal ZIP3-range guard for Arbor's operating states (IN, KY, OH). If a page
// ever publishes a ZIP whose ZIP3 can't belong to the parsed state, drop it to
// an honest null rather than propagate a self-contradictory fact. Extensible.
const STATE_ZIP3: Record<string, [number, number][]> = {
IN: [[460, 479]],
KY: [[400, 427]],
OH: [[430, 459]],
};
function zipConsistentWithState(zip: string | null, state: string | null): boolean {
if (!zip || !state) return true;
const ranges = STATE_ZIP3[state.toUpperCase()];
if (!ranges) return true;
const z3 = Number(zip.slice(0, 3));
return ranges.some(([lo, hi]) => z3 >= lo && z3 <= hi);
}
/**
* Parse ONE Arbor per-home page into a home, or null if it isn't a resolvable
* per-home detail page. Address/geo/price/community/id come from the per-home
* JSON-LD (authoritative); beds/baths/stories/sqft/lot/plan/status come from the
* PRIMARY home's HomeOverview list (never a sibling HomeCard).
*/
export function parseHome(html: string, pageUrl: string): ArborHome | null {
const ld = homeJsonLd(html);
if (!ld) return null; // no per-home Product JSON-LD → not a resolvable detail page
// ---- authoritative address from JSON-LD PostalAddress ----
const addr = asObj(ld["address"]);
const street = clean(addr?.["streetAddress"]);
if (!street) return null; // a Product block with no street isn't a home listing
const city = clean(addr?.["addressLocality"]);
const state = normalizeStateCode(clean(addr?.["addressRegion"]));
const zipRaw = (clean(addr?.["postalCode"]) ?? "").match(/\d{5}/)?.[0] ?? null;
const zip = zipConsistentWithState(zipRaw, state) ? zipRaw : null;
// ---- authoritative geo from JSON-LD GeoCoordinates ----
const geo = asObj(ld["geo"]);
const lat = asNum(geo?.["latitude"]);
const lon = asNum(geo?.["longitude"]);
// ---- authoritative price from JSON-LD offers[].price ----
const offers = ld["offers"];
const firstOffer = Array.isArray(offers) ? asObj(offers[0]) : asObj(offers);
const price = posInt(firstOffer?.["price"]);
// ---- authoritative community from JSON-LD containedIn.name ----
const containedIn = asObj(ld["containedIn"]);
const community = clean(containedIn?.["name"]);
// ---- authoritative stable per-home id from JSON-LD productId / sku ----
const builderInventoryId = clean(ld["productId"]) ?? clean(ld["sku"]);
// ---- PRIMARY home specs: the HomeOverview icon list (NOT sibling HomeCards) ----
// Each item: <li class="HomeOverview_iconListItem" ...><b ...>VALUE</b>Label</li>
// Baths render as "<b>2 .5</b>Baths" (stray space) — posDec strips it.
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
let stories: number | null = null;
let lotNumber: string | null = null;
for (const m of html.matchAll(/<li\s+class="HomeOverview_iconListItem"[^>]*>([\s\S]*?)<\/li>/gi)) {
const item = m[1] ?? "";
const valueMatch = item.match(/<b[^>]*>([\s\S]*?)<\/b>/i);
const value = clean(valueMatch?.[1] ?? null);
// Label = the item text after removing the <b>value</b> and any inner tags/comments.
const label = (clean(item.replace(/<b[^>]*>[\s\S]*?<\/b>/i, "").replace(/<!--[\s\S]*?-->/g, "")) ?? "").toLowerCase();
if (/^beds?\b/.test(label)) beds = posInt(value);
else if (/^baths?\b/.test(label)) bathsTotal = posDec(value);
else if (/^stor(?:y|ies)\b/.test(label)) stories = posInt(value);
else if (/^sq\.?\s*ft\.?\b/.test(label)) sqft = posInt(value);
else if (/^lot\s*#/.test(label)) lotNumber = value;
}
// ---- floor-plan name: the HomeOverview "Floor Plan" item's <a><b>The Norway</b> ----
const planItem = html.match(/Floor Plan[\s\S]{0,160}?<a[^>]*>(?:<b[^>]*>)?([^<]+)</i);
const planName = clean(planItem?.[1] ?? null);
// ---- status: the DetailHeader lead span under the h1 ----
const lead = clean(html.match(/class="DetailHeader_h2Lead"[^>]*>([\s\S]*?)<\/span>/i)?.[1] ?? null);
const status = statusFromLead(lead);
// ---- sales phone: first tel: link on the page (consultant / office) ----
const phone = clean(html.match(/href="tel:(\+?[\d]+)"/i)?.[1] ?? null);
// garageSpaces: Arbor's per-home page publishes no garage field → honest null.
const garages = null;
return {
url: pageUrl,
street,
city,
state,
zip,
community,
planName,
lotNumber,
price,
beds,
bathsTotal,
sqft,
stories,
garages,
lat,
lon,
phone,
status,
builderInventoryId,
};
}
/**
* A per-home inventory DETAIL URL: /homes/{metro}/{city}/{community}/{address}
* — exactly FOUR path segments after /homes/. Rejects the 1-seg metro pages,
* 2-seg city pages, and any /undefined/ partial. (3-seg community listing pages
* do not occur in Arbor's sitemap; the guard length===4 excludes them anyway.)
*/
function isDetailUrl(u: string): boolean {
const tail = u.split("/homes/")[1];
if (!tail) return false;
const segs = tail.replace(/\/$/, "").split(/[?#]/)[0]!.split("/");
if (segs.length !== 4) return false;
return segs.every((s) => !!s && s !== "undefined");
}
export const arborAdapter: SourceAdapter = {
key: "arbor-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter(isDetailUrl);
if (METRO_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const metro = u.split("/homes/")[1]?.split("/")[0]?.toLowerCase();
return metro ? METRO_FILTER.includes(metro) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
// The REAL street address (never a "Lot X" fallback) — from the
// per-home JSON-LD PostalAddress.streetAddress. `?? h.url` never fires
// because parseHome already returned null when street was absent, but
// it keeps the hint type-safe. Parenthesized per the standing rule.
address: (h.street ?? h.url),
builderInventoryId: h.builderInventoryId ?? h.url,
planName: h.planName ?? undefined,
lotNumber: h.lotNumber ?? undefined,
lat: h.lat ?? undefined,
lon: h.lon ?? undefined,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price}` : null,
page.url,
h.price != null ? `Offer price $${h.price}` : null,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url), // honest null
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Arbor Homes single-family inventory home"),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
leadEvidence(html),
page.url,
leadEvidence(html),
),
estCompletionDate: fv<string>(null, null, page.url), // no per-home completion date on the page
lotNumber: fv(h.lotNumber, h.lotNumber, page.url),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
planName: fv(h.planName, h.planName, page.url),
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};
function leadEvidence(html: string): string | null {
return clean(html.match(/class="DetailHeader_h2Lead"[^>]*>([\s\S]*?)<\/span>/i)?.[1] ?? null);
}