← back to Homesonspec
collectors/goodall-homes/src/index.ts
368 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Goodall Homes adapter — TN/AL site-built spec-home builder, a Clayton
* Properties Group brand (slug "goodall-homes"; recon + built 2026-08-12,
* TK-10487). This is the first of the per-CPG-brand adapters the
* `clayton-properties` RECON pointed at: Clayton has no unified site-built
* feed, so each brand (Goodall, Brohn, …) gets its own adapter on its own
* domain.
*
* goodallhomes.com is a React SSR site (react-helmet + `data-reactid` markers)
* that is FULLY server-rendered — a plain honest-UA GET of a per-home page
* returns ~220–325 KB of complete HTML (gzip on the wire; `fetch()` in the
* shared LiveFetcher auto-decompresses). NO browser, NO bot-wall, NO XHR/WS
* for the facts. robots.txt is fully open ("User-agent: * Allow: /", no
* Disallow rules) and references the sitemap.
*
* Sitemap https://www.goodallhomes.com/sitemap.xml (single urlset, ~662 URLs)
* -> per-home inventory DETAIL pages are the 5-segment
* /homes/{area}/{city}/{community}/{street-address-slug}
* (~114 of them, e.g.
* /homes/knoxville-area/farragut-tn/ivey-farms/510-ivey-farms-road).
* Shorter /homes/{area} + /homes/{area}/{city} URLs are aggregate LISTING
* pages (many sibling HomeCards) and are filtered out; a handful of
* /homes/undefined/... partials (a builder-CMS quirk) are also rejected.
* /communities/... and /plan/... are community + floor-plan pages (skip).
*
* Each per-home page carries TWO complementary structured sources:
*
* (1) A per-home JSON-LD block, `@type: ["SingleFamilyResidence","Product"]`:
* name (= "street City, ST, ZIP"), address{streetAddress,addressLocality,
* addressRegion,postalCode}, geo{latitude,longitude} (PER-HOME lat/lon),
* offers[{price,priceCurrency}], brand.name (= "Goodall Homes"),
* containedIn.name (= the community/subdivision), sku/productId (a stable
* 24-hex builder id -> builderInventoryId). A first Organization/
* LocalBusiness block carries the builder-wide telephone (sales phone).
*
* (2) A `HomeDetails_iconsWrapper` <div> — exactly ONE rendered instance per
* page (the current home; sibling "other homes" use HomeCard_* classes,
* so the first match is unambiguous) — with icon spans carrying the
* facts NOT in the JSON-LD: iconBeds / iconBaths / iconSqft. A sibling
* `HomeDetails_stories` block carries "Stories N" + "Garages N-Car", and
* a `HomeDetails_statusWrapper` carries the status label
* ("Ready to Move-In" -> MOVE_IN_READY; any other -> UNDER_CONSTRUCTION).
* React splits the numeric text with <!--react-text--> comments
* (e.g. `<b>4<!----> <!----></b>Beds`, baths `<b>3<!---->.5</b>`), so the
* wrapper inner is comment-stripped before the value regex runs.
*
* Facts-only: images OMITTED (the Product block DOES list s3 photos, but v1
* stays images-off / mediaRights=NONE per HomesOnSpec policy). Per-home geo IS
* captured (homes map). Plain HTTP; one detail page == one inventory_home.
*
* Verified 14/14 fixtures across TN + KY (5 communities): 100% coverage on
* address / price / beds / baths / sqft / stories / garage / geo / community /
* plan / status / builderInventoryId, all addresses + ids distinct (genuinely
* per-home, not aggregate).
*
* Batch control: GOODALL_PAGE_LIMIT caps per-home pages/run (default 40).
* Optional GOODALL_AREA filters the sitemap URL list by the first /homes/{area}
* slug (comma-list, e.g. "knoxville-area,nashville-area").
*/
const BUILDER_SLUG = "goodall-homes";
const ORIGIN = "https://www.goodallhomes.com";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const AREA_FILTER = (process.env.GOODALL_AREA ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.GOODALL_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN -> null.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths may be 2.5), or null.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A signed decimal (geo lat/lon may be negative — must NOT strip the minus), or null.
const coord = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.\-]/g, "")) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, "")
.replace(/'|'|'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
interface GoodallHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
planName: string | null;
price: number | null;
beds: number | null;
bathsTotal: number | null; // decimal, half-baths folded (site publishes a single "3.5")
sqft: number | null;
stories: number | null;
garages: number | null;
lat: number | null;
lon: number | null;
phone: string | null;
status: "MOVE_IN_READY" | "UNDER_CONSTRUCTION";
builderInventoryId: string | null;
}
/** Pull every parsed JSON-LD object out of the page (flattening arrays). */
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
for (const it of Array.isArray(data) ? data : [data]) {
if (it && typeof it === "object") out.push(it as Record<string, unknown>);
}
}
return out;
}
/** Does this JSON-LD object's @type include the target token? */
function hasType(obj: Record<string, unknown>, token: string): boolean {
const t = obj["@type"];
const types = Array.isArray(t) ? t : [t];
return types.some((x) => x === token);
}
/** Parse ONE Goodall per-home page into a home, or null if not a home page. */
export function parseHome(html: string, pageUrl: string): GoodallHome | null {
const objs = jsonLdObjects(html);
const prod = objs.find((o) => hasType(o, "Product") || hasType(o, "SingleFamilyResidence")) ?? null;
if (!prod) return null;
const addr = ((prod.address ?? {}) as Record<string, unknown>) || {};
const street = clean(addr.streetAddress);
if (!street) return null; // not a resolvable per-home listing
const geo = ((prod.geo ?? {}) as Record<string, unknown>) || {};
const offers0 = Array.isArray(prod.offers) ? (prod.offers[0] as Record<string, unknown>) : ((prod.offers ?? {}) as Record<string, unknown>);
const brand = ((prod.brand ?? {}) as Record<string, unknown>) || {};
const containedIn = ((prod.containedIn ?? {}) as Record<string, unknown>) || {};
const community = clean(containedIn.name) ?? clean(brand.name);
// sales phone: the builder-wide telephone on the Organization/LocalBusiness block.
const org = objs.find((o) => hasType(o, "Organization") || hasType(o, "LocalBusiness"));
const phone = clean(org?.telephone);
// ---- spec strip (the ONE rendered HomeDetails_iconsWrapper == current home;
// sibling homes render as HomeCard_* so the first match is unambiguous) ----
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
const wm = html.match(/HomeDetails_iconsWrapper"[^>]*>((?:<span class="HomeDetails_iconsItem[\s\S]*?<\/span>)+)/i);
if (wm) {
const inner = wm[1]!.replace(/<!--[^>]*-->/g, ""); // strip react-text comments splitting the value
beds = posInt(inner.match(/iconBeds"[^>]*>\s*<b[^>]*>\s*([\d.,]+)/i)?.[1] ?? null);
bathsTotal = posDec(inner.match(/iconBaths"[^>]*>\s*<b[^>]*>\s*([\d.,]+)/i)?.[1] ?? null);
sqft = posInt(inner.match(/iconSqft"[^>]*>\s*<b[^>]*>\s*([\d.,]+)/i)?.[1] ?? null);
}
// ---- stories + garages (first rendered HomeDetails_stories block) ----
let stories: number | null = null;
let garages: number | null = null;
const stm = html.match(/HomeDetails_stories"[^>]*>([\s\S]{0,500}?)<\/div>/i);
if (stm) {
const s = stm[1]!.replace(/<!--[^>]*-->/g, "");
stories = posInt(s.match(/<b[^>]*>\s*Stories\s*<\/b>\s*([0-9]+)/i)?.[1] ?? null);
garages = posInt(s.match(/<b[^>]*>\s*Garages?\s*<\/b>\s*([0-9]+)/i)?.[1] ?? null);
}
// ---- status (first rendered HomeDetails_statusWrapper) ----
let statusRaw: string | null = null;
const swm = html.match(/HomeDetails_statusWrapper"[^>]*>([\s\S]{0,300}?)<\/div>/i);
if (swm) statusRaw = clean(swm[1]!.replace(/<!--[^>]*-->/g, "").replace(/<b[^>]*>\s*Status\s*<\/b>/i, ""));
// "Ready to Move-In" -> MOVE_IN_READY; anything else -> UNDER_CONSTRUCTION.
const status: GoodallHome["status"] = statusRaw && /ready\s*to\s*move[-\s]?in|move[-\s]?in\s*ready/i.test(statusRaw)
? "MOVE_IN_READY"
: statusRaw
? "UNDER_CONSTRUCTION"
: "MOVE_IN_READY"; // no status label -> standing move-in-ready QMI listing
// ---- plan name (the "Floor Plan:" community link) ----
const noComment = html.replace(/<!--[^>]*-->/g, "");
const planName =
clean(noComment.match(/Floor Plan:<\/b>\s*<a[^>]*>([^<]+)/i)?.[1]) ??
clean(html.match(/href="\/plan\/[^"]*"[^>]*>([^<]+)</i)?.[1]);
return {
url: clean(prod.url) ?? pageUrl,
street,
city: clean(addr.addressLocality),
state: normalizeStateCode(clean(addr.addressRegion)),
zip: (clean(addr.postalCode) ?? "").match(/\d{5}/)?.[0] ?? null,
community,
planName,
price: posInt(offers0.price),
beds,
bathsTotal,
sqft,
stories,
garages,
lat: coord(geo.latitude),
lon: coord(geo.longitude),
phone,
status,
builderInventoryId: clean(prod.sku) ?? clean(prod.productId),
};
}
/**
* A per-home inventory DETAIL URL: /homes/{area}/{city}/{community}/{addr}
* — exactly four path segments after /homes/, with a REAL final address segment.
* Rejects aggregate listing pages (/homes/{area}, /homes/{area}/{city}) which
* render many sibling HomeCards, and any /homes/undefined/... partial.
*/
function isDetailUrl(u: string): boolean {
const m = u.match(/^https?:\/\/[^/]+\/homes\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/?$/);
if (!m) return false;
const [, area, city, community, addr] = m;
return (
!!addr && addr !== "undefined" &&
area !== "undefined" && city !== "undefined" && community !== "undefined"
);
}
export const goodallHomesAdapter: SourceAdapter = {
key: "goodall-homes-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter(isDetailUrl);
if (AREA_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const area = u.match(/\/homes\/([^/]+)\//)?.[1]?.toLowerCase();
return area ? AREA_FILTER.includes(area) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
// Carries the per-home geo so the community also maps, and the builder-wide
// sales phone.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
address: h.street,
builderInventoryId: h.builderInventoryId ?? h.url,
lat: h.lat ?? undefined,
lon: h.lon ?? undefined,
planName: h.planName ?? undefined,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price}` : null,
page.url,
h.price != null ? `Product offer price $${h.price}` : null,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url),
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Goodall Homes single-family inventory home"),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
null,
page.url,
h.status === "MOVE_IN_READY" ? "Status: Ready to Move-In" : "Status: not move-in-ready",
),
estCompletionDate: fv<string>(null, null, page.url), // no per-home completion date on the page
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
planName: fv(h.planName, h.planName, page.url),
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};