← back to Homesonspec
collectors/brohn-homes/src/index.ts
375 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Brohn Homes adapter — Austin/Houston/San-Antonio TX site-built spec-home
* builder, a Clayton Properties Group brand (slug "brohn-homes"; recon + built
* 2026-08-12, TK-10487).
*
* brohnhomes.com is a WordPress site (Total/wpex theme) that is FULLY
* server-rendered — a plain honest-UA GET of a per-home page returns ~230 KB of
* complete HTML with every fact inline. NO browser, NO bot-wall, NO XHR — this
* is a plain-fetch adapter. (Distinct from its Clayton sibling claytonhomes.com,
* whose only feed is robots-DISALLOWED /api/ + manufactured/off-target homes;
* Brohn is a genuine site-built spec-inventory site with an open HTML feed.)
*
* robots.txt (brohnhomes.com) = `User-agent: *` `Disallow: /wp-admin/`
* `Allow: /wp-admin/admin-ajax.php`, and declares the sitemap index. The
* `/homes/...` per-home pages we read are ALLOWED.
*
* Sitemap index https://brohnhomes.com/sitemap_index.xml
* -> homes-sitemap1.xml + homes-sitemap2.xml = the PER-HOME inventory
* (~247 URLs) of the form
* /homes/{state}/{metro}/{city}/{community-slug}/{street-address-slug}/
* (a leading template row with a literal `%metro%` placeholder is
* filtered out). The separate floorplans-sitemapN.xml are plan-level
* aggregates (SKIP — not per-home). One detail page == one home.
*
* Each per-home page carries the facts in the delivered DOM (facts-only):
* <h1 class="vcex-page-title__heading"> = street address ("31315 Cass River Lane")
* <div class="brohn-home-details__community"> "Community Name <span
* class="brohn-home-details__city-state"> | City, ST</span>"
* <div class="home-price"> "$283,990" (absent / no $ -> price null)
* <div class="brohn-home-stats"> N x <div class="stat"><span
* class="stat-number">V</span><span class="stat-label">L</span></div>
* with labels beds / baths / cars / story / sq. ft. — `baths` is a single
* decimal ("2.5") with half-baths already folded (schema bathsTotal is one
* number), `cars` -> garageSpaces, `story` -> stories.
* <a class="brohn-get-directions" href="...maps...query={full address}"> — the
* authoritative "Street, City, ST ZIP" used to recover city/state/zip
* (the ZIP is NOT in the visible city-state span, only in the maps query).
* <div class="home-item-status-template"> label — "NOW" -> MOVE_IN_READY,
* "SOLD" -> MOVE_IN_READY (a real, now-closed spec home; kept, price intact),
* any other label left as UNDER_CONSTRUCTION (no future ready-date field is
* published on the page, so estCompletionDate stays null).
* <body class="... postid-NNNNN single-homes ..."> — the WordPress post id is
* the stable per-home key -> builderInventoryId.
* Sales phone "(512) 334-6775" appears in the header CTA (builder-wide number).
*
* homeType: a final URL segment of `unit-NN` (e.g. .../cross-creek/unit-34/) is a
* townhome/condo unit -> TOWNHOME; everything else -> SINGLE_FAMILY.
*
* NO per-home geo lat/lon is published (the map is an embedded widget with no
* coordinates in the HTML) -> left null, exactly like richmond-american /
* smith-douglas. NO plan NAME is exposed as a labelled field for the primary
* home (the "Floor Plan" control is a CTA button, not a name) -> planName null.
* Plain HTTP; facts-only; images OMITTED (mediaRights=NONE).
*
* Metro scope via BROHN_METRO (comma-list of metro slugs, e.g. "austin,houston";
* default = all). Per-run home cap via BROHN_PAGE_LIMIT (default 40).
*/
const BUILDER_SLUG = "brohn-homes";
const ORIGIN = "https://brohnhomes.com";
const HOME_SITEMAPS = [`${ORIGIN}/homes-sitemap1.xml`, `${ORIGIN}/homes-sitemap2.xml`];
const METRO_FILTER = (process.env.BROHN_METRO ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.BROHN_PAGE_LIMIT ?? "40");
/** A per-home DETAIL URL: /homes/{state}/{metro}/{city}/{community}/{street}/ */
const HOME_URL_RE =
/^https?:\/\/[^/]+\/homes\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/?$/i;
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive integer (beds / sqft), or null. 0 / negative / NaN -> null.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A non-negative integer (garages / stories may legitimately be small), or null.
const nonNegInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths — half already folded, e.g. 2.5), or null.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, " ")
.replace(/�?38;|&/g, "&")
.replace(/'|'|'/g, "'")
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
/** Percent-decode the get-directions maps `query=` value (facts-only address). */
function decodeQuery(s: string): string {
try {
return decodeURIComponent(s.replace(/\+/g, " "));
} catch {
return s;
}
}
type Status = "MOVE_IN_READY" | "UNDER_CONSTRUCTION";
type HomeType = "SINGLE_FAMILY" | "TOWNHOME";
interface ParsedHome {
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
price: number | null;
priceRaw: string | null;
beds: number | null;
bathsTotal: number | null;
garages: number | null;
stories: number | null;
sqft: number | null;
status: Status;
statusRaw: string | null;
homeType: HomeType;
phone: string | null;
builderInventoryId: string | null;
}
/**
* Parse ONE Brohn Homes per-home detail page into a home, or null if the page
* has no home-details block (a non-detail page slipped through). Exported for
* the self-test / fixtures.
*/
export function parseHome(html: string, pageUrl: string): ParsedHome | null {
// Only accept genuine single-homes post pages.
const bodyClass = html.match(/<body[^>]*class="([^"]*)"/i)?.[1] ?? "";
if (!/single-homes/.test(bodyClass) && !/brohn-home-stats/.test(html)) return null;
// Street address: the vcex page-title heading text node.
const street =
clean(html.match(/vcex-page-title__heading[^>]*>([\s\S]*?)<\/h1>/i)?.[1]) ??
clean(html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1]);
if (!street) return null;
// Full "Street, City, ST ZIP" from the get-directions maps query — the only
// place the ZIP is emitted. The maps `query=` sits in the anchor href, which
// comes BEFORE the class="brohn-get-directions" in the DOM, so match the maps
// search URL directly and require the anchor to carry that class. Fall back to
// the visible city-state span for city/state when no directions link exists.
const dirMatch = html.match(
/<a\b[^>]*\/maps\/search\/\?[^"']*?query=([^"'&]+)[^>]*brohn-get-directions/i,
);
const q = dirMatch?.[1] ?? html.match(/\/maps\/search\/\?[^"']*?query=([^"'&]+)/i)?.[1];
const full = q ? decodeQuery(q) : null;
let city: string | null = null;
let state: string | null = null;
let zip: string | null = null;
if (full) {
const m = full.match(/^(.*?),\s*(.+?),\s*([A-Za-z]{2})\s*(\d{5})/);
if (m) {
city = clean(m[2]);
state = normalizeStateCode(m[3] ?? null);
zip = m[4] ?? null;
}
}
if (!city || !state) {
// "Community Name <span class="brohn-home-details__city-state"> | City, ST</span>"
const cs = html.match(/brohn-home-details__city-state[^>]*>([\s\S]*?)<\/span>/i)?.[1];
const csText = clean(cs)?.replace(/^\|\s*/, "") ?? null;
const m = csText?.match(/^(.*?),\s*([A-Za-z]{2})$/);
if (m) {
city = city ?? clean(m[1]);
state = state ?? normalizeStateCode(m[2] ?? null);
}
}
// Community: the text node preceding the city-state span inside the header.
const community = clean(
html.match(/brohn-home-details__community[^>]*>\s*([^<]+?)\s*<span/i)?.[1],
);
// Price: .home-price -> "$NNN,NNN". Absent / non-$ -> null (correct).
const priceRaw = clean(html.match(/home-price[^>]*>([\s\S]*?)<\/div>/i)?.[1]);
const priceMatch = priceRaw?.match(/\$\s*([0-9][0-9,]{2,})/);
const price = priceMatch ? posInt(priceMatch[1]) : null;
// Stats block — scope to brohn-home-stats so sibling cards never leak in.
const statsScope = html.match(/brohn-home-stats[^>]*>([\s\S]*?)<\/div>\s*<\/div>\s*<\/div>/i)?.[1] ?? "";
const stats: Record<string, string> = {};
for (const m of statsScope.matchAll(
/stat-number[^>]*>\s*([0-9,.]+)\s*<\/span>\s*<span[^>]*stat-label[^>]*>\s*([^<]+?)\s*<\/span>/gi,
)) {
stats[m[2]!.toLowerCase().trim()] = m[1]!.replace(/,/g, "");
}
const beds = posInt(stats["beds"] ?? null);
const bathsTotal = posDec(stats["baths"] ?? null);
const garages = nonNegInt(stats["cars"] ?? null);
const stories = nonNegInt(stats["story"] ?? stats["stories"] ?? null);
const sqft = posInt(stats["sq. ft."] ?? stats["sqft"] ?? stats["sq ft"] ?? null);
// Status label.
const statusRaw = clean(html.match(/home-item-status-template[^>]*>([^<]+)</i)?.[1]);
// "NOW"/"SOLD" -> a real standing/closed home (MOVE_IN_READY); anything else
// (a future label) -> UNDER_CONSTRUCTION. No ready-date field is on the page.
const status: Status = statusRaw && /^(now|sold|move[- ]?in)/i.test(statusRaw)
? "MOVE_IN_READY"
: "UNDER_CONSTRUCTION";
// homeType: a `unit-NN` final URL segment is a townhome/condo unit.
const homeType: HomeType = /\/unit-\d+\/?$/i.test(pageUrl) ? "TOWNHOME" : "SINGLE_FAMILY";
// Sales phone (builder-wide CTA number).
const ph = html.match(/\((\d{3})\)\s*(\d{3})-(\d{4})/);
const phone = ph ? `(${ph[1]}) ${ph[2]}-${ph[3]}` : null;
// Stable per-home id: the WordPress body-class postid. Anchor to the <body>
// class attribute — a `.postid-NNN` also appears inside CSS <style> rules, so
// a naive global match would grab the wrong id.
const postId = bodyClass.match(/\bpostid-(\d+)\b/)?.[1] ?? null;
const builderInventoryId = postId ? `postid-${postId}` : null;
return {
street,
city,
state,
zip,
community,
price,
priceRaw: priceMatch ? priceMatch[0] : priceRaw,
beds,
bathsTotal,
garages,
stories,
sqft,
status,
statusRaw,
homeType,
phone,
builderInventoryId,
};
}
export const brohnHomesAdapter: SourceAdapter = {
key: "brohn-homes-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let homeUrls: string[] = [];
for (const sm of HOME_SITEMAPS) {
try {
const index = await fetcher.fetch(sm);
homeUrls.push(
...[...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter((u) => HOME_URL_RE.test(u) && !u.includes("%")),
);
} catch (error) {
console.warn(` skip sitemap ${sm}: ${error instanceof Error ? error.message : String(error)}`);
}
}
// dedupe, preserve order
homeUrls = [...new Set(homeUrls)];
if (METRO_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const metro = u.match(/\/homes\/[^/]+\/([^/]+)\//)?.[1]?.toLowerCase();
return metro ? METRO_FILTER.includes(metro) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home needs. No
// per-home geo is published; the sales phone is builder-wide so it is
// carried on the community as well as the home for convenience.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
address: h.street,
builderInventoryId: h.builderInventoryId ?? page.url,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(h.price, h.priceRaw, page.url, h.priceRaw ? `price ${h.priceRaw}` : null),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url),
homeType: fv(h.homeType as never, null, page.url, `Brohn Homes ${h.homeType.toLowerCase()} inventory home`),
constructionStatus: fv(
h.status as never,
h.statusRaw,
page.url,
h.statusRaw ? `status "${h.statusRaw}"` : "no status label on page",
),
estCompletionDate: fv<string>(null, null, page.url), // no ready-date field on the page
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
// per-home geo NOT published by Brohn — left null (like richmond-american / smith-douglas)
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
planName: fv<string>(null, null, page.url), // no labelled plan name for the primary home
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};