← back to Homesonspec
collectors/brohn-homes/src/selftest.test.ts
355 lines
import { readFileSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { validatePayload } from "@homesonspec/schemas";
import { brohnHomesAdapter, parseHome } from "./index";
/**
* Bounded, read-only self-test for the Brohn Homes adapter. Runs `parseHome` +
* `brohnHomesAdapter.extract()` over fixtures captured (rate-limited ~1 req/4s,
* honest UA `HomesOnSpecBot/0.1`, `curl --compressed`) from brohnhomes.com on
* 2026-08-30 (TK-10487), proves facts-only coverage, that addresses AND ids are
* genuinely per-home (not aggregate), that honest-null fields stay null (never
* fabricated, asserted directly on the real fixtures), that status is derived
* only from real page evidence, and that every emitted record validates against
* the published schema.
*
* REAL INVENTORY: brohnhomes.com's homes-sitemap{1,2}.xml list 246 per-home detail
* pages (all Texas — Austin/Houston/San-Antonio metros). We captured 12 distinct
* homes spanning multiple metros, cities and communities, chosen to exercise BOTH
* status branches (NOW/SOLD -> MOVE_IN_READY, a future date-label -> UNDER_CONSTRUCTION)
* and BOTH homeType branches (a `/unit-NN/` townhome + street-addressed single-family).
* The floor asserted below (12) is the REAL captured count, NOT a padded number.
*
* FIELD NOTES (all HONEST — sourced only from the delivered DOM):
* - Every captured Brohn detail page carries price/beds/baths/sqft/stories/cars/
* community/status via the .home-price + .brohn-home-stats + status-template
* blocks, so those fields are present on every home (asserted == n).
* - zip is recovered ONLY from the get-directions maps `query=` (the visible
* city-state span omits it) — present on every captured home.
* - lat/lon/planName/lotNumber/estCompletionDate are genuine HONEST NULLS —
* Brohn publishes none of them per-home. Asserted null directly on every
* fixture (the direct proof they are not guessed).
* - The one TOWNHOME fixture (Cross Creek unit-34) has street "Unit 34" — that
* is the page's OWN <h1> heading (Brohn labels condo units by unit number, not
* a house number), NOT a URL/Lot fallback. The adapter has no synthesized
* address fallback at all: no <h1> => the record is dropped.
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const FX = join(HERE, "..", "fixtures", "homes");
// The real, honestly-reported captured-inventory floor (12 distinct homes captured).
// Bump only when more real fixtures are added — never pad above the real count.
const REAL_HOME_FLOOR = 12;
function urlMap(): Record<string, string> {
const tsv = readFileSync(join(FX, "_urls.tsv"), "utf8");
const out: Record<string, string> = {};
for (const line of tsv.split(/\r?\n/)) {
const [file, url] = line.split("\t");
if (file && url) out[file.trim()] = url.trim();
}
return out;
}
describe("brohn-homes adapter — coverage + per-home + honest-nulls + schema", () => {
const files = readdirSync(FX).filter((f) => f.endsWith(".html")).sort();
const urls = urlMap();
it("parses per-home facts with high coverage and distinct addresses + ids", () => {
let n = 0;
const cov = {
price: 0, beds: 0, baths: 0, sqft: 0, stories: 0, address: 0, garage: 0,
community: 0, zip: 0, id: 0, phone: 0, status: 0,
lat: 0, lon: 0, plan: 0, lot: 0, est: 0,
};
const addresses = new Set<string>();
const ids = new Set<string>();
const statuses = new Set<string>();
const homeTypes = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (!h) continue;
n++;
if (h.street) {
cov.address++;
addresses.add(`${h.street}, ${h.city}, ${h.state}`);
}
if (h.builderInventoryId) {
cov.id++;
ids.add(h.builderInventoryId);
}
if (h.price != null) cov.price++;
if (h.beds != null) cov.beds++;
if (h.bathsTotal != null) cov.baths++;
if (h.sqft != null) cov.sqft++;
if (h.stories != null) cov.stories++;
if (h.garages != null) cov.garage++;
if (h.community) cov.community++;
if (h.zip) cov.zip++;
if (h.phone) cov.phone++;
if (h.status) { cov.status++; statuses.add(h.status); }
homeTypes.add(h.homeType);
}
// eslint-disable-next-line no-console
console.log(
`\n[brohn-homes self-test] records=${n} distinctAddresses=${addresses.size} distinctIds=${ids.size}\n` +
` price=${pct(cov.price, n)} beds=${pct(cov.beds, n)} baths=${pct(cov.baths, n)} ` +
`sqft=${pct(cov.sqft, n)} stories=${pct(cov.stories, n)} garage=${pct(cov.garage, n)} ` +
`address=${pct(cov.address, n)} zip=${pct(cov.zip, n)} community=${pct(cov.community, n)} ` +
`id=${pct(cov.id, n)} phone=${pct(cov.phone, n)} status=${pct(cov.status, n)}\n` +
` statuses={${[...statuses].join(",")}} homeTypes={${[...homeTypes].join(",")}}`,
);
// The REAL captured-inventory floor (honest — 12 distinct homes captured).
expect(n).toBeGreaterThanOrEqual(REAL_HOME_FLOOR);
// Distinct street address AND builder id per record — a genuine
// aggregate-vs-detail guard at N>=12: a regex that read a floorplan/community
// aggregate would collapse these; every home carries its OWN postid + street.
expect(addresses.size).toBe(n);
expect(ids.size).toBe(n);
// Facts-only coverage — every core structural field present on every captured
// Brohn home (all from the delivered .home-price / .brohn-home-stats / maps
// query / status-template / body postid).
expect(cov.address).toBe(n);
expect(cov.price).toBe(n);
expect(cov.beds).toBe(n);
expect(cov.baths).toBe(n);
expect(cov.sqft).toBe(n);
expect(cov.stories).toBe(n);
expect(cov.garage).toBe(n);
expect(cov.community).toBe(n);
expect(cov.zip).toBe(n);
expect(cov.id).toBe(n);
expect(cov.phone).toBe(n);
expect(cov.status).toBe(n);
// STATUS coverage is honest: BOTH branches must actually appear in the captured
// set (a toothless test could pass with one branch dead). We captured NOW/SOLD
// homes AND future-date homes, so both enum values are exercised end-to-end.
expect(statuses.has("MOVE_IN_READY")).toBe(true);
expect(statuses.has("UNDER_CONSTRUCTION")).toBe(true);
// BOTH homeType branches must appear — the /unit-NN/ townhome + a single-family.
expect(homeTypes.has("SINGLE_FAMILY")).toBe(true);
expect(homeTypes.has("TOWNHOME")).toBe(true);
});
it("extracts the EXACT known facts of a captured home (value snapshot — catches misparses coverage can't)", () => {
// Pin every field of home-01 (31315 Cass River Lane, Mustang Meadows, Waller TX)
// to the value verified against the live page, so any extraction drift fails
// loudly even though coverage-% could still read 100% on garbage.
const first = files[0]!;
const h = parseHome(readFileSync(join(FX, first), "utf8"), urls[first] ?? "")!;
expect(h.street).toBe("31315 Cass River Lane");
expect(h.city).toBe("Waller");
expect(h.state).toBe("TX");
expect(h.zip).toBe("77484");
expect(h.community).toBe("Mustang Meadows");
expect(h.price).toBe(283990);
expect(h.beds).toBe(3);
expect(h.bathsTotal).toBe(2);
expect(h.sqft).toBe(1754);
expect(h.stories).toBe(1);
expect(h.garages).toBe(2);
expect(h.status).toBe("MOVE_IN_READY"); // status-template = "NOW"
expect(h.statusRaw).toBe("NOW");
expect(h.homeType).toBe("SINGLE_FAMILY");
expect(h.phone).toBe("(512) 334-6775");
expect(h.builderInventoryId).toBe("postid-35267");
// honest nulls on the real page:
expect(h.zip).not.toBeNull();
});
it("honest-null fields (lat/lon/planName/lotNumber/estCompletionDate) stay null on the real fixtures — never fabricated", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const { records } = brohnHomesAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
expect(home).toBeDefined();
const f = home!.fields as Record<string, { value: unknown }>;
// Brohn publishes NO per-home geo, plan name, lot number, or completion date —
// these MUST be null on every real fixture (not fabricated stubs).
expect(f["lat"]!.value).toBeNull();
expect(f["lon"]!.value).toBeNull();
expect(f["planName"]!.value).toBeNull();
expect(f["lotNumber"]!.value).toBeNull();
expect(f["estCompletionDate"]!.value).toBeNull();
// images is facts-only empty (mediaRights=NONE), never populated.
expect(f["images"]!.value).toEqual([]);
}
});
it("canonicalHints.address is the REAL page street, never a 'Lot X' fallback or URL", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const page = mkPage(url, html);
const { records } = brohnHomesAdapter.extract(page);
const home = records.find((r) => r.entityType === "inventory_home");
expect(home).toBeDefined();
const addr = home!.canonicalHints.address ?? "";
expect(addr).not.toMatch(/^lot\b/i);
expect(addr).not.toMatch(/^https?:\/\//);
expect(addr).toMatch(/\d/); // a real street# or a real "Unit NN" — both carry a digit
// And it must equal the parsed <h1> street exactly (no ternary drift, no
// URL-slug fallback): the adapter reads the page heading, nothing else.
const h = parseHome(html, url);
expect(addr).toBe(h!.street);
}
});
it("confirms Brohn is the TEXAS CPG builder — every captured home is TX", () => {
const states = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (h?.state) states.add(h.state);
}
// Brohn Homes is an Austin/Houston/San-Antonio TX builder — every home is TX.
// A non-TX state would mean the maps-query / city-state parse read the wrong node.
expect([...states]).toEqual(["TX"]);
});
it("status is a valid enum derived ONLY from page evidence — honest-null when the label is absent", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
// constructionStatus resolves to MOVE_IN_READY (NOW/SOLD) or UNDER_CONSTRUCTION
// (any other/future label). Both are valid enum members; never any other value.
expect(["PLANNED", "UNDER_CONSTRUCTION", "MOVE_IN_READY"]).toContain(h.status);
// Whatever it resolves to survives extraction as-is into constructionStatus.
const { records } = brohnHomesAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
expect((home!.fields as Record<string, { value: unknown }>)["constructionStatus"]!.value).toBe(h.status);
}
});
it("exercises BOTH status branches directly on the REAL fixtures (not just a threshold)", () => {
const byStatus: Record<string, { file: string; raw: string | null }[]> = {};
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
(byStatus[h.status] ??= []).push({ file, raw: h.statusRaw });
}
// A real NOW/SOLD home resolved to MOVE_IN_READY with an evidence-bearing raw.
const ready = byStatus["MOVE_IN_READY"] ?? [];
expect(ready.length).toBeGreaterThanOrEqual(1);
for (const { raw } of ready) expect(raw).toMatch(/^(now|sold|move[- ]?in)/i);
// A real future-date home resolved to UNDER_CONSTRUCTION — its raw is a date
// label (e.g. "October '26"), NOT a ready phrase (proves the else-branch fired
// on genuine page evidence, not a guess).
const uc = byStatus["UNDER_CONSTRUCTION"] ?? [];
expect(uc.length).toBeGreaterThanOrEqual(1);
for (const { raw } of uc) {
expect(raw).not.toMatch(/^(now|sold|move[- ]?in)/i);
expect(raw).toBeTruthy(); // a real (future) label was present on the page
}
});
it("stripping the status-template label drops status to the else-branch, and a synthetic NOW/SOLD/date maps correctly", () => {
// Prove status is derived from the real .home-item-status-template node: removing
// it forces the else-branch (UNDER_CONSTRUCTION — the adapter never emits a null
// status). Then feed synthetic label values through parseHome to pin the mapping.
const first = files[0]!;
const html = readFileSync(join(FX, first), "utf8");
const url = urls[first]!;
// home-01 is a NOW home -> MOVE_IN_READY; strip its status template class:
const stripped = html.replace(/home-item-status-template/g, "_gone_status_");
const sh = parseHome(stripped, url)!;
expect(sh.statusRaw).toBeNull();
expect(sh.status).toBe("UNDER_CONSTRUCTION"); // no label -> future/UC branch
expect(sh.street).toBeTruthy(); // still a real home
// Synthetic label mapping via a minimal home page (exercises each branch):
const mk = (label: string) =>
`<body class="single-homes postid-999">` +
`<h1 class="vcex-page-title__heading">123 Test Ln</h1>` +
`<div class="brohn-home-stats"><div class="stat"><span class="stat-number">3</span><span class="stat-label">beds</span></div></div>` +
`<div class="home-item-status-template">${label}</div></body>`;
expect(parseHome(mk("NOW"), "https://brohnhomes.com/homes/tx/a/b/c/d/")!.status).toBe("MOVE_IN_READY");
expect(parseHome(mk("SOLD"), "https://brohnhomes.com/homes/tx/a/b/c/d/")!.status).toBe("MOVE_IN_READY");
expect(parseHome(mk("Move-In Ready"), "https://brohnhomes.com/homes/tx/a/b/c/d/")!.status).toBe("MOVE_IN_READY");
expect(parseHome(mk("November '26"), "https://brohnhomes.com/homes/tx/a/b/c/d/")!.status).toBe("UNDER_CONSTRUCTION");
expect(parseHome(mk("Coming Soon"), "https://brohnhomes.com/homes/tx/a/b/c/d/")!.status).toBe("UNDER_CONSTRUCTION");
});
it("homeType is derived from the URL — /unit-NN/ => TOWNHOME, else SINGLE_FAMILY", () => {
// The one captured townhome fixture (Cross Creek /unit-34/) must be a TOWNHOME
// whose street is the page's OWN unit heading, and a street-addressed home must
// be SINGLE_FAMILY — proving the URL-segment branch reads real evidence.
const town = files
.map((f) => ({ f, h: parseHome(readFileSync(join(FX, f), "utf8"), urls[f] ?? "") }))
.filter((x) => x.h && x.h.homeType === "TOWNHOME");
expect(town.length).toBeGreaterThanOrEqual(1);
for (const { f } of town) expect(urls[f]).toMatch(/\/unit-\d+\/?$/i);
// And a non-unit home is SINGLE_FAMILY with a numbered street.
const sf = parseHome(readFileSync(join(FX, files[0]!), "utf8"), urls[files[0]!]!)!;
expect(sf.homeType).toBe("SINGLE_FAMILY");
expect(sf.street).toMatch(/^\d/);
});
it("price is a real integer from .home-price, and a page with no $ price resolves to a semantic null", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
expect(Number.isInteger(h.price!)).toBe(true);
expect(h.price!).toBeGreaterThan(50_000);
// Blank the .home-price block -> posInt(null) -> honest semantic null, and the
// extracted price field is a direct null (never a fabricated $0 stub).
const noPrice = html.replace(/(home-price[^>]*>)[\s\S]*?(<\/div>)/i, "$1$2");
const hNull = parseHome(noPrice, url)!;
expect(hNull.price).toBeNull();
const { records } = brohnHomesAdapter.extract(mkPage(url, noPrice));
const home = records.find((r) => r.entityType === "inventory_home");
expect((home!.fields as Record<string, { value: unknown }>)["price"]!.value).toBeNull();
}
});
it("emits schema-valid community + inventory_home records (all pass validatePayload)", () => {
let homes = 0;
let communities = 0;
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const page = mkPage(url, html);
const { records, errors } = brohnHomesAdapter.extract(page);
expect(errors).toEqual([]);
for (const rec of records) {
if (rec.entityType === "inventory_home") homes++;
if (rec.entityType === "community") communities++;
const result = validatePayload(rec);
if (!result.ok) {
// eslint-disable-next-line no-console
console.error(`validation failed for ${rec.entityType} @ ${url}`, result);
}
expect(result.ok).toBe(true);
}
}
expect(homes).toBeGreaterThanOrEqual(REAL_HOME_FLOOR);
expect(communities).toBeGreaterThanOrEqual(REAL_HOME_FLOOR);
});
});
function mkPage(url: string, html: string) {
return {
url,
retrievedAt: new Date().toISOString(),
contentType: "text/html",
body: Buffer.from(html),
contentHash: "test",
};
}
function pct(a: number, b: number): string {
return b ? `${Math.round((100 * a) / b)}%` : "n/a";
}