← back to Homesonspec
collectors/berkeley-building/src/selftest.test.ts
376 lines
import { readFileSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { validatePayload } from "@homesonspec/schemas";
import { berkeleyBuildingAdapter, parseHome, statusFromLabel, selfCardStatusLabel } from "./index";
/**
* Bounded, read-only self-test for the Berkeley Building Co. adapter. Runs
* `parseHome` + `berkeleyBuildingAdapter.extract()` over the fixtures captured
* (rate-limited, honest UA, --compressed) from berkeleybuildingco.com on
* 2026-08-30, proves facts-only coverage, that addresses AND ids are genuinely
* per-home (not aggregate), that honest-null fields stay null (never fabricated,
* asserted directly on the real fixtures), that status is derived only from real
* page evidence, and that every emitted record validates against the schema.
*
* REAL INVENTORY: berkeleybuildingco.com's sitemap lists ~35 three-segment
* per-home detail pages across six IDAHO cities (Caldwell / Eagle / Kuna /
* Meridian / Nampa / Star). We captured a diverse 12-home sample; the floor is
* therefore the REAL captured count (12), NOT padded. N=12 makes the
* distinct-address / distinct-id assertions a genuine aggregate-vs-detail guard.
*
* SELECTOR DRIFT documented in the field-specific tests below:
* - garageSpaces + lotNumber are HONEST NULLS — Berkeley's primary
* HomeOverview__ block omits both (unlike Silverthorne's garage / arbor's lot).
* Asserted null directly on every real fixture.
* - price is genuinely < 100% coverage — 4 of the 12 captured homes are MODEL
* HOMES with no offers[].price → honest-null price (a real, unpadded case).
* - constructionStatus comes from the authoritative self-card HomeCard__status
* matched by the home's own detail path, present on EVERY captured home — so
* status is present on every home (asserted). PLANNED is exercised directly
* via statusFromLabel; stripping the self-card drops status to an honest null.
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const FX = join(HERE, "..", "fixtures", "homes");
// The real, honestly-reported captured-inventory floor (12 fixtures across all
// six ID cities). Bump when more fixtures are captured — never pad above real.
const REAL_HOME_FLOOR = 12;
// Real count of MODEL HOMES in the sample that carry no offers price (honest-null
// price). Exactly 4 of 12 — so priced homes = 8. Keeps the price-coverage bound
// honest and biting (not a toothless <= n).
const PRICED_HOMES = 8;
function urlMap(): Record<string, string> {
const tsv = readFileSync(join(FX, "_urls.tsv"), "utf8");
const out: Record<string, string> = {};
for (const line of tsv.split(/\r?\n/)) {
const [file, url] = line.split("\t");
if (file && url) out[file.trim()] = url.trim();
}
return out;
}
describe("berkeley-building adapter — coverage + per-home + honest-nulls + schema", () => {
const files = readdirSync(FX).filter((f) => f.endsWith(".html")).sort();
const urls = urlMap();
it("parses per-home facts with high coverage and distinct addresses + ids", () => {
let n = 0;
const cov = {
price: 0, beds: 0, baths: 0, sqft: 0, stories: 0, address: 0, garage: 0, lot: 0,
community: 0, plan: 0, id: 0, phone: 0, status: 0, geo: 0, zip: 0,
};
const addresses = new Set<string>();
const ids = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (!h) continue;
n++;
if (h.street) {
cov.address++;
addresses.add(`${h.street}, ${h.city}, ${h.state}`);
}
if (h.builderInventoryId) {
cov.id++;
ids.add(h.builderInventoryId);
}
if (h.price != null) cov.price++;
if (h.beds != null) cov.beds++;
if (h.bathsTotal != null) cov.baths++;
if (h.sqft != null) cov.sqft++;
if (h.stories != null) cov.stories++;
if (h.garages != null) cov.garage++;
if (h.lotNumber != null) cov.lot++;
if (h.community) cov.community++;
if (h.planName) cov.plan++;
if (h.phone) cov.phone++;
if (h.status) cov.status++;
if (h.lat != null && h.lon != null) cov.geo++;
if (h.zip) cov.zip++;
}
// eslint-disable-next-line no-console
console.log(
`\n[berkeley-building self-test] records=${n} distinctAddresses=${addresses.size} distinctIds=${ids.size}\n` +
` price=${pct(cov.price, n)} beds=${pct(cov.beds, n)} baths=${pct(cov.baths, n)} ` +
`sqft=${pct(cov.sqft, n)} stories=${pct(cov.stories, n)} garage=${pct(cov.garage, n)} ` +
`lot=${pct(cov.lot, n)} address=${pct(cov.address, n)} zip=${pct(cov.zip, n)} geo=${pct(cov.geo, n)} ` +
`community=${pct(cov.community, n)} plan=${pct(cov.plan, n)} ` +
`id=${pct(cov.id, n)} phone=${pct(cov.phone, n)} status=${pct(cov.status, n)}`,
);
// The REAL captured-inventory floor (honest — 12 fixtures across six ID cities).
expect(n).toBeGreaterThanOrEqual(REAL_HOME_FLOOR);
// Distinct street address AND builder id per record — a genuine
// aggregate-vs-detail guard at N=12 (every home carries its OWN JSON-LD
// address + productId; a regex that read an aggregate would collapse these).
expect(addresses.size).toBe(n);
expect(ids.size).toBe(n);
// Facts-only coverage — every core structural field present on every captured
// home (all sourced from the authoritative per-home JSON-LD + primary
// HomeOverview__ list + Floor Plan link).
expect(cov.address).toBe(n);
expect(cov.beds).toBe(n);
expect(cov.baths).toBe(n);
expect(cov.sqft).toBe(n);
expect(cov.stories).toBe(n);
expect(cov.geo).toBe(n);
expect(cov.zip).toBe(n);
expect(cov.community).toBe(n);
expect(cov.plan).toBe(n);
expect(cov.id).toBe(n);
expect(cov.phone).toBe(n);
// PRICE is genuinely < 100% — model homes carry no offers price. Exactly the
// real priced-home count, and strictly fewer than n (proves the honest null
// isn't being papered over by a fabricated price).
expect(cov.price).toBe(PRICED_HOMES);
expect(cov.price).toBeLessThan(n);
// GARAGE + LOT # are HONEST NULLS on Berkeley (the primary block omits both) —
// must be 0% coverage, never fabricated (the direct proof they're not guessed).
expect(cov.garage).toBe(0);
expect(cov.lot).toBe(0);
// STATUS: every Berkeley detail page carries a matching self-card
// HomeCard__status, so status is present on EVERY captured home. (An == n bound,
// not a toothless <= n — if the self-card match ever regressed to reading a
// sibling card or nothing, this bites immediately.)
expect(cov.status).toBe(n);
});
it("extracts the EXACT known facts of a captured home (value snapshot — catches misparses coverage can't)", () => {
// Pin every field of home-08 (2069 E Deep Purple Ln, Lavender Place, Meridian)
// to the value verified against the live JSON-LD + HomeOverview__ list + the
// self-card status, so any extraction drift fails loudly even though a
// coverage-% could still read 100% on garbage.
const h = parseHome(readFileSync(join(FX, "home-08.html"), "utf8"), urls["home-08.html"] ?? "")!;
expect(h.street).toBe("2069 E Deep Purple Ln");
expect(h.city).toBe("Meridian");
expect(h.state).toBe("ID");
expect(h.zip).toBe("83642");
expect(h.community).toBe("Lavender Place");
expect(h.planName).toBe("Violet");
expect(h.price).toBe(419900);
expect(h.beds).toBe(3);
expect(h.bathsTotal).toBe(2.5); // react-text-split "2"+".5" collapsed
expect(h.sqft).toBe(1595);
expect(h.stories).toBe(2);
expect(h.garages).toBeNull(); // honest null — Berkeley publishes no garage field
expect(h.lotNumber).toBeNull(); // honest null — Berkeley publishes no lot #
expect(h.status).toBe("UNDER_CONSTRUCTION"); // self-card "Available December 2026"
expect(h.builderInventoryId).toBe("6a3c1e9d5122a2a3ba56fb2d");
expect(h.lat).toBeCloseTo(43.5472, 2);
expect(h.lon).toBeCloseTo(-116.3689, 2);
});
it("honest-null fields (garageSpaces, lotNumber, estCompletionDate) stay null on the real fixtures — never fabricated", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const { records } = berkeleyBuildingAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
expect(home).toBeDefined();
const f = home!.fields as Record<string, { value: unknown }>;
// garageSpaces + lotNumber are genuine honest-nulls — Berkeley's primary
// block omits both — so they MUST be null on every real fixture.
expect(f["garageSpaces"]!.value).toBeNull();
expect(f["lotNumber"]!.value).toBeNull();
// estCompletionDate is a genuine honest-null — no per-home ISO date on page.
expect(f["estCompletionDate"]!.value).toBeNull();
// images is facts-only empty (mediaRights=NONE), never populated.
expect(f["images"]!.value).toEqual([]);
}
});
it("price is a genuine SEMANTIC null on model homes — never a fabricated stub", () => {
// 4 of 12 captured homes are model homes with no offers[].price. Prove those
// resolve to a real semantic null (not a $0 stub), and that priced homes carry
// a real integer > $50k.
let priced = 0;
let nulls = 0;
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
const { records } = berkeleyBuildingAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
const priceField = (home!.fields as Record<string, { value: unknown }>)["price"]!;
if (h.price == null) {
nulls++;
expect(priceField.value).toBeNull(); // direct semantic null, not a $0 stub
} else {
priced++;
expect(Number.isInteger(h.price)).toBe(true);
expect(h.price).toBeGreaterThan(50_000);
expect(priceField.value).toBe(h.price);
}
}
expect(priced).toBe(PRICED_HOMES);
expect(nulls).toBe(files.length - PRICED_HOMES);
// And stripping the offers price from a priced home → honest null (never a stub).
const html = readFileSync(join(FX, "home-08.html"), "utf8");
const noPrice = html.replace(/"price"\s*:\s*\d+/g, '"price":null');
expect(parseHome(noPrice, urls["home-08.html"]!)!.price).toBeNull();
});
it("canonicalHints.address is the REAL street, never a 'Lot X' fallback or URL", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const { records } = berkeleyBuildingAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
expect(home).toBeDefined();
const addr = home!.canonicalHints.address ?? "";
expect(addr).not.toMatch(/^lot\b/i);
expect(addr).not.toMatch(/^https?:\/\//);
expect(addr).toMatch(/\d/); // real street addresses carry a house number
// And it must equal the parsed street exactly (no ternary drift).
const h = parseHome(html, url);
expect(addr).toBe(h!.street);
}
});
it("confirms Berkeley is the Boise-IDAHO CPG builder — every home is ID (metro footprint ID/OR/WA)", () => {
const states = new Set<string>();
let sawID = false;
const cities = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (h?.state) {
states.add(h.state);
if (h.state === "ID") sawID = true;
}
if (h?.city) cities.add(h.city);
}
// Only Berkeley's real operating-metro states may appear — the sanity check
// that we're on the CPG Boise Berkeley Building Co.
for (const s of states) expect(["ID", "OR", "WA"]).toContain(s);
// The captured (Treasure Valley) inventory must be Idaho.
expect(sawID).toBe(true);
expect(states.size).toBe(1); // homogeneously ID in this sample
// Multiple distinct ID cities captured (diversity guard).
expect(cities.size).toBeGreaterThanOrEqual(4);
});
it("status is a valid enum-or-null derived from the self-card, and survives extraction as-is", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
expect(["PLANNED", "UNDER_CONSTRUCTION", "MOVE_IN_READY", null]).toContain(h.status);
const { records } = berkeleyBuildingAdapter.extract(mkPage(url, html));
const home = records.find((r) => r.entityType === "inventory_home");
expect((home!.fields as Record<string, { value: unknown }>)["constructionStatus"]!.value).toBe(h.status);
}
// Stripping the self-card (the ONLY status source) from home-08 must drop
// status to null while the home still parses (a DIRECT semantic-null assertion):
// proves status is derived ONLY from real page evidence and never guessed.
const html = readFileSync(join(FX, "home-08.html"), "utf8");
const url = urls["home-08.html"]!;
const stripped = html.replace(/class="HomeCard__status/g, 'class="_gone_');
const strippedHome = parseHome(stripped, url);
expect(strippedHome?.status).toBeNull();
expect(strippedHome?.street, "null-status home still needs a real street").toBeTruthy();
});
it("selfCardStatusLabel is isolated per home path (no sibling leak) + covers every status branch end-to-end", () => {
// Cody catch: the old test proved isolation only against a NONEXISTENT path —
// a decoy. This drives REAL sibling card paths on ONE captured community-grid
// page (home-08) so each path resolves to its OWN card's label + status. Same
// page, different paths, DIFFERENT verified statuses = genuine exact-path
// isolation. It also exercises the Pending->null and Available<Month>/Under-
// Construction branches END-TO-END (no captured PRIMARY home is Pending/PLANNED,
// so the fixture would otherwise never drive those paths through parseHome).
const html = readFileSync(join(FX, "home-08.html"), "utf8");
const cases: [string, string | null, string | null][] = [
// [path, expected raw card label, expected mapped status]
["/homes/meridian/lavender-place/2069-e-deep-purple-ln", "Available December 2026", "UNDER_CONSTRUCTION"], // self
["/homes/meridian/oaklawn/8209-w-gallup-st", "Model Home", "MOVE_IN_READY"], // sibling
["/homes/caldwell/solstice/3405-elliptical-ln", "Quick Move-In!", "MOVE_IN_READY"], // sibling
["/homes/eagle/valnova/6480-w-sollas-ct", "Under Construction", "UNDER_CONSTRUCTION"], // sibling
["/homes/undefined/undefined/1485-e-andes-dr", "Pending", null], // sibling — Pending -> honest null
["/homes/nowhere/none/0-nope", null, null], // absent path -> null (never a stray sibling)
];
for (const [path, label, status] of cases) {
expect(selfCardStatusLabel(html, path), `card label for ${path}`).toBe(label);
expect(statusFromLabel(selfCardStatusLabel(html, path)), `mapped status for ${path}`).toBe(status);
}
});
it("statusFromLabel maps every branch — no dead MOVE_IN_READY/UNDER_CONSTRUCTION/PLANNED code", () => {
// MOVE_IN_READY
expect(statusFromLabel("Quick Move-In!")).toBe("MOVE_IN_READY");
expect(statusFromLabel("Move-In Ready")).toBe("MOVE_IN_READY");
expect(statusFromLabel("Model Home")).toBe("MOVE_IN_READY");
expect(statusFromLabel("Available Now")).toBe("MOVE_IN_READY");
// UNDER_CONSTRUCTION
expect(statusFromLabel("Under Construction")).toBe("UNDER_CONSTRUCTION");
expect(statusFromLabel("currently being built")).toBe("UNDER_CONSTRUCTION");
expect(statusFromLabel("Available December 2026")).toBe("UNDER_CONSTRUCTION");
expect(statusFromLabel("Available September 2026")).toBe("UNDER_CONSTRUCTION");
// PLANNED
expect(statusFromLabel("Coming Soon")).toBe("PLANNED");
expect(statusFromLabel("To Be Built")).toBe("PLANNED");
expect(statusFromLabel("Presale")).toBe("PLANNED");
// "Pending" is a sale state, NOT a construction stage → honest null.
expect(statusFromLabel("Pending")).toBeNull();
// Unrecognized / absent → honest null, never a guess.
expect(statusFromLabel("a beautiful two-story home")).toBeNull();
expect(statusFromLabel("")).toBeNull();
expect(statusFromLabel(null)).toBeNull();
});
it("geo comes from real sources (Idaho / Treasure Valley bounding box)", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
// Geo is present on every home (JSON-LD GeoCoordinates) and must sit in a
// Boise-metro bounding box — proves real coords, not placeholders.
expect(h.lat!).toBeGreaterThan(43);
expect(h.lat!).toBeLessThan(44);
expect(h.lon!).toBeLessThan(-116);
expect(h.lon!).toBeGreaterThan(-117);
}
});
it("emits schema-valid community + inventory_home records", () => {
let homes = 0;
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const { records, errors } = berkeleyBuildingAdapter.extract(mkPage(url, html));
expect(errors).toEqual([]);
for (const rec of records) {
if (rec.entityType === "inventory_home") homes++;
const result = validatePayload(rec);
if (!result.ok) {
// eslint-disable-next-line no-console
console.error(`validation failed for ${rec.entityType} @ ${url}`, result);
}
expect(result.ok).toBe(true);
}
}
expect(homes).toBeGreaterThanOrEqual(REAL_HOME_FLOOR);
});
});
function mkPage(url: string, html: string) {
return {
url,
retrievedAt: new Date().toISOString(),
contentType: "text/html",
body: Buffer.from(html),
contentHash: "test",
};
}
function pct(a: number, b: number): string {
return b ? `${Math.round((100 * a) / b)}%` : "n/a";
}