← back to Homesonspec
collectors/arbor/src/selftest.test.ts
252 lines
import { readFileSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { validatePayload } from "@homesonspec/schemas";
import { arborAdapter, parseHome, statusFromLead } from "./index";
/**
* Bounded, read-only self-test for the Arbor Homes adapter. Runs `parseHome` +
* `arborAdapter.extract()` over fixtures captured (rate-limited, honest UA) from
* yourarborhome.com across IN + KY + OH / multiple communities on 2026-08-30,
* proves facts-only coverage, that addresses AND ids are genuinely per-home (not
* aggregate), that honest-null fields stay null (never fabricated), that status
* is derived only from real page evidence, and that every emitted record
* validates against the published schema.
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const FX = join(HERE, "..", "fixtures", "homes");
function urlMap(): Record<string, string> {
const tsv = readFileSync(join(FX, "_urls.tsv"), "utf8");
const out: Record<string, string> = {};
for (const line of tsv.split(/\r?\n/)) {
const [file, url] = line.split("\t");
if (file && url) out[file.trim()] = url.trim();
}
return out;
}
describe("arbor adapter — coverage + per-home + honest-nulls + schema", () => {
const files = readdirSync(FX).filter((f) => f.endsWith(".html")).sort();
const urls = urlMap();
it("parses per-home facts with high coverage and distinct addresses + ids", () => {
let n = 0;
const cov = {
price: 0, beds: 0, baths: 0, sqft: 0, stories: 0, address: 0, garage: 0,
community: 0, plan: 0, lot: 0, id: 0, phone: 0, status: 0, geo: 0, zip: 0,
};
const addresses = new Set<string>();
const ids = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (!h) continue;
n++;
if (h.street) {
cov.address++;
addresses.add(`${h.street}, ${h.city}, ${h.state}`);
}
if (h.builderInventoryId) {
cov.id++;
ids.add(h.builderInventoryId);
}
if (h.price != null) cov.price++;
if (h.beds != null) cov.beds++;
if (h.bathsTotal != null) cov.baths++;
if (h.sqft != null) cov.sqft++;
if (h.stories != null) cov.stories++;
if (h.garages != null) cov.garage++;
if (h.community) cov.community++;
if (h.planName) cov.plan++;
if (h.lotNumber) cov.lot++;
if (h.phone) cov.phone++;
if (h.status) cov.status++;
if (h.lat != null && h.lon != null) cov.geo++;
if (h.zip) cov.zip++;
}
// eslint-disable-next-line no-console
console.log(
`\n[arbor self-test] records=${n} distinctAddresses=${addresses.size} distinctIds=${ids.size}\n` +
` price=${pct(cov.price, n)} beds=${pct(cov.beds, n)} baths=${pct(cov.baths, n)} ` +
`sqft=${pct(cov.sqft, n)} stories=${pct(cov.stories, n)} address=${pct(cov.address, n)} ` +
`zip=${pct(cov.zip, n)} geo=${pct(cov.geo, n)} community=${pct(cov.community, n)} ` +
`plan=${pct(cov.plan, n)} lot=${pct(cov.lot, n)} id=${pct(cov.id, n)} ` +
`phone=${pct(cov.phone, n)} status=${pct(cov.status, n)} garage=${pct(cov.garage, n)}(honest-null)`,
);
expect(n).toBeGreaterThanOrEqual(11);
// Genuinely per-home: one distinct street address AND one distinct builder id
// per parsed record (proves detail pages, not floorplan/community aggregates).
expect(addresses.size).toBe(n);
expect(ids.size).toBe(n);
// Facts-only coverage — every core structural field present on every Arbor
// QMI (all sourced from the authoritative per-home JSON-LD + primary
// HomeOverview).
expect(cov.address).toBe(n);
expect(cov.beds).toBe(n);
expect(cov.baths).toBe(n);
expect(cov.sqft).toBe(n);
expect(cov.stories).toBe(n);
expect(cov.geo).toBe(n);
expect(cov.zip).toBe(n);
expect(cov.community).toBe(n);
expect(cov.plan).toBe(n);
expect(cov.id).toBe(n);
// STATUS is honest-null when the lead span carries no recognizable state
// (e.g. a "Model" home) — never guessed. Assert a strong majority carry it.
expect(cov.status / n).toBeGreaterThanOrEqual(0.85);
// PRICE is an HONEST-NULL field — model homes and some coming-soon QMIs
// publish no offers[].price, so a minority legitimately lack it. Assert a
// strong majority carry it (never fabricated for the ones that don't).
expect(cov.price / n).toBeGreaterThanOrEqual(0.6);
// Lot + phone present on the overwhelming majority; small slack for a page
// that omits one (a model home carries no lot #).
expect(cov.lot / n).toBeGreaterThanOrEqual(0.85);
expect(cov.phone / n).toBeGreaterThanOrEqual(0.9);
// HONEST NULL — Arbor's per-home page publishes no garage field; it must
// NEVER be fabricated. Assert it stays 0.
expect(cov.garage).toBe(0);
});
it("canonicalHints.address is the REAL street, never a 'Lot X' fallback or URL", () => {
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const page = mkPage(url, html);
const { records } = arborAdapter.extract(page);
const home = records.find((r) => r.entityType === "inventory_home");
expect(home).toBeDefined();
const addr = home!.canonicalHints.address ?? "";
expect(addr).not.toMatch(/^lot\b/i);
expect(addr).not.toMatch(/^https?:\/\//);
expect(addr).toMatch(/\d/); // real street addresses carry a house number
// And it must equal the parsed street exactly (no ternary drift).
const h = parseHome(html, url);
expect(addr).toBe(h!.street);
}
});
it("confirms Arbor is the Indianapolis CPG builder — every home is IN/KY/OH", () => {
const states = new Set<string>();
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url);
if (h?.state) states.add(h.state);
}
// Only Arbor's real operating states may appear — a sanity check that we're
// on the CPG Indianapolis Arbor (not the unrelated Oregon arborhomes.com).
for (const s of states) expect(["IN", "KY", "OH"]).toContain(s);
// And the captured set spans more than one state.
expect(states.size).toBeGreaterThanOrEqual(2);
});
it("status is a valid enum derived from page evidence — null when absent", () => {
const first = files[0]!;
const html = readFileSync(join(FX, first), "utf8");
const url = urls[first]!;
const status = parseHome(html, url)?.status;
expect(["PLANNED", "UNDER_CONSTRUCTION", "MOVE_IN_READY", null]).toContain(status);
// Removing the DetailHeader lead span must drop status to a null (never guessed).
const stripped = html.replace(/class="DetailHeader_h2Lead"/g, 'class="_gone_"');
expect(parseHome(stripped, url)?.status).toBeNull();
});
it("the REAL Model home returns a semantic null status (not just the stripped path)", () => {
// Cody catch: the threshold-coverage test could not distinguish a correct
// null from a wrong non-null on the one genuinely-unlabeled home. Assert the
// actual fixture (a "Model" home with no QMI lead) directly resolves to null.
const nulls = files
.map((f) => ({ f, s: parseHome(readFileSync(join(FX, f), "utf8"), urls[f] ?? "") }))
.filter((x) => x.s && x.s.status === null);
expect(nulls.length, "expected at least one real null-status (Model) fixture").toBeGreaterThanOrEqual(1);
for (const { f, s } of nulls) {
// A null-status home is a Model / unlabeled home — it must NOT be silently
// coerced to a non-null status, and it still parses as a real home.
expect(s!.status).toBeNull();
expect(s!.street, `null-status home ${f} still needs a real street`).toBeTruthy();
}
});
it("statusFromLead maps every branch — no dead PLANNED/UNDER_CONSTRUCTION code", () => {
// These two branches fire on live "Coming Soon" / "Under Construction" homes
// that no captured fixture happens to include — exercise them directly so a
// regression in the mapping can't ship untested. (Cody dead-branch catch)
expect(statusFromLead("Quick Move-In Home")).toBe("MOVE_IN_READY");
expect(statusFromLead("Available Now")).toBe("MOVE_IN_READY");
expect(statusFromLead("Under Construction")).toBe("UNDER_CONSTRUCTION");
expect(statusFromLead("Currently being built")).toBe("UNDER_CONSTRUCTION");
expect(statusFromLead("Coming Soon")).toBe("PLANNED");
expect(statusFromLead("To Be Built")).toBe("PLANNED");
expect(statusFromLead("Presale")).toBe("PLANNED");
// Unrecognized / absent -> honest null, never a guess.
expect(statusFromLead("Model")).toBeNull();
expect(statusFromLead("")).toBeNull();
expect(statusFromLead(null)).toBeNull();
});
it("price + geo come from JSON-LD (an integer + real coords), never a fabricated value", () => {
let priced = 0;
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const h = parseHome(html, url)!;
// Price is honest-null on unpriced homes; when present it must be a real
// integer new-home price (from offers[].price), never a fabricated stub.
if (h.price != null) {
priced++;
expect(Number.isInteger(h.price)).toBe(true);
expect(h.price).toBeGreaterThan(50_000);
}
// Geo is present on every home (from the JSON-LD GeoCoordinates) and must
// sit in a US-Midwest bounding box — proves real coords, not placeholders.
expect(h.lat!).toBeGreaterThan(36);
expect(h.lat!).toBeLessThan(42);
expect(h.lon!).toBeLessThan(-82);
expect(h.lon!).toBeGreaterThan(-88);
}
expect(priced).toBeGreaterThanOrEqual(7); // most fixtures carry a real price
});
it("emits schema-valid community + inventory_home records", () => {
let homes = 0;
for (const file of files) {
const html = readFileSync(join(FX, file), "utf8");
const url = urls[file] ?? `fixture://${file}`;
const page = mkPage(url, html);
const { records, errors } = arborAdapter.extract(page);
expect(errors).toEqual([]);
for (const rec of records) {
if (rec.entityType === "inventory_home") homes++;
const result = validatePayload(rec);
if (!result.ok) {
// eslint-disable-next-line no-console
console.error(`validation failed for ${rec.entityType} @ ${url}`, result);
}
expect(result.ok).toBe(true);
}
}
expect(homes).toBeGreaterThanOrEqual(11);
});
});
function mkPage(url: string, html: string) {
return {
url,
retrievedAt: new Date().toISOString(),
contentType: "text/html",
body: Buffer.from(html),
contentHash: "test",
};
}
function pct(a: number, b: number): string {
return b ? `${Math.round((100 * a) / b)}%` : "n/a";
}