`collectors/harris-doyle/src/selftest.test.ts`

188 lines
import { readFileSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { validatePayload } from "@homesonspec/schemas";
import { harrisDoyleAdapter, parseHome } from "./index";

/**
 * Bounded, read-only self-test for the Harris Doyle Homes adapter. Runs
 * `parseHome` + `harrisDoyleAdapter.extract()` over 14 fixtures captured
 * (rate-limited, honest UA) from harrisdoyle.com across AL + FL / 11 communities
 * on 2026-08-30, proves facts-only coverage, that addresses AND ids are
 * genuinely per-home (not aggregate), that honest-null fields stay null
 * (never fabricated), and that every emitted record validates against the
 * published schema.
 */

const HERE = dirname(fileURLToPath(import.meta.url));
const FX = join(HERE, "..", "fixtures", "homes");

function urlMap(): Record<string, string> {
  const tsv = readFileSync(join(FX, "_urls.tsv"), "utf8");
  const out: Record<string, string> = {};
  for (const line of tsv.split(/\r?\n/)) {
    const [file, url] = line.split("\t");
    if (file && url) out[file.trim()] = url.trim();
  }
  return out;
}

describe("harris-doyle adapter — coverage + per-home + honest-nulls + schema", () => {
  const files = readdirSync(FX).filter((f) => f.endsWith(".html")).sort();
  const urls = urlMap();

  it("parses per-home facts with high coverage and distinct addresses + ids", () => {
    let n = 0;
    const cov = {
      price: 0, beds: 0, baths: 0, sqft: 0, address: 0, garage: 0, community: 0,
      plan: 0, lot: 0, id: 0, phone: 0, status: 0, geo: 0, stories: 0,
    };
    const addresses = new Set<string>();
    const ids = new Set<string>();

    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const h = parseHome(html, url);
      if (!h) continue;
      n++;
      if (h.street) {
        cov.address++;
        addresses.add(`${h.street}, ${h.city}, ${h.state}`);
      }
      if (h.builderInventoryId) {
        cov.id++;
        ids.add(h.builderInventoryId);
      }
      if (h.price != null) cov.price++;
      if (h.beds != null) cov.beds++;
      if (h.bathsTotal != null) cov.baths++;
      if (h.sqft != null) cov.sqft++;
      if (h.garages != null) cov.garage++;
      if (h.community) cov.community++;
      if (h.planName) cov.plan++;
      if (h.lotNumber) cov.lot++;
      if (h.phone) cov.phone++;
      if (h.status) cov.status++;
      if (h.lat != null && h.lon != null) cov.geo++;
      if (h.stories != null) cov.stories++;
    }

    // eslint-disable-next-line no-console
    console.log(
      `\n[harris-doyle self-test] records=${n} distinctAddresses=${addresses.size} distinctIds=${ids.size}\n` +
        `  price=${pct(cov.price, n)}  beds=${pct(cov.beds, n)}  baths=${pct(cov.baths, n)}  ` +
        `sqft=${pct(cov.sqft, n)}  address=${pct(cov.address, n)}  garage=${pct(cov.garage, n)}  ` +
        `community=${pct(cov.community, n)}  plan=${pct(cov.plan, n)}  lot=${pct(cov.lot, n)}  ` +
        `id=${pct(cov.id, n)}  phone=${pct(cov.phone, n)}  status=${pct(cov.status, n)}  ` +
        `geo=${pct(cov.geo, n)}(honest-null)  stories=${pct(cov.stories, n)}(honest-null)`,
    );

    expect(n).toBeGreaterThanOrEqual(13);
    // Genuinely per-home: one distinct street address AND one distinct builder id
    // per parsed record (proves detail pages, not floorplan/community aggregates).
    expect(addresses.size).toBe(n);
    expect(ids.size).toBe(n);
    // Facts-only coverage — every core field present on every Harris Doyle QMI.
    expect(cov.address).toBe(n);
    expect(cov.price).toBe(n);
    expect(cov.beds).toBe(n);
    expect(cov.baths).toBe(n);
    expect(cov.sqft).toBe(n);
    expect(cov.garage).toBe(n);
    expect(cov.community).toBe(n);
    expect(cov.plan).toBe(n);
    expect(cov.id).toBe(n);
    expect(cov.status).toBe(n);
    // Lot + phone present on the overwhelming majority; small slack for a page
    // that omits one.
    expect(cov.lot / n).toBeGreaterThanOrEqual(0.9);
    expect(cov.phone / n).toBeGreaterThanOrEqual(0.9);
    // HONEST NULLS — these are genuinely absent from the page and must NEVER be
    // fabricated: no per-home geo, no per-home stories field. Assert they stay 0.
    expect(cov.geo).toBe(0);
    expect(cov.stories).toBe(0);
  });

  it("canonicalHints.address is the REAL street, never a 'Lot X' fallback", () => {
    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const page = mkPage(url, html);
      const { records } = harrisDoyleAdapter.extract(page);
      const home = records.find((r) => r.entityType === "inventory_home");
      expect(home).toBeDefined();
      const addr = home!.canonicalHints.address ?? "";
      // Must be a real street (has a number + a street word) and NOT a "Lot N"
      // placeholder and NOT the page URL fallback.
      expect(addr).not.toMatch(/^lot\b/i);
      expect(addr).not.toMatch(/^https?:\/\//);
      expect(addr).toMatch(/\d/); // real street addresses carry a house number
      // And it must equal the parsed street exactly (no ternary drift).
      const h = parseHome(html, url);
      expect(addr).toBe(h!.street);
    }
  });

  it("uses only explicit pageType status evidence", () => {
    const file = files[0]!;
    const html = readFileSync(join(FX, file), "utf8");
    const url = urls[file]!;
    expect(parseHome(html, url)?.status).toBe("MOVE_IN_READY");
    expect(parseHome(html.replace(/"pageType":"Move In Ready"/, '"pageType":"Unknown"'), url)?.status).toBeNull();
  });

  it("community names are naturally cased from the dataLayer (never slug-mangled prepositions)", () => {
    // The dead entity-encoded-href regex used to fall back to a slug titlecaser
    // that produced "The Foothills At Blackridge" (wrong). The dataLayer carries
    // the correct case ("…at…"). Assert no community carries a title-cased
    // joining word — proves the dataLayer path, not the slug fallback, is winning.
    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const community = parseHome(html, url)?.community ?? "";
      // Whitespace-delimited so a legitimately-capitalized leading "The " is not
      // flagged — only a MID-NAME title-cased joining word (the slug-mangle bug).
      expect(community, `mis-cased preposition in "${community}" @ ${file}`).not.toMatch(
        /\s(At|Of|In|The|And|On|By|For)\s/,
      );
    }
  });

  it("emits schema-valid community + inventory_home records", () => {
    let homes = 0;
    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const page = mkPage(url, html);
      const { records, errors } = harrisDoyleAdapter.extract(page);
      expect(errors).toEqual([]);
      for (const rec of records) {
        if (rec.entityType === "inventory_home") homes++;
        const result = validatePayload(rec);
        if (!result.ok) {
          // eslint-disable-next-line no-console
          console.error(`validation failed for ${rec.entityType} @ ${url}`, result);
        }
        expect(result.ok).toBe(true);
      }
    }
    expect(homes).toBeGreaterThanOrEqual(13);
  });
});

function mkPage(url: string, html: string) {
  return {
    url,
    retrievedAt: new Date().toISOString(),
    contentType: "text/html",
    body: Buffer.from(html),
    contentHash: "test",
  };
}

function pct(a: number, b: number): string {
  return b ? `${Math.round((100 * a) / b)}%` : "n/a";
}