`collectors/chafin-communities/src/selftest.test.ts`

165 lines
import { readFileSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { validatePayload } from "@homesonspec/schemas";
import { chafinCommunitiesAdapter, parseCommunityPage } from "./index";

/**
 * Bounded, read-only self-test for the Chafin Communities adapter. Runs
 * `parseCommunityPage` + `chafinCommunitiesAdapter.extract()` over 8 fixtures
 * captured (rate-limited ~1 req/4s, honest UA) from chafincommunities.com on
 * 2026-08-30 across 6 Metro-Atlanta GA counties / 8 distinct communities, and
 * proves:
 *   - facts-only per-field coverage (price may legitimately be null on Pending
 *     homes; geo/sqft are legitimately absent from this DOM section),
 *   - every per-home street address AND mlsid (builderInventoryId) is DISTINCT
 *     across the whole set (proves genuine per-home listings, not a repeated
 *     community/floor-plan aggregate),
 *   - every emitted community + inventory_home record validates against the
 *     published schema.
 *
 * NB: unlike Goodall (one page == one detail home), Chafin publishes its
 * available-home inventory INLINE on each community page — so each fixture is a
 * community page carrying N home listings, and each yields 1 community record +
 * N inventory_home records.
 */

const HERE = dirname(fileURLToPath(import.meta.url));
const FX = join(HERE, "..", "fixtures", "homes");

function urlMap(): Record<string, string> {
  const tsv = readFileSync(join(FX, "_urls.tsv"), "utf8");
  const out: Record<string, string> = {};
  for (const line of tsv.split(/\r?\n/)) {
    const [file, url] = line.split("\t");
    if (file && url) out[file.trim()] = url.trim();
  }
  return out;
}

describe("chafin-communities adapter — coverage + per-home + schema", () => {
  const files = readdirSync(FX)
    .filter((f) => f.endsWith(".html"))
    .sort();
  const urls = urlMap();

  it("parses per-home facts with high coverage and distinct addresses + ids", () => {
    let n = 0;
    const cov = {
      price: 0,
      beds: 0,
      baths: 0,
      sqft: 0,
      address: 0,
      status: 0,
      id: 0,
      geo: 0,
    };
    const addresses = new Set<string>();
    const ids = new Set<string>();

    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const parsed = parseCommunityPage(html, url);
      expect(parsed, `parseCommunityPage returned null for ${file}`).not.toBeNull();
      for (const h of parsed!.homes) {
        n++;
        if (h.street) {
          cov.address++;
          addresses.add(`${h.street}, ${h.city ?? ""}, ${h.state ?? ""} ${h.zip ?? ""}`);
        }
        if (h.builderInventoryId) {
          cov.id++;
          ids.add(h.builderInventoryId);
        }
        if (h.price != null) cov.price++;
        if (h.beds != null) cov.beds++;
        if (h.baths != null) cov.baths++;
        if (h.status) cov.status++;
        // sqft + geo are legitimately ABSENT from the available-homes DOM
        // section — asserted at 0, an honest null (not a parse failure).
      }
    }

    // eslint-disable-next-line no-console
    console.log(
      `\n[chafin-communities self-test] homes=${n} distinctAddresses=${addresses.size} distinctIds=${ids.size}\n` +
        `  price=${pct(cov.price, n)}  beds=${pct(cov.beds, n)}  baths=${pct(cov.baths, n)}  ` +
        `address=${pct(cov.address, n)}  id=${pct(cov.id, n)}  status=${pct(cov.status, n)}  ` +
        `sqft=${pct(cov.sqft, n)}  geo=${pct(cov.geo, n)}`,
    );

    // Enough homes to be a meaningful test.
    expect(n).toBeGreaterThanOrEqual(40);

    // Genuinely per-home: one distinct street address AND one distinct mlsid
    // per parsed home (proves real inventory listings, not a repeated aggregate).
    expect(addresses.size).toBe(n);
    expect(ids.size).toBe(n);

    // Facts always present on every listing: address, mlsid, beds, baths, status.
    expect(cov.address).toBe(n);
    expect(cov.id).toBe(n);
    expect(cov.beds).toBe(n);
    expect(cov.baths).toBe(n);
    expect(cov.status).toBe(n);

    // Price is published for Active/available homes but NOT for Pending ones —
    // an honest null. Assert coverage without requiring 100%.
    expect(cov.price / n).toBeGreaterThanOrEqual(0.5);

    // sqft + geo are honest nulls for this source's inventory DOM: assert we did
    // NOT fabricate them (coverage stays 0), but do NOT fail on the null.
    expect(cov.sqft).toBe(0);
    expect(cov.geo).toBe(0);
  });

  it("emits schema-valid community + inventory_home records", () => {
    let homes = 0;
    let communities = 0;
    for (const file of files) {
      const html = readFileSync(join(FX, file), "utf8");
      const url = urls[file] ?? `fixture://${file}`;
      const page = {
        url,
        retrievedAt: new Date().toISOString(),
        contentType: "text/html",
        body: Buffer.from(html),
        contentHash: "test",
      };
      const { records, errors } = chafinCommunitiesAdapter.extract(page);
      expect(errors).toEqual([]);
      for (const rec of records) {
        if (rec.entityType === "inventory_home") {
          homes++;
          // Guard the DEDUPE KEY, not just the parsed struct: every home here
          // has a real street (address coverage is 100%), so the canonical
          // address hint must be that street — never a "Lot X" fallback. This
          // is the assertion that catches the ?? / ?: precedence bug that made
          // every home collide under "Lot X" (TK-10487 contrarian catch).
          const canonAddr = rec.canonicalHints?.address ?? "";
          expect(canonAddr, `canonicalHints.address fell back to "${canonAddr}" @ ${url}`).not.toMatch(
            /^Lot\s/i,
          );
          expect(canonAddr).toMatch(/\d/); // a real street carries a house number
        }
        if (rec.entityType === "community") communities++;
        const result = validatePayload(rec);
        if (!result.ok) {
          // eslint-disable-next-line no-console
          console.error(`validation failed for ${rec.entityType} @ ${url}`, result);
        }
        expect(result.ok).toBe(true);
      }
    }
    expect(communities).toBe(files.length);
    expect(homes).toBeGreaterThanOrEqual(40);
  });
});

function pct(a: number, b: number): string {
  return b ? `${Math.round((100 * a) / b)}%` : "n/a";
}