`collectors/meridian-homes/src/extract.test.ts`

76 lines
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { sha256 } from "@homesonspec/collectors-common";
import { meridianHomesAdapter } from "./index";

const FIXTURE_DIR = join(import.meta.dirname, "../../../fixtures/raw-pages/meridian-homes");
const EXPECTED_PATH = join(import.meta.dirname, "../../../fixtures/expected-records/meridian-homes.json");

interface ExpectedRecord {
  entityType: string;
  canonicalHints?: Record<string, unknown>;
  values: Record<string, unknown>;
}
interface ExpectedFile {
  pages: Record<string, ExpectedRecord[]>;
}

async function extractPage(file: string) {
  const body = await readFile(join(FIXTURE_DIR, file));
  const firstLine = body.toString("utf8").split("\n", 1)[0] ?? "";
  const url = firstLine.match(/source-url:\s*(\S+)/)?.[1] ?? `fixture://${file}`;
  return meridianHomesAdapter.extract({
    url,
    retrievedAt: new Date().toISOString(),
    contentType: "text/html",
    body,
    contentHash: sha256(body),
  });
}

describe("meridian-homes extractor vs hand-authored expected records", async () => {
  const expected: ExpectedFile = JSON.parse(await readFile(EXPECTED_PATH, "utf8"));

  for (const [file, expectedRecords] of Object.entries(expected.pages)) {
    it(`${file} extracts the contracted records exactly`, async () => {
      const { records, errors } = await extractPage(file);
      expect(errors).toEqual([]);
      expect(records).toHaveLength(expectedRecords.length);

      records.forEach((record, i) => {
        const want = expectedRecords[i]!;
        expect(record.entityType).toBe(want.entityType);
        if (want.canonicalHints) {
          for (const [key, value] of Object.entries(want.canonicalHints)) {
            expect(record.canonicalHints[key as keyof typeof record.canonicalHints], `hint ${key}`).toBe(value);
          }
        }
        for (const [field, value] of Object.entries(want.values)) {
          expect(record.fields[field]?.value, `${file} field ${field}`).toEqual(value);
        }
      });
    });
  }

  it("every extracted field carries a sourceUrl and confidence", async () => {
    for (const file of Object.keys(expected.pages)) {
      const { records } = await extractPage(file);
      for (const record of records) {
        for (const [field, envelope] of Object.entries(record.fields)) {
          expect(envelope.sourceUrl, `${file}/${field} sourceUrl`).toMatch(/^https?:\/\/|^fixture:\/\//);
          expect(envelope.confidence, `${file}/${field} confidence`).toBeGreaterThanOrEqual(0);
          expect(envelope.confidence).toBeLessThanOrEqual(1);
        }
      }
    }
  });

  it("null values (missing from source) always have confidence 0 — never guessed", async () => {
    const { records } = await extractPage("03-home-124-silverleaf.html");
    const home = records[0]!;
    expect(home.fields.price?.value).toBeNull();
    expect(home.fields.price?.confidence).toBe(0);
  });
});