`collectors/common/src/adapter.ts`

82 lines
import { createHash } from "node:crypto";
import { readFile, readdir } from "node:fs/promises";
import { join } from "node:path";
import type { ExtractedRecord } from "@homesonspec/schemas";

/**
 * The contract every builder collector implements. v1 ships fixture mode
 * only — live fetching arrives later as a second FetchContext implementation
 * that honors the source registry's rate limits and permissions. Extraction
 * is pure and deterministic: same bytes in, same records out.
 */

export interface RawPage {
  url: string;
  retrievedAt: string; // ISO
  contentType: string;
  body: Buffer;
  contentHash: string; // sha256 hex
}

export interface SourceRegistryEntry {
  key: string;
  collectionMethod: "FEED" | "CRAWL" | "MANUAL" | "FIXTURE" | "SYNTHETIC";
  rateLimitRpm?: number | null;
  mediaRights: "NONE" | "LINK_ONLY" | "LICENSED";
}

export interface FetchContext {
  mode: "fixture" | "live";
  fixtureDir?: string; // fixtures/raw-pages/<key>/
  registry: SourceRegistryEntry;
}

export interface ExtractionOutput {
  records: ExtractedRecord[];
  errors: { url: string; reason: string }[];
}

export interface SourceAdapter {
  /** Must equal the SourceRegistry.key row this adapter collects for. */
  key: string;
  /** Stamped as extractor_version on every evidence + staged record. */
  version: string;
  fetch(ctx: FetchContext): AsyncIterable<RawPage>;
  extract(page: RawPage): ExtractionOutput;
  /**
   * Optional LLM-assist hook (terminology mapping, incentive parsing).
   * v1 is identity. Whatever this returns still faces the deterministic
   * validators — normalization can never bless a record into publishing.
   */
  normalize?(record: ExtractedRecord): Promise<ExtractedRecord>;
}

export function sha256(buffer: Buffer | string): string {
  return createHash("sha256").update(buffer).digest("hex");
}

/**
 * Shared fixture fetcher: yields every .html/.json file in the adapter's
 * fixture directory as a RawPage. The "url" is reconstructed from a
 * `<!-- source-url: ... -->` comment on line 1 when present, else a
 * fixture:// pseudo-URL — keeping evidence traceable even in fixture mode.
 */
export async function* fetchFixtures(ctx: FetchContext): AsyncIterable<RawPage> {
  if (ctx.mode !== "fixture" || !ctx.fixtureDir) {
    throw new Error("fetchFixtures requires mode=fixture and a fixtureDir");
  }
  const files = (await readdir(ctx.fixtureDir)).filter((f) => /\.(html|json)$/.test(f)).sort();
  for (const file of files) {
    const body = await readFile(join(ctx.fixtureDir, file));
    const firstLine = body.toString("utf8").split("\n", 1)[0] ?? "";
    const match = firstLine.match(/source-url:\s*(\S+)/);
    yield {
      url: match?.[1] ?? `fixture://${ctx.registry.key}/${file}`,
      retrievedAt: new Date().toISOString(),
      contentType: file.endsWith(".json") ? "application/json" : "text/html",
      body,
      contentHash: sha256(body),
    };
  }
}