← back to Homesonspec
TK-11125: forward-only spurious-generation guard in extractStage
319168ad20f88b70ecd2db7ad06d9a191a02a298 · 2026-09-02 13:52:12 -0700 · Steve Abrams
Skip minting a new StagedRecord generation + SourceEvidence set when a new
raw snapshot carries byte-identical extracted property data as the latest
generation (only volatile page bytes moved). Compares a canonical payload
hash against the latest prior-snapshot generation, whose payload is already
stored verbatim in StagedRecord.payload — so NO new column and NO migration.
Preserves history: every genuine field change still mints a generation, and
a same-snapshot FORCE_REEXTRACT still updates in place (snapshotId-equality
escape). Guard can only ever skip an exact duplicate, never a real change.
Proven at the DB boundary in payload-guard.itest.ts (5 cases, all green).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016JRL7REtkaBnRrfHk42iYm
Files touched
A apps/workers/src/payload-guard.itest.tsM apps/workers/src/pipeline.ts
Diff
commit 319168ad20f88b70ecd2db7ad06d9a191a02a298
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 2 13:52:12 2026 -0700
TK-11125: forward-only spurious-generation guard in extractStage
Skip minting a new StagedRecord generation + SourceEvidence set when a new
raw snapshot carries byte-identical extracted property data as the latest
generation (only volatile page bytes moved). Compares a canonical payload
hash against the latest prior-snapshot generation, whose payload is already
stored verbatim in StagedRecord.payload — so NO new column and NO migration.
Preserves history: every genuine field change still mints a generation, and
a same-snapshot FORCE_REEXTRACT still updates in place (snapshotId-equality
escape). Guard can only ever skip an exact duplicate, never a real change.
Proven at the DB boundary in payload-guard.itest.ts (5 cases, all green).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016JRL7REtkaBnRrfHk42iYm
---
apps/workers/src/payload-guard.itest.ts | 178 ++++++++++++++++++++++++++++++++
apps/workers/src/pipeline.ts | 33 +++++-
2 files changed, 210 insertions(+), 1 deletion(-)
diff --git a/apps/workers/src/payload-guard.itest.ts b/apps/workers/src/payload-guard.itest.ts
new file mode 100644
index 00000000..8c94df41
--- /dev/null
+++ b/apps/workers/src/payload-guard.itest.ts
@@ -0,0 +1,178 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import { prisma } from "@homesonspec/database";
+import { canonicalKey } from "@homesonspec/shared";
+import type { ExtractedRecord } from "@homesonspec/schemas";
+import type { RawPage, SourceAdapter } from "@homesonspec/collectors-common";
+import { extractStage } from "./pipeline";
+
+/**
+ * Integration: the TK-11125 forward-only spurious-generation guard, proven at
+ * the StagedRecord + SourceEvidence data boundary (e2e-proof R3). A new raw
+ * snapshot with byte-identical extracted data must NOT mint a new generation;
+ * a real field change must; history is preserved throughout.
+ */
+
+const SOURCE_KEY = "payload-guard-fixtures";
+// One physical home — canonicalKey is stable across every snapshot below.
+const HINTS = { builderSlug: "guard-test", communityName: "Guard Bend", address: "1 Guard Way" };
+const KEY = canonicalKey(HINTS);
+
+const fv = (value: unknown, raw: string | null = null) => ({
+ value,
+ raw,
+ evidenceText: raw,
+ sourceUrl: "fixture://guard",
+ confidence: value === null ? 0 : 1,
+});
+
+// A synthetic adapter whose extract() returns whatever record the test set.
+let currentRecord: ExtractedRecord;
+const adapter: SourceAdapter = {
+ key: SOURCE_KEY,
+ version: "guard-test-1",
+ // eslint-disable-next-line require-yield
+ async *fetch() {
+ /* unused: tests call extractStage directly */
+ },
+ extract() {
+ return { records: [currentRecord], errors: [] };
+ },
+};
+
+function record(fields: Record<string, ReturnType<typeof fv>>): ExtractedRecord {
+ return { entityType: "sales_office", canonicalHints: HINTS, fields } as ExtractedRecord;
+}
+
+// A distinct RawSnapshot per call = "volatile page bytes changed".
+let snapCounter = 0;
+async function newSnapshot(sourceId: string): Promise<string> {
+ snapCounter += 1;
+ const contentHash = `guardhash-${snapCounter}-${"0".repeat(50)}`.slice(0, 64);
+ const snap = await prisma.rawSnapshot.create({
+ data: {
+ sourceId,
+ url: "https://guard.test/home/1",
+ retrievedAt: new Date(),
+ contentHash,
+ storagePath: `payload-guard/${contentHash}.json`,
+ contentType: "application/json",
+ httpStatus: 200,
+ },
+ });
+ return snap.id;
+}
+
+const gens = () => prisma.stagedRecord.count({ where: { canonicalKey: KEY } });
+const evidenceRows = async () => {
+ const staged = await prisma.stagedRecord.findMany({
+ where: { canonicalKey: KEY },
+ select: { id: true },
+ });
+ return prisma.sourceEvidence.count({
+ where: { stagedRecordId: { in: staged.map((s) => s.id) } },
+ });
+};
+
+let sourceId: string;
+
+beforeAll(async () => {
+ expect(process.env.DATABASE_URL).toMatch(/homesonspec_test/);
+ // Clean only this test's rows (co-exists with the other itest file).
+ const staged = await prisma.stagedRecord.findMany({
+ where: { canonicalKey: KEY },
+ select: { id: true },
+ });
+ await prisma.sourceEvidence.deleteMany({ where: { stagedRecordId: { in: staged.map((s) => s.id) } } });
+ await prisma.stagedRecord.deleteMany({ where: { canonicalKey: KEY } });
+ await prisma.rawSnapshot.deleteMany({ where: { source: { key: SOURCE_KEY } } });
+ await prisma.sourceRegistry.deleteMany({ where: { key: SOURCE_KEY } });
+ await prisma.builder.deleteMany({ where: { slug: "guard-test" } });
+
+ const builder = await prisma.builder.create({
+ data: { slug: "guard-test", name: "Guard Test Homes", isDemo: true },
+ });
+ const source = await prisma.sourceRegistry.create({
+ data: {
+ key: SOURCE_KEY,
+ name: "Payload Guard (synthetic)",
+ builderId: builder.id,
+ collectionMethod: "SYNTHETIC",
+ mediaRights: "NONE",
+ },
+ });
+ sourceId = source.id;
+});
+
+async function extractOn(snapshotId: string) {
+ const source = await prisma.sourceRegistry.findUniqueOrThrow({ where: { key: SOURCE_KEY } });
+ const page = {
+ url: "https://guard.test/home/1",
+ retrievedAt: new Date().toISOString(),
+ contentType: "application/json",
+ body: Buffer.from("{}"),
+ contentHash: "unused-in-extractStage",
+ } as RawPage;
+ return extractStage(adapter, source, page, snapshotId);
+}
+
+describe("TK-11125 spurious-generation guard", () => {
+ it("case 4a — a brand-new home → exactly 1 generation + evidence", async () => {
+ currentRecord = record({ name: fv("Guard Bend Office"), phone: fv("512-555-0100") });
+ const snap1 = await newSnapshot(sourceId);
+ const { stagedIds } = await extractOn(snap1);
+ expect(stagedIds).toHaveLength(1);
+ expect(await gens()).toBe(1);
+ expect(await evidenceRows()).toBe(2); // name + phone
+ });
+
+ it("case 2 — new snapshot, byte-identical fields → 0 new generations (the fix)", async () => {
+ const evBefore = await evidenceRows();
+ currentRecord = record({ name: fv("Guard Bend Office"), phone: fv("512-555-0100") });
+ const snap2 = await newSnapshot(sourceId); // volatile bytes changed → different snapshot
+ const { stagedIds } = await extractOn(snap2);
+ expect(stagedIds).toHaveLength(0); // SKIPPED
+ expect(await gens()).toBe(1); // still one generation
+ expect(await evidenceRows()).toBe(evBefore); // no fresh evidence set minted
+ });
+
+ it("case 3 — new snapshot, a real field change → exactly 1 new generation", async () => {
+ currentRecord = record({ name: fv("Guard Bend Office"), phone: fv("512-555-0199") }); // phone changed
+ const snap3 = await newSnapshot(sourceId);
+ const { stagedIds } = await extractOn(snap3);
+ expect(stagedIds).toHaveLength(1); // NEW generation
+ expect(await gens()).toBe(2); // history preserved: 2 generations now
+ expect(await evidenceRows()).toBe(4); // gen1 (2) + gen2 (2)
+ });
+
+ it("case 2b — re-seeing the just-changed data on yet another snapshot → 0 new (steady state)", async () => {
+ const gensBefore = await gens();
+ currentRecord = record({ name: fv("Guard Bend Office"), phone: fv("512-555-0199") });
+ const snap4 = await newSnapshot(sourceId);
+ const { stagedIds } = await extractOn(snap4);
+ expect(stagedIds).toHaveLength(0);
+ expect(await gens()).toBe(gensBefore); // no drift once data settles
+ });
+
+ it("case 4b — FORCE_REEXTRACT same snapshot adding a field → updates the row, no new generation", async () => {
+ // Re-extract the SAME latest snapshot with an added field. payloadHash differs
+ // → guard's snapshotId-equality escape lets it fall through to the upsert UPDATE,
+ // preserving today's FORCE_REEXTRACT semantics (row updated in place, not forked).
+ const latest = await prisma.stagedRecord.findFirstOrThrow({
+ where: { canonicalKey: KEY },
+ orderBy: { createdAt: "desc" },
+ select: { id: true, snapshotId: true },
+ });
+ const gensBefore = await gens();
+ currentRecord = record({
+ name: fv("Guard Bend Office"),
+ phone: fv("512-555-0199"),
+ hours: fv("9-5"), // newly backfilled field
+ });
+ const { stagedIds } = await extractOn(latest.snapshotId!);
+ expect(stagedIds).toHaveLength(1);
+ expect(stagedIds[0]).toBe(latest.id); // SAME row updated, not a new generation
+ expect(await gens()).toBe(gensBefore);
+ const ev = await prisma.sourceEvidence.count({ where: { stagedRecordId: latest.id } });
+ expect(ev).toBe(3); // name + phone + hours (evidence replaced for the row)
+ });
+});
diff --git a/apps/workers/src/pipeline.ts b/apps/workers/src/pipeline.ts
index 9287c442..a60248b4 100644
--- a/apps/workers/src/pipeline.ts
+++ b/apps/workers/src/pipeline.ts
@@ -7,7 +7,7 @@ import {
type SourceRegistry,
} from "@homesonspec/database";
import { validatePayload, type ExtractedRecord } from "@homesonspec/schemas";
-import { canonicalKey } from "@homesonspec/shared";
+import { canonicalKey, payloadHash } from "@homesonspec/shared";
import {
ALL_RULES,
runValidation,
@@ -116,6 +116,37 @@ export async function extractStage(
extractorVersion: adapter.version,
status: "EXTRACTED" as const,
};
+ // ── Forward-only spurious-generation guard (TK-11125) ─────────────────
+ // A new RawSnapshot is minted whenever ANY volatile page byte changes (ad
+ // tokens, CSRF nonces, render timestamps). Extracting it would upsert a
+ // brand-new StagedRecord generation + a full fresh SourceEvidence set even
+ // when the actual property data is byte-identical to the latest generation
+ // — that is the evidence firehose this ticket bounds. Skip that spurious
+ // write while preserving every GENUINE change as its own history row.
+ //
+ // Compare a canonical hash of the extracted fields against the latest
+ // generation for this (source, home). StagedRecord.payload already stores
+ // `normalized.fields` verbatim, so the prior hash is recomputed on read —
+ // no schema column, no migration; history is untouched. Any real change
+ // (or a same-snapshot re-extract that adds a field) differs and falls
+ // through below, so the guard can only ever SKIP an exact duplicate.
+ const newPayloadHash = payloadHash(normalized.fields);
+ const latest = await prisma.stagedRecord.findFirst({
+ where: { sourceId: source.id, canonicalKey: key }, // uses @@index([canonicalKey])
+ orderBy: { createdAt: "desc" },
+ select: { snapshotId: true, payload: true },
+ });
+ if (
+ latest &&
+ latest.snapshotId !== snapshotId && // same-snapshot re-extract keeps today's FORCE_REEXTRACT update semantics
+ payloadHash(latest.payload) === newPayloadHash
+ ) {
+ // Identical property data from a different (prior) snapshot → spurious.
+ // Create NO new StagedRecord and NO evidence: the existing generation
+ // already carries this exact data and its published entity is unchanged.
+ continue;
+ }
+
// Upsert on (source, snapshot, home) so re-extracting the same snapshot (e.g.
// FORCE_REEXTRACT to backfill a new field) updates the row instead of piling
// up duplicate staged/evidence rows.
← 671fbb26 TK-11125: add stableStringify + payloadHash canonical hash h
·
back to Homesonspec
·
TK-11125: e2e-proof evidence bundle for the payload guard (R 6a1800df →