← back to Homesonspec
worker pipeline (fetch/extract/validate/publish) + CLI + synthetic seed; fixture pipeline e2e green, idempotent re-run, dup rule scoped to zip
75e00bb8a399328d3dae20a6d883ec553fc3c8af · 2026-07-22 11:26:56 -0700 · Steve Abrams
Files touched
A apps/workers/src/cli.tsA apps/workers/src/index.tsA apps/workers/src/pipeline.tsA packages/database/prisma/seed.tsM packages/validation/src/rules.test.tsM packages/validation/src/rules.tsM packages/validation/src/types.ts
Diff
commit 75e00bb8a399328d3dae20a6d883ec553fc3c8af
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 22 11:26:56 2026 -0700
worker pipeline (fetch/extract/validate/publish) + CLI + synthetic seed; fixture pipeline e2e green, idempotent re-run, dup rule scoped to zip
---
apps/workers/src/cli.ts | 31 +++
apps/workers/src/index.ts | 25 ++
apps/workers/src/pipeline.ts | 417 ++++++++++++++++++++++++++++++
packages/database/prisma/seed.ts | 465 ++++++++++++++++++++++++++++++++++
packages/validation/src/rules.test.ts | 18 +-
packages/validation/src/rules.ts | 9 +-
packages/validation/src/types.ts | 2 +-
7 files changed, 962 insertions(+), 5 deletions(-)
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
new file mode 100644
index 0000000..a66ca17
--- /dev/null
+++ b/apps/workers/src/cli.ts
@@ -0,0 +1,31 @@
+import { prisma } from "@spechomes/database";
+import { meridianHomesAdapter } from "@spechomes/collector-meridian-homes";
+import { runPipeline } from "./pipeline.js";
+
+/**
+ * Direct pipeline runner — no queue required.
+ * pnpm --filter @spechomes/workers pipeline -- --adapter=meridian-homes-fixtures
+ */
+const ADAPTERS = {
+ [meridianHomesAdapter.key]: meridianHomesAdapter,
+};
+
+async function main() {
+ const arg = process.argv.find((a) => a.startsWith("--adapter="));
+ const key = arg?.split("=")[1] ?? meridianHomesAdapter.key;
+ const adapter = ADAPTERS[key];
+ if (!adapter) {
+ console.error(`Unknown adapter "${key}". Known: ${Object.keys(ADAPTERS).join(", ")}`);
+ process.exit(1);
+ }
+ console.log(`Running pipeline for ${key} (extractor v${adapter.version})…`);
+ const summary = await runPipeline(adapter);
+ console.log(JSON.stringify(summary, null, 2));
+}
+
+main()
+ .catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ })
+ .finally(() => prisma.$disconnect());
diff --git a/apps/workers/src/index.ts b/apps/workers/src/index.ts
new file mode 100644
index 0000000..44c79e4
--- /dev/null
+++ b/apps/workers/src/index.ts
@@ -0,0 +1,25 @@
+import { work } from "@spechomes/shared";
+import { meridianHomesAdapter } from "@spechomes/collector-meridian-homes";
+import { runPipeline } from "./pipeline.js";
+
+/**
+ * Queue-backed worker daemon (pg-boss). Stages stay pure functions; this
+ * process merely subscribes to job names and invokes them. Scheduling of
+ * refresh jobs per source-registry intervals lands with live sources.
+ */
+const ADAPTERS = { [meridianHomesAdapter.key]: meridianHomesAdapter };
+
+async function main() {
+ await work<{ adapterKey: string }>("pipeline.run", async ({ adapterKey }) => {
+ const adapter = ADAPTERS[adapterKey];
+ if (!adapter) throw new Error(`unknown adapter ${adapterKey}`);
+ const summary = await runPipeline(adapter);
+ console.log(`[pipeline.run] ${adapterKey}:`, JSON.stringify(summary));
+ });
+ console.log("workers: subscribed to pipeline.run");
+}
+
+main().catch((error) => {
+ console.error(error);
+ process.exit(1);
+});
diff --git a/apps/workers/src/pipeline.ts b/apps/workers/src/pipeline.ts
new file mode 100644
index 0000000..61491e3
--- /dev/null
+++ b/apps/workers/src/pipeline.ts
@@ -0,0 +1,417 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+ prisma,
+ type EntityType,
+ type Prisma,
+ type SourceRegistry,
+} from "@spechomes/database";
+import { validatePayload, type ExtractedRecord } from "@spechomes/schemas";
+import { canonicalKey, normalizeAddress } from "@spechomes/shared";
+import {
+ ALL_RULES,
+ runValidation,
+ VALIDATOR_VERSION,
+ type RuleDb,
+ type StagedCandidate,
+} from "@spechomes/validation";
+import type { RawPage, SourceAdapter } from "@spechomes/collectors-common";
+
+/**
+ * Pipeline stages. Each is a pure-ish function over (adapter, db) — the
+ * queue merely transports invocations; the CLI and tests call these
+ * directly. Raw and published tables stay strictly separate: nothing here
+ * writes a published table except publishStagedRecord, and that only after
+ * a deterministic `publishable` verdict (or explicit human approval).
+ */
+
+// Anchor to the repo root — the CLI may run with cwd anywhere in the workspace.
+const REPO_ROOT = join(import.meta.dirname, "../../..");
+const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(REPO_ROOT, "var/snapshots");
+
+const ENTITY_TYPE_MAP: Record<ExtractedRecord["entityType"], EntityType> = {
+ community: "COMMUNITY",
+ floor_plan: "FLOOR_PLAN",
+ inventory_home: "INVENTORY_HOME",
+ incentive: "INCENTIVE",
+ sales_office: "SALES_OFFICE",
+ lot: "LOT",
+};
+
+// ── Stage 1: fetch → raw snapshots ─────────────────────────────────────
+
+export async function fetchStage(adapter: SourceAdapter, source: SourceRegistry) {
+ const pages: { page: RawPage; snapshotId: string; changed: boolean }[] = [];
+ const dir = join(SNAPSHOT_DIR, source.key);
+ await mkdir(dir, { recursive: true });
+
+ for await (const page of adapter.fetch({
+ mode: "fixture",
+ fixtureDir: join(REPO_ROOT, "fixtures/raw-pages", adapter.key.replace(/-fixtures$/, "")),
+ registry: {
+ key: source.key,
+ collectionMethod: source.collectionMethod,
+ rateLimitRpm: source.rateLimitRpm,
+ mediaRights: source.mediaRights,
+ },
+ })) {
+ const existing = await prisma.rawSnapshot.findUnique({
+ where: { sourceId_contentHash: { sourceId: source.id, contentHash: page.contentHash } },
+ });
+ if (existing) {
+ // Same bytes already snapshotted — source unchanged for this url.
+ pages.push({ page, snapshotId: existing.id, changed: false });
+ continue;
+ }
+ const storagePath = join(dir, `${page.contentHash}.html`);
+ await writeFile(storagePath, page.body);
+ const snapshot = await prisma.rawSnapshot.create({
+ data: {
+ sourceId: source.id,
+ url: page.url,
+ retrievedAt: new Date(page.retrievedAt),
+ contentHash: page.contentHash,
+ storagePath,
+ contentType: page.contentType,
+ httpStatus: 200,
+ },
+ });
+ pages.push({ page, snapshotId: snapshot.id, changed: true });
+ }
+ return pages;
+}
+
+// ── Stage 2: extract → staged records + evidence ───────────────────────
+
+export async function extractStage(
+ adapter: SourceAdapter,
+ source: SourceRegistry,
+ page: RawPage,
+ snapshotId: string,
+) {
+ const { records, errors } = adapter.extract(page);
+ const stagedIds: string[] = [];
+
+ for (const record of records) {
+ const normalized = adapter.normalize ? await adapter.normalize(record) : record;
+
+ const shape = validatePayload(normalized);
+ if (!shape.ok) {
+ // Shape failure is an extraction bug, not bad data — log, don't stage.
+ errors.push({ url: page.url, reason: `payload schema: ${shape.issues.join("; ")}` });
+ continue;
+ }
+
+ const key = canonicalKey(normalized.canonicalHints);
+ const staged = await prisma.stagedRecord.create({
+ data: {
+ sourceId: source.id,
+ snapshotId,
+ entityType: ENTITY_TYPE_MAP[normalized.entityType],
+ canonicalKey: key,
+ payload: normalized.fields as unknown as Prisma.InputJsonValue,
+ extractorVersion: adapter.version,
+ status: "EXTRACTED",
+ },
+ });
+ stagedIds.push(staged.id);
+
+ await prisma.sourceEvidence.createMany({
+ data: Object.entries(normalized.fields).map(([field, envelope]) => ({
+ stagedRecordId: staged.id,
+ field,
+ sourceUrl: envelope.sourceUrl,
+ retrievedAt: new Date(page.retrievedAt),
+ contentHash: page.contentHash,
+ extractorVersion: adapter.version,
+ rawValue: envelope.raw,
+ normalizedValue: envelope.value === null ? null : String(envelope.value),
+ evidenceText: envelope.evidenceText,
+ confidence: envelope.confidence,
+ })),
+ });
+ }
+ return { stagedIds, errors };
+}
+
+// ── Stage 3: validate → ValidationEvents + verdict ─────────────────────
+
+const ruleDb: RuleDb = {
+ async activeHomeExistsAtAddress(normalizedAddress, zip, excludeCanonicalKey) {
+ const found = await prisma.inventoryHome.findFirst({
+ where: {
+ normalizedAddress,
+ ...(zip ? { zip } : {}),
+ status: "PUBLISHED",
+ canonicalKey: { not: excludeCanonicalKey },
+ },
+ select: { id: true },
+ });
+ return found !== null;
+ },
+ async publishedPriceForCanonicalKey(key) {
+ const home = await prisma.inventoryHome.findUnique({
+ where: { canonicalKey: key },
+ select: { price: true },
+ });
+ return home?.price ? Number(home.price) : null;
+ },
+ async communityExists(name, builderSlug) {
+ const found = await prisma.community.findFirst({
+ where: { name, builder: { slug: builderSlug } },
+ select: { id: true },
+ });
+ return found !== null;
+ },
+ async builderExists(slug) {
+ const found = await prisma.builder.findUnique({ where: { slug }, select: { id: true } });
+ return found !== null;
+ },
+};
+
+const REVERSE_ENTITY_MAP: Record<string, ExtractedRecord["entityType"]> = {
+ COMMUNITY: "community",
+ FLOOR_PLAN: "floor_plan",
+ INVENTORY_HOME: "inventory_home",
+ INCENTIVE: "incentive",
+ SALES_OFFICE: "sales_office",
+ LOT: "lot",
+};
+
+export async function validateStage(stagedRecordId: string, sourceKey: string) {
+ const staged = await prisma.stagedRecord.findUniqueOrThrow({ where: { id: stagedRecordId } });
+
+ const candidate: StagedCandidate = {
+ entityType: REVERSE_ENTITY_MAP[staged.entityType]!,
+ canonicalKey: staged.canonicalKey,
+ canonicalHints: reconstructHints(staged.payload as Record<string, { value: unknown }>, sourceKey),
+ fields: staged.payload as StagedCandidate["fields"],
+ };
+
+ const { events, verdict } = await runValidation(candidate, ALL_RULES, {
+ db: ruleDb,
+ sourceKey,
+ now: new Date(),
+ });
+
+ await prisma.validationEvent.createMany({
+ data: events.map((event) => ({
+ stagedRecordId: staged.id,
+ ruleId: event.ruleId,
+ severity: event.severity,
+ passed: event.passed,
+ message: event.message,
+ details: (event.details ?? undefined) as Prisma.InputJsonValue | undefined,
+ validatorVersion: VALIDATOR_VERSION,
+ })),
+ });
+
+ const status =
+ verdict === "publishable" ? "VALIDATED" : verdict === "needs_review" ? "NEEDS_REVIEW" : "REJECTED";
+ await prisma.stagedRecord.update({ where: { id: staged.id }, data: { status } });
+
+ if (verdict === "needs_review") {
+ const failing = events.filter((e) => !e.passed && e.severity === "review");
+ await prisma.reviewItem.create({
+ data: {
+ stagedRecordId: staged.id,
+ reason: failing.map((e) => `${e.ruleId}: ${e.message}`).join(" | ") || "review required",
+ },
+ });
+ }
+ return { verdict, status };
+}
+
+/** Hints were computed at extract time from the record itself; rebuild them from the payload. */
+function reconstructHints(
+ payload: Record<string, { value: unknown }>,
+ _sourceKey: string,
+): StagedCandidate["canonicalHints"] {
+ const get = (field: string) => (payload[field]?.value ?? null) as string | null;
+ const num = (field: string) => (payload[field]?.value ?? null) as number | null;
+ return {
+ builderSlug: "meridian-homes", // v1: single fixture builder; live adapters carry this in payload
+ communityName: get("communityName") ?? "Cedar Bend",
+ address: get("street"),
+ lotNumber: get("lotNumber"),
+ builderInventoryId: get("builderInventoryId"),
+ lat: num("lat"),
+ lon: num("lon"),
+ planName: get("planName"),
+ };
+}
+
+// ── Stage 4: publish (the ONLY writer of published tables) ─────────────
+
+export async function publishStagedRecord(stagedRecordId: string) {
+ const staged = await prisma.stagedRecord.findUniqueOrThrow({
+ where: { id: stagedRecordId },
+ include: { source: true },
+ });
+ if (staged.status !== "VALIDATED" && staged.status !== "APPROVED") {
+ throw new Error(
+ `refusing to publish staged record ${staged.id} with status ${staged.status} — only VALIDATED or human-APPROVED records publish`,
+ );
+ }
+
+ const payload = staged.payload as Record<string, { value: unknown }>;
+ const value = <T>(field: string): T | null => (payload[field]?.value ?? null) as T | null;
+
+ let publishedEntityId: string | null = null;
+
+ if (staged.entityType === "INVENTORY_HOME") {
+ const builder = await prisma.builder.findUniqueOrThrow({ where: { slug: "meridian-homes" } });
+ const community = await prisma.community.findFirstOrThrow({
+ where: { name: "Cedar Bend", builderId: builder.id },
+ });
+ const priorPrice = await prisma.inventoryHome.findUnique({
+ where: { canonicalKey: staged.canonicalKey },
+ select: { price: true },
+ });
+
+ const price = value<number>("price");
+ const street = value<string>("street");
+ const home = await prisma.inventoryHome.upsert({
+ where: { canonicalKey: staged.canonicalKey },
+ create: {
+ canonicalKey: staged.canonicalKey,
+ communityId: community.id,
+ builderId: builder.id,
+ builderInventoryId: value<string>("builderInventoryId"),
+ street,
+ city: value<string>("city") ?? community.city,
+ state: value<string>("state") ?? community.state,
+ zip: value<string>("zip") ?? community.zip,
+ normalizedAddress: street ? normalizeAddress(street) : null,
+ lotNumber: value<string>("lotNumber"),
+ lat: value<number>("lat"),
+ lon: value<number>("lon"),
+ price,
+ beds: value<number>("beds"),
+ bathsTotal: value<number>("bathsTotal"),
+ sqft: value<number>("sqft"),
+ stories: value<number>("stories"),
+ garageSpaces: value<number>("garageSpaces"),
+ homeType: (value<string>("homeType") as never) ?? "SINGLE_FAMILY",
+ constructionStatus: (value<string>("constructionStatus") as never) ?? "UNDER_CONSTRUCTION",
+ estCompletionDate: value<string>("estCompletionDate")
+ ? new Date(value<string>("estCompletionDate")!)
+ : null,
+ status: "PUBLISHED",
+ freshness: "FRESH",
+ verificationLabel: "DEMONSTRATION", // fixture source — never a production claim
+ lastVerifiedAt: new Date(),
+ sourceId: staged.sourceId,
+ sourceUrl: payload.street?.value ? (payload.street as { sourceUrl?: string }).sourceUrl ?? null : null,
+ isDemo: true,
+ },
+ update: {
+ previousPrice: priorPrice?.price ?? undefined,
+ price,
+ lastVerifiedAt: new Date(),
+ freshness: "FRESH",
+ updatedAt: new Date(),
+ },
+ });
+ publishedEntityId = home.id;
+
+ if (priorPrice?.price && price !== null && Number(priorPrice.price) !== price) {
+ await prisma.changeEvent.create({
+ data: {
+ entityType: "INVENTORY_HOME",
+ entityId: home.id,
+ field: "price",
+ oldValue: String(priorPrice.price),
+ newValue: String(price),
+ sourceId: staged.sourceId,
+ },
+ });
+ }
+ } else if (staged.entityType === "INCENTIVE") {
+ const builder = await prisma.builder.findUniqueOrThrow({ where: { slug: "meridian-homes" } });
+ const community = await prisma.community.findFirstOrThrow({
+ where: { name: "Cedar Bend", builderId: builder.id },
+ });
+ const expiresAt = value<string>("expiresAt");
+ const incentive = await prisma.incentive.create({
+ data: {
+ scope: "COMMUNITY",
+ builderId: builder.id,
+ communityId: community.id,
+ title: value<string>("title") ?? "Untitled offer",
+ description: value<string>("description"),
+ valueUsd: value<number>("valueUsd"),
+ expiresAt: expiresAt ? new Date(expiresAt) : null,
+ // Spec rule: incentives need an expiration OR an explicit label.
+ evergreenLabel: expiresAt ? null : "expiration not provided",
+ sourceId: staged.sourceId,
+ isDemo: true,
+ },
+ });
+ publishedEntityId = incentive.id;
+ } else if (staged.entityType === "COMMUNITY") {
+ // Fixture communities are pre-seeded (FK target); publishing refreshes verification only.
+ const community = await prisma.community.findFirst({ where: { name: value<string>("name") ?? "" } });
+ publishedEntityId = community?.id ?? null;
+ }
+
+ await prisma.stagedRecord.update({
+ where: { id: staged.id },
+ data: { status: "PUBLISHED", publishedEntityId },
+ });
+ // Re-point evidence at the published entity so consumer pages can cite it.
+ if (publishedEntityId) {
+ await prisma.sourceEvidence.updateMany({
+ where: { stagedRecordId: staged.id },
+ data: { entityType: staged.entityType, entityId: publishedEntityId },
+ });
+ }
+ return publishedEntityId;
+}
+
+// ── Orchestration: run every stage for one adapter ─────────────────────
+
+export async function runPipeline(adapter: SourceAdapter) {
+ const source = await prisma.sourceRegistry.findUniqueOrThrow({ where: { key: adapter.key } });
+ if (!source.active) throw new Error(`source ${source.key} is inactive — refusing to run`);
+
+ const summary = {
+ pages: 0,
+ changed: 0,
+ staged: 0,
+ published: 0,
+ needsReview: 0,
+ rejected: 0,
+ errors: [] as { url: string; reason: string }[],
+ };
+
+ const pages = await fetchStage(adapter, source);
+ summary.pages = pages.length;
+ summary.changed = pages.filter((p) => p.changed).length;
+
+ for (const { page, snapshotId, changed } of pages) {
+ if (!changed) continue; // unchanged bytes → nothing new to extract
+ const { stagedIds, errors } = await extractStage(adapter, source, page, snapshotId);
+ summary.errors.push(...errors);
+ summary.staged += stagedIds.length;
+
+ for (const stagedId of stagedIds) {
+ const { verdict } = await validateStage(stagedId, source.key);
+ if (verdict === "publishable") {
+ await publishStagedRecord(stagedId);
+ summary.published += 1;
+ } else if (verdict === "needs_review") {
+ summary.needsReview += 1;
+ } else {
+ summary.rejected += 1;
+ }
+ }
+ }
+
+ await prisma.sourceRegistry.update({
+ where: { id: source.id },
+ data: { health: summary.errors.length > 0 ? "DEGRADED" : "HEALTHY" },
+ });
+
+ return summary;
+}
diff --git a/packages/database/prisma/seed.ts b/packages/database/prisma/seed.ts
new file mode 100644
index 0000000..f317885
--- /dev/null
+++ b/packages/database/prisma/seed.ts
@@ -0,0 +1,465 @@
+import { prisma } from "../src/index.js";
+import { canonicalKey, normalizeAddress } from "../../shared/src/canonical.js";
+
+/**
+ * Synthetic demonstration inventory — 3 fake-but-realistic markets.
+ * Every row is isDemo: true with verificationLabel DEMONSTRATION.
+ * Real zip prefixes / coordinates are used so the validators PASS —
+ * but no builder, community, address, or home here is real.
+ *
+ * Meridian's Cedar Bend inventory homes are NOT seeded here — they arrive
+ * through the fixture pipeline (pnpm pipeline) so their evidence rows are
+ * pipeline-real. This seed creates the FK targets (builder, community,
+ * source registry) the pipeline needs.
+ */
+
+// Deterministic LCG so re-seeding is stable.
+let seedState = 42;
+function rand(): number {
+ seedState = (seedState * 1664525 + 1013904223) % 2 ** 32;
+ return seedState / 2 ** 32;
+}
+function pick<T>(items: T[]): T {
+ return items[Math.floor(rand() * items.length)]!;
+}
+function between(min: number, max: number): number {
+ return min + rand() * (max - min);
+}
+
+const DEMO_SOURCE_URL = "https://demo.spechomes.local/synthetic";
+
+interface MarketSpec {
+ metro: string;
+ state: string;
+ county: string;
+ schoolDistricts: string[];
+ cities: { name: string; zip: string; lat: number; lon: number }[];
+ priceBand: [number, number];
+}
+
+const MARKETS: Record<string, MarketSpec> = {
+ austin: {
+ metro: "Austin-Round Rock",
+ state: "TX",
+ county: "Williamson County",
+ schoolDistricts: ["Leander ISD", "Round Rock ISD", "Georgetown ISD"],
+ cities: [
+ { name: "Leander", zip: "78641", lat: 30.5788, lon: -97.8531 },
+ { name: "Cedar Park", zip: "78613", lat: 30.5052, lon: -97.8203 },
+ { name: "Georgetown", zip: "78628", lat: 30.6333, lon: -97.6772 },
+ { name: "Round Rock", zip: "78665", lat: 30.5488, lon: -97.6259 },
+ ],
+ priceBand: [330_000, 640_000],
+ },
+ phoenix: {
+ metro: "Phoenix-Mesa",
+ state: "AZ",
+ county: "Maricopa County",
+ schoolDistricts: ["Gilbert Public Schools", "Queen Creek USD", "Peoria USD"],
+ cities: [
+ { name: "Mesa", zip: "85212", lat: 33.3125, lon: -111.6354 },
+ { name: "Queen Creek", zip: "85142", lat: 33.2487, lon: -111.6343 },
+ { name: "Buckeye", zip: "85326", lat: 33.3703, lon: -112.5838 },
+ { name: "Peoria", zip: "85383", lat: 33.7134, lon: -112.2765 },
+ ],
+ priceBand: [360_000, 720_000],
+ },
+ raleigh: {
+ metro: "Raleigh-Cary",
+ state: "NC",
+ county: "Wake County",
+ schoolDistricts: ["Wake County Public Schools"],
+ cities: [
+ { name: "Cary", zip: "27513", lat: 35.7915, lon: -78.7811 },
+ { name: "Apex", zip: "27502", lat: 35.7327, lon: -78.8503 },
+ { name: "Fuquay-Varina", zip: "27526", lat: 35.5843, lon: -78.8 },
+ { name: "Wake Forest", zip: "27587", lat: 35.9799, lon: -78.5097 },
+ ],
+ priceBand: [340_000, 780_000],
+ },
+};
+
+const PLAN_NAMES = [
+ "The Juniper", "The Silverleaf", "The Bracken", "The Laurel", "The Cypress",
+ "The Marigold", "The Saguaro", "The Ocotillo", "The Palo Verde", "The Dogwood",
+ "The Magnolia", "The Longleaf",
+];
+
+const STREET_NAMES = [
+ "Juniper Draw", "Silverleaf Pass", "Brackens Crossing", "Laurel Bend", "Cypress Hollow",
+ "Marigold Way", "Saguaro Vista", "Ocotillo Trail", "Palo Verde Lane", "Dogwood Court",
+ "Magnolia Grove", "Longleaf Circle", "Quarry Ridge", "Summit Draw", "Prairie Clover",
+];
+
+async function main() {
+ console.log("Seeding synthetic demonstration inventory…");
+
+ // ── Source registry ──────────────────────────────────────────────
+ const syntheticSource = await prisma.sourceRegistry.upsert({
+ where: { key: "synthetic-demo" },
+ update: {},
+ create: {
+ key: "synthetic-demo",
+ name: "Synthetic demonstration data",
+ collectionMethod: "SYNTHETIC",
+ mediaRights: "NONE",
+ notes: "Generated demo inventory. Not real homes, builders, or offers.",
+ },
+ });
+
+ // ── Builders ─────────────────────────────────────────────────────
+ const builderSpecs = [
+ { slug: "meridian-homes", name: "Meridian Homes", coverage: "TX, AZ, NC" },
+ { slug: "bluebonnet-builders", name: "Bluebonnet Builders", coverage: "Central Texas" },
+ { slug: "sunridge-communities", name: "SunRidge Communities", coverage: "AZ desert metros" },
+ { slug: "harborline-homes", name: "Harborline Homes", coverage: "Southeast" },
+ ];
+ const builders: Record<string, { id: string }> = {};
+ for (const spec of builderSpecs) {
+ builders[spec.slug] = await prisma.builder.upsert({
+ where: { slug: spec.slug },
+ update: {},
+ create: {
+ slug: spec.slug,
+ name: spec.name,
+ legalName: `${spec.name} LLC (fictional)`,
+ websiteUrl: `https://fixtures.${spec.slug.replace(/-/g, "")}.example`,
+ coverageArea: spec.coverage,
+ logoRightsStatus: "none",
+ isDemo: true,
+ },
+ });
+ }
+
+ // Fixture source for the Meridian pipeline (FK target for pnpm pipeline).
+ await prisma.sourceRegistry.upsert({
+ where: { key: "meridian-homes-fixtures" },
+ update: {},
+ create: {
+ key: "meridian-homes-fixtures",
+ name: "Meridian Homes (fixture pages)",
+ builderId: builders["meridian-homes"]!.id,
+ baseUrl: "https://fixtures.meridianhomes.example",
+ collectionMethod: "FIXTURE",
+ mediaRights: "NONE",
+ freshIntervalHours: 24,
+ agingIntervalHours: 72,
+ notes: "Local fixture HTML — demonstration pipeline source. No live fetching.",
+ },
+ });
+
+ // ── Divisions ────────────────────────────────────────────────────
+ const divisionSpecs: [string, string, string, string[]][] = [
+ ["meridian-homes", "Meridian Texas", "Central Texas", ["TX"]],
+ ["meridian-homes", "Meridian Arizona", "Phoenix Valley", ["AZ"]],
+ ["meridian-homes", "Meridian Carolinas", "Research Triangle", ["NC"]],
+ ["bluebonnet-builders", "Bluebonnet Central", "Central Texas", ["TX"]],
+ ["sunridge-communities", "SunRidge Desert", "Phoenix Valley", ["AZ"]],
+ ["harborline-homes", "Harborline Southeast", "Carolinas", ["NC", "TX", "AZ"]],
+ ];
+ const divisions: Record<string, string> = {};
+ for (const [builderSlug, name, region, states] of divisionSpecs) {
+ const existing = await prisma.division.findFirst({
+ where: { name, builderId: builders[builderSlug]!.id },
+ });
+ const division =
+ existing ??
+ (await prisma.division.create({
+ data: { builderId: builders[builderSlug]!.id, name, region, states },
+ }));
+ divisions[name] = division.id;
+ }
+
+ // ── Communities (12 = 4 per market) ──────────────────────────────
+ interface CommunitySpec {
+ slug: string;
+ name: string;
+ builderSlug: string;
+ division: string;
+ market: keyof typeof MARKETS;
+ cityIndex: number;
+ ageRestricted?: boolean;
+ hoa?: number | null;
+ }
+ const communitySpecs: CommunitySpec[] = [
+ // Austin
+ { slug: "cedar-bend", name: "Cedar Bend", builderSlug: "meridian-homes", division: "Meridian Texas", market: "austin", cityIndex: 0, hoa: 65 },
+ { slug: "bluebonnet-ridge", name: "Bluebonnet Ridge", builderSlug: "bluebonnet-builders", division: "Bluebonnet Central", market: "austin", cityIndex: 1, hoa: 45 },
+ { slug: "mesquite-flats", name: "Mesquite Flats", builderSlug: "bluebonnet-builders", division: "Bluebonnet Central", market: "austin", cityIndex: 2, hoa: null },
+ { slug: "harpers-crossing", name: "Harper's Crossing", builderSlug: "harborline-homes", division: "Harborline Southeast", market: "austin", cityIndex: 3, hoa: 80 },
+ // Phoenix
+ { slug: "sol-mesa-vista", name: "Sol Mesa Vista", builderSlug: "sunridge-communities", division: "SunRidge Desert", market: "phoenix", cityIndex: 0, hoa: 95 },
+ { slug: "agave-trails", name: "Agave Trails", builderSlug: "sunridge-communities", division: "SunRidge Desert", market: "phoenix", cityIndex: 1, ageRestricted: true, hoa: 210 },
+ { slug: "copper-sky", name: "Copper Sky", builderSlug: "meridian-homes", division: "Meridian Arizona", market: "phoenix", cityIndex: 2, hoa: 70 },
+ { slug: "dune-crest", name: "Dune Crest", builderSlug: "harborline-homes", division: "Harborline Southeast", market: "phoenix", cityIndex: 3, hoa: null },
+ // Raleigh
+ { slug: "longleaf-preserve", name: "Longleaf Preserve", builderSlug: "harborline-homes", division: "Harborline Southeast", market: "raleigh", cityIndex: 0, hoa: 110 },
+ { slug: "pinehollow", name: "Pinehollow", builderSlug: "meridian-homes", division: "Meridian Carolinas", market: "raleigh", cityIndex: 1, hoa: 85 },
+ { slug: "carolina-fern", name: "Carolina Fern", builderSlug: "bluebonnet-builders", division: "Bluebonnet Central", market: "raleigh", cityIndex: 2, hoa: 60 },
+ { slug: "wakefield-commons", name: "Wakefield Commons", builderSlug: "sunridge-communities", division: "SunRidge Desert", market: "raleigh", cityIndex: 3, hoa: 75 },
+ ];
+
+ const communities: { id: string; spec: CommunitySpec; city: MarketSpec["cities"][number]; market: MarketSpec }[] = [];
+ for (const spec of communitySpecs) {
+ const market = MARKETS[spec.market]!;
+ const city = market.cities[spec.cityIndex]!;
+ const lat = Number((city.lat + between(-0.02, 0.02)).toFixed(6));
+ const lon = Number((city.lon + between(-0.02, 0.02)).toFixed(6));
+ const community = await prisma.community.upsert({
+ where: { slug: spec.slug },
+ update: {},
+ create: {
+ builderId: builders[spec.builderSlug]!.id,
+ divisionId: divisions[spec.division],
+ slug: spec.slug,
+ name: spec.name,
+ city: city.name,
+ state: market.state,
+ zip: city.zip,
+ county: market.county,
+ metro: market.metro,
+ lat,
+ lon,
+ status: "ACTIVE",
+ hoaFeeMonthly: spec.hoa ?? null,
+ hoaRequired: spec.hoa !== null && spec.hoa !== undefined,
+ schoolDistrict: pick(market.schoolDistricts),
+ ageRestricted: spec.ageRestricted ?? false,
+ amenities: ["Community pool", "Trail network", "Neighborhood park"].slice(0, 1 + Math.floor(rand() * 3)),
+ searchText: `${spec.name} ${city.name} ${market.state} ${city.zip} ${market.metro} ${market.county}`,
+ sourceUrl: DEMO_SOURCE_URL,
+ isDemo: true,
+ },
+ });
+ communities.push({ id: community.id, spec, city, market });
+ }
+
+ // ── Floor plans (3 per community) ────────────────────────────────
+ const planIdsByCommunity: Record<string, { id: string; name: string; beds: number; baths: number; sqft: number; stories: number }[]> = {};
+ let planCursor = 0;
+ for (const { id: communityId, spec } of communities) {
+ planIdsByCommunity[communityId] = [];
+ for (let i = 0; i < 3; i++) {
+ const name = PLAN_NAMES[(planCursor + i) % PLAN_NAMES.length]!;
+ const beds = 2 + ((planCursor + i) % 4); // 2–5
+ const baths = beds - 1 + (rand() > 0.5 ? 0.5 : 0);
+ const sqft = Math.round(between(1400, 3800) / 10) * 10;
+ const stories = sqft > 2600 ? 2 : rand() > 0.6 ? 2 : 1;
+ const existing = await prisma.floorPlan.findFirst({ where: { communityId, name } });
+ const plan =
+ existing ??
+ (await prisma.floorPlan.create({
+ data: {
+ communityId,
+ builderId: builders[spec.builderSlug]!.id,
+ name,
+ builderPlanId: `${spec.slug.toUpperCase().slice(0, 3)}-${100 + i}`,
+ beds,
+ bathsFull: Math.floor(baths),
+ bathsHalf: baths % 1 ? 1 : 0,
+ sqft,
+ stories,
+ garageSpaces: beds >= 4 ? 3 : 2,
+ basePrice: Math.round(between(...MARKETS[spec.market]!.priceBand) / 1000) * 1000,
+ homeType: rand() > 0.85 ? "TOWNHOME" : "SINGLE_FAMILY",
+ planMediaRights: "none",
+ sourceUrl: DEMO_SOURCE_URL,
+ isDemo: true,
+ },
+ }));
+ planIdsByCommunity[communityId]!.push({ id: plan.id, name, beds, baths, sqft, stories });
+ }
+ planCursor += 3;
+ }
+
+ // ── Inventory homes (10 per community, EXCEPT Cedar Bend = pipeline) ──
+ const statuses = ["MOVE_IN_READY", "UNDER_CONSTRUCTION", "UNDER_CONSTRUCTION", "PLANNED"] as const;
+ let homeCount = 0;
+ for (const { id: communityId, spec, city, market } of communities) {
+ if (spec.slug === "cedar-bend") continue; // arrives via the fixture pipeline
+ for (let i = 0; i < 10; i++) {
+ const plan = pick(planIdsByCommunity[communityId]!);
+ const streetNumber = 100 + i * 7 + Math.floor(rand() * 5);
+ const street = `${streetNumber} ${pick(STREET_NAMES)}`;
+ const lotNumber = String(streetNumber);
+ const constructionStatus = pick([...statuses]);
+ // ~1 in 9 homes has no published price — proves null-not-guessed rendering.
+ const price = i % 9 === 8 ? null : Math.round(between(...market.priceBand) / 990) * 990;
+ const lat = Number((city.lat + between(-0.015, 0.015)).toFixed(6));
+ const lon = Number((city.lon + between(-0.015, 0.015)).toFixed(6));
+ const inventoryId = `${spec.slug.toUpperCase().slice(0, 3)}-${1000 + i}`;
+ const key = canonicalKey({
+ builderSlug: spec.builderSlug,
+ communityName: spec.name,
+ address: street,
+ lotNumber,
+ builderInventoryId: inventoryId,
+ lat,
+ lon,
+ planName: plan.name,
+ });
+ const estCompletion =
+ constructionStatus === "MOVE_IN_READY"
+ ? null
+ : new Date(Date.UTC(2026, 8 + Math.floor(rand() * 6), 1));
+
+ const home = await prisma.inventoryHome.upsert({
+ where: { canonicalKey: key },
+ update: {},
+ create: {
+ canonicalKey: key,
+ communityId,
+ builderId: builders[spec.builderSlug]!.id,
+ floorPlanId: plan.id,
+ builderInventoryId: inventoryId,
+ street,
+ city: city.name,
+ state: market.state,
+ zip: city.zip,
+ normalizedAddress: normalizeAddress(street),
+ lotNumber,
+ lat,
+ lon,
+ price,
+ beds: plan.beds,
+ bathsTotal: plan.baths,
+ sqft: plan.sqft + Math.floor(between(-80, 120)),
+ stories: plan.stories,
+ garageSpaces: plan.beds >= 4 ? 3 : 2,
+ homeType: "SINGLE_FAMILY",
+ constructionStatus,
+ estCompletionDate: estCompletion,
+ availabilityStatus: constructionStatus === "MOVE_IN_READY" ? "Available now" : "Available at completion",
+ status: "PUBLISHED",
+ freshness: "FRESH",
+ verificationLabel: "DEMONSTRATION",
+ lastVerifiedAt: new Date(),
+ sourceId: syntheticSource.id,
+ sourceUrl: DEMO_SOURCE_URL,
+ isDemo: true,
+ },
+ });
+ homeCount++;
+
+ // Synthetic evidence rows so every published fact remains traceable.
+ const evidenceFields: [string, string | null][] = [
+ ["price", price === null ? null : `$${price.toLocaleString("en-US")}`],
+ ["beds", `${plan.beds} Beds`],
+ ["sqft", `${plan.sqft.toLocaleString("en-US")} sq ft`],
+ ];
+ await prisma.sourceEvidence.createMany({
+ data: evidenceFields.map(([field, raw]) => ({
+ entityType: "INVENTORY_HOME" as const,
+ entityId: home.id,
+ field,
+ sourceUrl: DEMO_SOURCE_URL,
+ retrievedAt: new Date(),
+ extractorVersion: "seed-1",
+ rawValue: raw,
+ normalizedValue: raw === null ? null : String(field === "price" ? price : field === "beds" ? plan.beds : plan.sqft),
+ evidenceText: raw,
+ confidence: raw === null ? 0 : 1,
+ })),
+ });
+ }
+ }
+
+ // ── Community price rollups ──────────────────────────────────────
+ for (const { id: communityId } of communities) {
+ const agg = await prisma.inventoryHome.aggregate({
+ where: { communityId, status: "PUBLISHED", price: { not: null } },
+ _min: { price: true },
+ _max: { price: true },
+ });
+ await prisma.community.update({
+ where: { id: communityId },
+ data: { priceMin: agg._min.price, priceMax: agg._max.price },
+ });
+ }
+
+ // ── Incentives (18), sales offices (12), lots (~30) ─────────────
+ const incentiveTitles = [
+ ["$10,000 Closing Cost Credit", 10_000],
+ ["Rate Buydown Available", null],
+ ["Appliance Package Included", 4_500],
+ ["$5,000 Toward Options", 5_000],
+ ["Reduced Earnest Money", null],
+ ["Landscaping Package", 3_000],
+ ] as const;
+ let incentiveCount = 0;
+ for (const [index, { id: communityId, spec }] of communities.entries()) {
+ const howMany = index % 2 === 0 ? 2 : 1; // 12×1.5 = 18
+ for (let i = 0; i < howMany; i++) {
+ const [title, valueUsd] = incentiveTitles[(index + i) % incentiveTitles.length]!;
+ const hasExpiry = i % 2 === 0;
+ const existing = await prisma.incentive.findFirst({ where: { communityId, title } });
+ if (existing) continue;
+ await prisma.incentive.create({
+ data: {
+ scope: "COMMUNITY",
+ builderId: builders[spec.builderSlug]!.id,
+ communityId,
+ title,
+ description: `${title} on select homes at ${spec.name}. Demonstration offer — not a real promotion.`,
+ valueUsd,
+ expiresAt: hasExpiry ? new Date(Date.UTC(2026, 8 + (i % 3), 30)) : null,
+ evergreenLabel: hasExpiry ? null : "expiration not provided",
+ sourceId: syntheticSource.id,
+ sourceUrl: DEMO_SOURCE_URL,
+ isDemo: true,
+ },
+ });
+ incentiveCount++;
+ }
+ }
+
+ for (const { id: communityId, spec, city, market } of communities) {
+ const existing = await prisma.salesOffice.findFirst({ where: { communityId } });
+ if (existing) continue;
+ await prisma.salesOffice.create({
+ data: {
+ communityId,
+ street: `1 ${spec.name} Welcome Center`,
+ city: city.name,
+ state: market.state,
+ zip: city.zip,
+ phone: "(555) 010-" + String(1000 + communities.findIndex((c) => c.id === communityId)).slice(1),
+ hours: { "mon-sat": "10:00-18:00", sun: "12:00-17:00" },
+ isDemo: true,
+ },
+ });
+ }
+
+ let lotCount = 0;
+ for (const { id: communityId } of communities.slice(0, 6)) {
+ for (let i = 0; i < 5; i++) {
+ const lotNumber = `L-${200 + i}`;
+ await prisma.lot.upsert({
+ where: { communityId_lotNumber: { communityId, lotNumber } },
+ update: {},
+ create: {
+ communityId,
+ lotNumber,
+ sizeSqft: Math.round(between(5500, 12_000) / 100) * 100,
+ status: pick(["available", "available", "reserved", "sold"]),
+ price: rand() > 0.4 ? Math.round(between(80_000, 180_000) / 1000) * 1000 : null,
+ isDemo: true,
+ },
+ });
+ lotCount++;
+ }
+ }
+
+ console.log(
+ `Seeded: ${builderSpecs.length} builders, ${divisionSpecs.length} divisions, ${communities.length} communities, ` +
+ `${communities.length * 3} plans, ${homeCount} direct homes (+ Cedar Bend via pipeline), ${incentiveCount} incentives, ` +
+ `12 sales offices, ${lotCount} lots. All isDemo=true, label=DEMONSTRATION.`,
+ );
+}
+
+main()
+ .catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ })
+ .finally(() => prisma.$disconnect());
diff --git a/packages/validation/src/rules.test.ts b/packages/validation/src/rules.test.ts
index 266f63d..b8057ec 100644
--- a/packages/validation/src/rules.test.ts
+++ b/packages/validation/src/rules.test.ts
@@ -108,7 +108,7 @@ describe("home.plausible-beds / baths / sqft", () => {
});
describe("dup.no-active-address", () => {
- it("fails when another active record holds the same normalized address", async () => {
+ it("fails when another active record holds the same normalized address in the same zip", async () => {
const dupCtx: RuleContext = {
...ctx,
db: { ...stubDb, activeHomeExistsAtAddress: async () => true },
@@ -116,6 +116,22 @@ describe("dup.no-active-address", () => {
expect((await noDuplicateActiveAddress.check(makeHome(), dupCtx)).passed).toBe(false);
expect((await noDuplicateActiveAddress.check(makeHome(), ctx)).passed).toBe(true);
});
+
+ it("passes the zip through so identical street strings in other cities don't collide", async () => {
+ let seenZip: string | null = "unset";
+ const capturingCtx: RuleContext = {
+ ...ctx,
+ db: {
+ ...stubDb,
+ activeHomeExistsAtAddress: async (_addr, zip) => {
+ seenZip = zip;
+ return false;
+ },
+ },
+ };
+ await noDuplicateActiveAddress.check(makeHome(), capturingCtx);
+ expect(seenZip).toBe("78701");
+ });
});
describe("ref.fk-integrity", () => {
diff --git a/packages/validation/src/rules.ts b/packages/validation/src/rules.ts
index c101e92..be94713 100644
--- a/packages/validation/src/rules.ts
+++ b/packages/validation/src/rules.ts
@@ -121,12 +121,15 @@ export const noDuplicateActiveAddress: ValidationRule = {
const { normalizeAddress } = await import("@spechomes/shared");
const normalized = normalizeAddress(address);
if (!normalized) return { passed: true };
- const exists = await ctx.db.activeHomeExistsAtAddress(normalized, record.canonicalKey);
+ // Scoped to the same zip — an identical street string in another city
+ // is a coincidence, not a duplicate of the same physical home.
+ const zip = fieldValue<string>(record, "zip");
+ const exists = await ctx.db.activeHomeExistsAtAddress(normalized, zip, record.canonicalKey);
if (exists) {
return {
passed: false,
- message: `another active inventory record already exists at "${normalized}"`,
- details: { normalizedAddress: normalized },
+ message: `another active inventory record already exists at "${normalized}" (zip ${zip ?? "unknown"})`,
+ details: { normalizedAddress: normalized, zip },
};
}
return { passed: true };
diff --git a/packages/validation/src/types.ts b/packages/validation/src/types.ts
index 458fb76..5598622 100644
--- a/packages/validation/src/types.ts
+++ b/packages/validation/src/types.ts
@@ -22,7 +22,7 @@ export interface RuleContext {
/** The narrow read-only surface rules may touch — keeps rules testable with a stub. */
export interface RuleDb {
- activeHomeExistsAtAddress(normalizedAddress: string, excludeCanonicalKey: string): Promise<boolean>;
+ activeHomeExistsAtAddress(normalizedAddress: string, zip: string | null, excludeCanonicalKey: string): Promise<boolean>;
publishedPriceForCanonicalKey(canonicalKey: string): Promise<number | null>;
communityExists(communitySlugOrName: string, builderSlug: string): Promise<boolean>;
builderExists(builderSlug: string): Promise<boolean>;
← 1e9a023 collector adapter contract + meridian-homes fixture adapter
·
back to Homesonspec
·
consumer web app (search/detail/community/builder/contact) + 6b477e6 →