← back to Homesonspec

snapshot store: store storagePath relative + resolver for volume-move safety (TK-10809)

0a7e6b34f17300a59e14b7d06dc1e9a1a8b15f91 · 2026-08-30 09:20:32 -0700 · steve

Files touched

• M apps/workers/src/pipeline.ts
• A apps/workers/src/snapshot-path.test.ts
• A apps/workers/src/snapshot-path.ts
• M scripts/backfill-drh-geo-from-cache.mjs

Diff

commit 0a7e6b34f17300a59e14b7d06dc1e9a1a8b15f91
Author: steve <steve@designerwallcoverings.com>
Date:   Sun Aug 30 09:20:32 2026 -0700

    snapshot store: store storagePath relative + resolver for volume-move safety (TK-10809)
---
 apps/workers/src/pipeline.ts            | 14 +++----
 apps/workers/src/snapshot-path.test.ts  | 65 +++++++++++++++++++++++++++++++++
 apps/workers/src/snapshot-path.ts       | 22 +++++++++++
 scripts/backfill-drh-geo-from-cache.mjs |  8 +++-
 4 files changed, 101 insertions(+), 8 deletions(-)

diff --git a/apps/workers/src/pipeline.ts b/apps/workers/src/pipeline.ts
index 631c9976..9287c442 100644
--- a/apps/workers/src/pipeline.ts
+++ b/apps/workers/src/pipeline.ts
@@ -18,6 +18,7 @@ import {
 import type { RawPage, SourceAdapter } from "@homesonspec/collectors-common";
 import { publishStagedRecord } from "@homesonspec/publisher";
 import { recordSourceRun } from "./verify";
+import { SNAPSHOT_DIR, snapshotRelativePath } from "./snapshot-path";
 
 export { publishStagedRecord };
 
@@ -31,7 +32,6 @@ export { publishStagedRecord };
 
 // Anchor to the repo root — the CLI may run with cwd anywhere in the workspace.
 const REPO_ROOT = join(import.meta.dirname, "../../..");
-const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(REPO_ROOT, "var/snapshots");
 
 const ENTITY_TYPE_MAP: Record<ExtractedRecord["entityType"], EntityType> = {
   community: "COMMUNITY",
@@ -68,15 +68,15 @@ export async function fetchStage(adapter: SourceAdapter, source: SourceRegistry)
       pages.push({ page, snapshotId: existing.id, changed: false });
       continue;
     }
-    const storagePath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
-    await writeFile(storagePath, page.body);
+    const diskPath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
+    await writeFile(diskPath, page.body);
     const snapshot = await prisma.rawSnapshot.create({
       data: {
         sourceId: source.id,
         url: page.url,
         retrievedAt: new Date(page.retrievedAt),
         contentHash: page.contentHash,
-        storagePath,
+        storagePath: snapshotRelativePath(source.key, page.contentHash, page.contentType),
         contentType: page.contentType,
         httpStatus: 200,
       },
@@ -291,15 +291,15 @@ export async function runPipeline(adapter: SourceAdapter) {
     if (existing) {
       snapshotId = existing.id;
     } else {
-      const storagePath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
-      await writeFile(storagePath, page.body);
+      const diskPath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
+      await writeFile(diskPath, page.body);
       const snapshot = await prisma.rawSnapshot.create({
         data: {
           sourceId: source.id,
           url: page.url,
           retrievedAt: new Date(page.retrievedAt),
           contentHash: page.contentHash,
-          storagePath,
+          storagePath: snapshotRelativePath(source.key, page.contentHash, page.contentType),
           contentType: page.contentType,
           httpStatus: 200,
         },
diff --git a/apps/workers/src/snapshot-path.test.ts b/apps/workers/src/snapshot-path.test.ts
new file mode 100644
index 00000000..ed523b33
--- /dev/null
+++ b/apps/workers/src/snapshot-path.test.ts
@@ -0,0 +1,65 @@
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { resolveSnapshotPath, snapshotExt, snapshotRelativePath } from "./snapshot-path";
+
+describe("snapshotExt", () => {
+  it("json content-type → .json", () => {
+    expect(snapshotExt("application/json")).toBe(".json");
+    expect(snapshotExt("text/json; charset=utf-8")).toBe(".json");
+  });
+  it("anything else (incl. undefined) → .html", () => {
+    expect(snapshotExt("text/html")).toBe(".html");
+    expect(snapshotExt(undefined)).toBe(".html");
+    expect(snapshotExt("")).toBe(".html");
+  });
+});
+
+describe("snapshotRelativePath", () => {
+  it("builds sourceKey/hash.ext for json vs html", () => {
+    expect(snapshotRelativePath("dr-horton-site", "abc123", "application/json")).toBe(
+      "dr-horton-site/abc123.json",
+    );
+    expect(snapshotRelativePath("dr-horton-site", "abc123", "text/html")).toBe(
+      "dr-horton-site/abc123.html",
+    );
+  });
+  it("is POSIX-relative (never absolute)", () => {
+    expect(isAbsolute(snapshotRelativePath("k", "h", "text/html"))).toBe(false);
+  });
+});
+
+describe("resolveSnapshotPath", () => {
+  it("absolute storagePath passes through unchanged (legacy row)", () => {
+    const abs = "/root/Projects/homesonspec/var/snapshots/k/h.html";
+    expect(resolveSnapshotPath(abs)).toBe(abs);
+    expect(resolveSnapshotPath(abs, "/mnt/vol/snapshots")).toBe(abs);
+  });
+  it("relative storagePath joins the default snapshot dir", () => {
+    expect(resolveSnapshotPath("k/h.html")).toMatch(/\/var\/snapshots\/k\/h\.html$/);
+  });
+  it("relative storagePath joins a custom snapshot dir", () => {
+    expect(resolveSnapshotPath("k/h.html", "/mnt/vol/snapshots")).toBe(
+      "/mnt/vol/snapshots/k/h.html",
+    );
+  });
+});
+
+describe("round-trip: write to disk, store relative, resolve, read back", () => {
+  it("returns identical bytes", () => {
+    const snapshotDir = mkdtempSync(join(tmpdir(), "hos-snap-"));
+    const sourceKey = "example-source";
+    const contentHash = "deadbeefcafef00d";
+    const body = Buffer.from(`<html><body>${contentHash}</body></html>`);
+
+    // write the file where the pipeline would (snapshotDir/sourceKey/hash.ext)
+    mkdirSync(join(snapshotDir, sourceKey), { recursive: true });
+    const rel = snapshotRelativePath(sourceKey, contentHash, "text/html");
+    writeFileSync(join(snapshotDir, rel), body);
+
+    // the DB stores the relative path; the reader resolves it back
+    const resolved = resolveSnapshotPath(rel, snapshotDir);
+    expect(readFileSync(resolved)).toEqual(body);
+  });
+});
diff --git a/apps/workers/src/snapshot-path.ts b/apps/workers/src/snapshot-path.ts
new file mode 100644
index 00000000..e151047d
--- /dev/null
+++ b/apps/workers/src/snapshot-path.ts
@@ -0,0 +1,22 @@
+import { isAbsolute, join } from "node:path";
+
+// Single source of truth for the raw-snapshot store location. Prod offloads this
+// to a dedicated volume by setting SNAPSHOT_DIR (see ops/offload-snapshots.mjs);
+// the default keeps the in-repo layout used by dev + fixtures.
+const REPO_ROOT = join(import.meta.dirname, "../../..");
+export const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(REPO_ROOT, "var/snapshots");
+
+export function snapshotExt(contentType?: string) {
+  return contentType?.includes("json") ? ".json" : ".html";
+}
+
+export function snapshotRelativePath(sourceKey: string, contentHash: string, contentType?: string) {
+  return `${sourceKey}/${contentHash}${snapshotExt(contentType)}`;
+}
+
+// Legacy rows stored an ABSOLUTE storagePath (pre-offload); those must keep
+// resolving verbatim. New rows store a POSIX-relative path resolved against the
+// active snapshot dir, so moving the volume + repointing SNAPSHOT_DIR is exact.
+export function resolveSnapshotPath(storagePath: string, snapshotDir = SNAPSHOT_DIR) {
+  return isAbsolute(storagePath) ? storagePath : join(snapshotDir, storagePath);
+}
diff --git a/scripts/backfill-drh-geo-from-cache.mjs b/scripts/backfill-drh-geo-from-cache.mjs
index f382d4c5..50eea68f 100644
--- a/scripts/backfill-drh-geo-from-cache.mjs
+++ b/scripts/backfill-drh-geo-from-cache.mjs
@@ -7,11 +7,17 @@
 // State-gated + coarse US bbox (a wrong pin is worse than null).
 import { execFileSync } from "node:child_process";
 import { readFileSync } from "node:fs";
+import { isAbsolute, join } from "node:path";
 
 const DB = process.env.DATABASE_URL || "postgresql://macstudio3@localhost/homesonspec?host=/tmp";
 const psql = (sql) => execFileSync("psql", [DB, "-tAc", sql], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 }).trim();
 const q = (s) => String(s).replace(/'/g, "''");
 
+// storagePath is stored absolute on legacy rows, POSIX-relative on new rows
+// (post snapshot-store offload). Resolve both against the active snapshot dir.
+const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(import.meta.dirname, "../var/snapshots");
+const resolveSnapshot = (p) => (isAbsolute(p) ? p : join(SNAPSHOT_DIR, p));
+
 // Same community-coord logic as the dr-horton adapter (lowercase ld+json lat/lon; state-gated + US bbox).
 function communityGeo(html, state) {
   const m = html.match(/"latitude"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"longitude"\s*:\s*(-?\d+(?:\.\d+)?)/);
@@ -42,7 +48,7 @@ for (const line of rows) {
     where b.slug='dr-horton' and h."sourceUrl"='${q(url)}' and h.lat is null limit 1`);
   if (!state) continue; // no ungeocoded homes here
   let html;
-  try { html = readFileSync(path, "utf8"); } catch { missing++; continue; }
+  try { html = readFileSync(resolveSnapshot(path), "utf8"); } catch { missing++; continue; }
   const g = communityGeo(html, state);
   if (!g) { skippedNoGeo++; continue; }
   communities++;

← 7d5c17e6 pm2(homesonspec): node-direct next start so pm2 owns the por  ·  back to Homesonspec  ·  ops: non-destructive snapshot offload+verify script + read-o 58cad18e →