← back to Homesonspec
ops: non-destructive snapshot offload+verify script + read-only integrity canary (TK-10809)
58cad18e6ae2c0b5a936de68cbfcfde3d27c6352 · 2026-08-30 09:23:40 -0700 · steve
Files touched
M .gitignoreA ops/offload-snapshots.mjsA ops/verify-snapshots.mjsM scripts/backfill-drh-geo-from-cache.mjs
Diff
commit 58cad18e6ae2c0b5a936de68cbfcfde3d27c6352
Author: steve <steve@designerwallcoverings.com>
Date: Sun Aug 30 09:23:40 2026 -0700
ops: non-destructive snapshot offload+verify script + read-only integrity canary (TK-10809)
---
.gitignore | 3 +
ops/offload-snapshots.mjs | 154 ++++++++++++++++++++++++++++++++
ops/verify-snapshots.mjs | 81 +++++++++++++++++
scripts/backfill-drh-geo-from-cache.mjs | 3 +-
4 files changed, 240 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 13e743b2..17ef0429 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,6 @@ ROLLBACK-DNS.txt
ops/import-sweep.log
ops/import-sweep.launchd.log
ops/.import-sweep.lock
+
+# snapshot-integrity canary heartbeat (runtime output)
+ops/data/
diff --git a/ops/offload-snapshots.mjs b/ops/offload-snapshots.mjs
new file mode 100644
index 00000000..805696ed
--- /dev/null
+++ b/ops/offload-snapshots.mjs
@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+// NON-DESTRUCTIVE snapshot-store offload (TK-10809). Copies the on-disk raw-snapshot
+// store to a dedicated volume and VERIFIES every sampled RawSnapshot row resolves +
+// hashes correctly against the destination. It NEVER deletes, moves, prunes, or
+// rewrites the source: rsync runs copy-only (no --delete), old copies are retained,
+// and deletion is a SEPARATE Steve-gated step, out of scope here.
+//
+// COPY phase: dry-run prints the exact `rsync -a` plan + per-builder counts/sizes;
+// --apply runs `rsync -a` (copy only).
+// VERIFY phase: READ-ONLY SELECT of RawSnapshot rows; resolve storagePath against
+// DEST, stat + sha256, compare to contentHash. Reports coverage %.
+//
+// Usage:
+// node ops/offload-snapshots.mjs --dest /mnt/vol/snapshots [--src DIR] [--apply] [--full]
+import { execFileSync } from "node:child_process";
+import { readFileSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { isAbsolute, join, basename } from "node:path";
+
+const argv = process.argv.slice(2);
+const flag = (name) => argv.includes(`--${name}`);
+const opt = (name, def) => {
+ const i = argv.indexOf(`--${name}`);
+ return i >= 0 && argv[i + 1] ? argv[i + 1] : def;
+};
+
+const DEST = opt("dest");
+const SRC = opt("src", process.env.SNAPSHOT_DIR ?? "/root/Projects/homesonspec/var/snapshots");
+const APPLY = flag("apply");
+const FULL = flag("full");
+const SAMPLE = Number(opt("sample", "500"));
+
+if (!DEST) {
+ console.error("FATAL: --dest <dir> is required.");
+ process.exit(2);
+}
+if (isAbsolute(DEST) === false || isAbsolute(SRC) === false) {
+ console.error("FATAL: --dest and --src must be absolute paths.");
+ process.exit(2);
+}
+if (DEST === SRC) {
+ console.error("FATAL: --dest must differ from --src (refusing to copy onto itself).");
+ process.exit(2);
+}
+
+const DB = process.env.DATABASE_URL || "postgresql://macstudio3@localhost/homesonspec?host=/tmp";
+const psql = (sql) =>
+ execFileSync("psql", [DB, "-tAc", sql], { encoding: "utf8", maxBuffer: 512 * 1024 * 1024 }).trim();
+const human = (n) => {
+ const u = ["B", "K", "M", "G", "T"];
+ let i = 0;
+ while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; }
+ return `${n.toFixed(1)}${u[i]}`;
+};
+
+console.log(`\n=== snapshot offload (${APPLY ? "APPLY" : "DRY-RUN"}) ===`);
+console.log(`SRC : ${SRC}`);
+console.log(`DEST: ${DEST}\n`);
+
+// ── COPY phase ─────────────────────────────────────────────────────────
+console.log("── COPY phase ──");
+// Per-builder (top-level dir = source.key) file counts + sizes, dry-run always.
+try {
+ const out = execFileSync(
+ "bash",
+ ["-c", `cd ${JSON.stringify(SRC)} && for d in */; do printf '%s\\t%s\\t%s\\n' "$d" "$(find "$d" -type f | wc -l | tr -d ' ')" "$(du -sk "$d" | cut -f1)"; done`],
+ { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 },
+ ).trim();
+ if (out) {
+ let files = 0, kb = 0;
+ for (const line of out.split("\n")) {
+ const [dir, cnt, k] = line.split("\t");
+ files += Number(cnt); kb += Number(k);
+ console.log(` ${dir.padEnd(32)} ${String(cnt).padStart(8)} files ${human(Number(k) * 1024).padStart(9)}`);
+ }
+ console.log(` ${"TOTAL".padEnd(32)} ${String(files).padStart(8)} files ${human(kb * 1024).padStart(9)}`);
+ } else {
+ console.log(" (no per-builder dirs found under SRC on this host)");
+ }
+} catch (e) {
+ console.log(` (could not enumerate SRC locally: ${e.message.split("\n")[0]})`);
+}
+
+const rsyncCmd = ["rsync", "-a", `${SRC.replace(/\/$/, "")}/`, `${DEST.replace(/\/$/, "")}/`];
+console.log(`\n rsync plan (COPY ONLY, no --delete):\n ${rsyncCmd.join(" ")}`);
+console.log(" SRC untouched; old copies retained; deletion is a separate Steve-gated step, out of scope.");
+
+if (APPLY) {
+ console.log("\n running rsync…");
+ execFileSync(rsyncCmd[0], rsyncCmd.slice(1), { stdio: "inherit" });
+ console.log(" rsync complete (copy only).");
+} else {
+ console.log("\n (dry-run — pass --apply to run the copy)");
+}
+
+// ── VERIFY phase ───────────────────────────────────────────────────────
+console.log("\n── VERIFY phase (READ-ONLY, resolves against DEST) ──");
+const orderClause = FULL ? "" : `order by random() limit ${SAMPLE}`;
+const rows = psql(
+ `select "storagePath" || E'\\t' || "contentHash" from "RawSnapshot" ${orderClause}`,
+)
+ .split("\n")
+ .filter(Boolean);
+console.log(` sampled ${rows.length} RawSnapshot rows (${FULL ? "FULL" : `sample ${SAMPLE}`})`);
+
+// resolve a stored path against DEST: relative → DEST/rel; absolute legacy → swap
+// its SRC prefix to DEST (the file was copied there under the same subtree).
+// New rows store sourceKey/hash.ext; some existing rows carry the store's own
+// "var/snapshots/" tail — strip it so the file resolves under DEST directly.
+const relTail = (p) => p.replace(/^(?:\.\/)?var\/snapshots\//, "");
+const resolveAgainstDest = (p) => {
+ if (!isAbsolute(p)) return join(DEST, relTail(p));
+ if (p.startsWith(SRC.replace(/\/$/, "") + "/")) return join(DEST, p.slice(SRC.replace(/\/$/, "").length + 1));
+ // absolute path not under SRC — fall back to matching basename subtree
+ return join(DEST, basename(p));
+};
+
+let ok = 0;
+const misses = [];
+const mismatches = [];
+for (const line of rows) {
+ const [storagePath, contentHash] = line.split("\t");
+ if (!storagePath || !contentHash) continue;
+ const dest = resolveAgainstDest(storagePath);
+ try {
+ statSync(dest);
+ } catch {
+ misses.push({ storagePath, dest });
+ continue;
+ }
+ const digest = createHash("sha256").update(readFileSync(dest)).digest("hex");
+ if (digest === contentHash) ok++;
+ else mismatches.push({ storagePath, dest, expected: contentHash, got: digest });
+}
+
+const total = rows.length || 1;
+const coverage = ((ok / total) * 100).toFixed(2);
+console.log(`\n coverage: ${coverage}% (ok ${ok} / ${rows.length})`);
+console.log(` missing : ${misses.length}`);
+console.log(` mismatch: ${mismatches.length}`);
+for (const m of misses.slice(0, 20)) console.log(` MISS ${m.storagePath} -> ${m.dest}`);
+if (misses.length > 20) console.log(` … and ${misses.length - 20} more misses`);
+for (const m of mismatches.slice(0, 20))
+ console.log(` HASH ${m.storagePath}: expected ${m.expected.slice(0, 12)} got ${m.got.slice(0, 12)}`);
+if (mismatches.length > 20) console.log(` … and ${mismatches.length - 20} more mismatches`);
+
+const clean = misses.length === 0 && mismatches.length === 0;
+console.log(`\n=== ${clean ? "VERIFY PASS" : "VERIFY INCOMPLETE"} ===`);
+
+// under --apply, a non-100% verify is a hard failure — the copy is not proven exact.
+if (APPLY && !clean) {
+ console.error("FAIL: verify coverage <100% under --apply; DO NOT proceed to repoint or delete.");
+ process.exit(1);
+}
diff --git a/ops/verify-snapshots.mjs b/ops/verify-snapshots.mjs
new file mode 100644
index 00000000..21f23dab
--- /dev/null
+++ b/ops/verify-snapshots.mjs
@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+// READ-ONLY snapshot-store integrity canary (TK-10809). Samples N RawSnapshot rows,
+// resolves storagePath against SNAPSHOT_DIR, confirms the file exists and
+// sha256(body) === contentHash, and emits a top-level verdict PASS/WARN/FAIL
+// (fleet-health-rollup vocabulary) plus ops/data/latest.json. Touches nothing.
+//
+// Usage: node ops/verify-snapshots.mjs [--full] [--sample 500]
+import { execFileSync } from "node:child_process";
+import { readFileSync, statSync, mkdirSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { isAbsolute, join } from "node:path";
+
+const argv = process.argv.slice(2);
+const flag = (name) => argv.includes(`--${name}`);
+const opt = (name, def) => {
+ const i = argv.indexOf(`--${name}`);
+ return i >= 0 && argv[i + 1] ? argv[i + 1] : def;
+};
+
+const FULL = flag("full");
+const SAMPLE = Number(opt("sample", "500"));
+const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(import.meta.dirname, "../var/snapshots");
+// New rows store sourceKey/hash.ext; some existing rows carry the store's own
+// "var/snapshots/" tail — strip it so both resolve under the active SNAPSHOT_DIR.
+const relTail = (p) => p.replace(/^(?:\.\/)?var\/snapshots\//, "");
+const resolveSnapshot = (p) => (isAbsolute(p) ? p : join(SNAPSHOT_DIR, relTail(p)));
+
+const DB = process.env.DATABASE_URL || "postgresql://macstudio3@localhost/homesonspec?host=/tmp";
+const psql = (sql) =>
+ execFileSync("psql", [DB, "-tAc", sql], { encoding: "utf8", maxBuffer: 512 * 1024 * 1024 }).trim();
+
+const writeLatest = (obj) => {
+ const dir = join(import.meta.dirname, "data");
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, "latest.json"), JSON.stringify(obj, null, 2));
+};
+
+let rows;
+try {
+ const orderClause = FULL ? "" : `order by random() limit ${SAMPLE}`;
+ rows = psql(`select "storagePath" || E'\\t' || "contentHash" from "RawSnapshot" ${orderClause}`)
+ .split("\n")
+ .filter(Boolean);
+} catch (e) {
+ const rec = { verdict: "FAIL", status: "FAIL", checked: 0, ok: 0, missing: 0, mismatch: 0, error: e.message.split("\n")[0] };
+ writeLatest(rec);
+ console.error(`FAIL: could not read RawSnapshot (${rec.error})`);
+ process.exit(1);
+}
+
+let ok = 0, missing = 0, mismatch = 0;
+const misses = [], mismatches = [];
+for (const line of rows) {
+ const [storagePath, contentHash] = line.split("\t");
+ if (!storagePath || !contentHash) continue;
+ const path = resolveSnapshot(storagePath);
+ try {
+ statSync(path);
+ } catch {
+ missing++; misses.push(storagePath); continue;
+ }
+ const digest = createHash("sha256").update(readFileSync(path)).digest("hex");
+ if (digest === contentHash) ok++;
+ else { mismatch++; mismatches.push(storagePath); }
+}
+
+const checked = rows.length;
+// PASS = all sampled files present + hash-match. WARN = only missing files (a move
+// mid-flight or an un-offloaded row). FAIL = any hash mismatch (corruption).
+let verdict;
+if (mismatch > 0) verdict = "FAIL";
+else if (missing > 0) verdict = "WARN";
+else verdict = "PASS";
+
+const rec = { verdict, status: verdict, checked, ok, missing, mismatch };
+writeLatest(rec);
+
+console.log(`snapshot integrity: ${verdict} checked=${checked} ok=${ok} missing=${missing} mismatch=${mismatch}`);
+for (const p of misses.slice(0, 10)) console.log(` MISS ${p}`);
+for (const p of mismatches.slice(0, 10)) console.log(` HASH ${p}`);
+process.exit(verdict === "FAIL" ? 1 : 0);
diff --git a/scripts/backfill-drh-geo-from-cache.mjs b/scripts/backfill-drh-geo-from-cache.mjs
index 50eea68f..abf66a8d 100644
--- a/scripts/backfill-drh-geo-from-cache.mjs
+++ b/scripts/backfill-drh-geo-from-cache.mjs
@@ -16,7 +16,8 @@ const q = (s) => String(s).replace(/'/g, "''");
// storagePath is stored absolute on legacy rows, POSIX-relative on new rows
// (post snapshot-store offload). Resolve both against the active snapshot dir.
const SNAPSHOT_DIR = process.env.SNAPSHOT_DIR ?? join(import.meta.dirname, "../var/snapshots");
-const resolveSnapshot = (p) => (isAbsolute(p) ? p : join(SNAPSHOT_DIR, p));
+const relTail = (p) => p.replace(/^(?:\.\/)?var\/snapshots\//, "");
+const resolveSnapshot = (p) => (isAbsolute(p) ? p : join(SNAPSHOT_DIR, relTail(p)));
// Same community-coord logic as the dr-horton adapter (lowercase ld+json lat/lon; state-gated + US bbox).
function communityGeo(html, state) {
← 0a7e6b34 snapshot store: store storagePath relative + resolver for vo
·
back to Homesonspec
·
TK-10809: single shared snapshot-path resolver (ops/lib), ki d461d39d →