← back to Wallco Ai
add scripts/sweep_orphan_generated.js — orphan-PNG sweeper for data/generated/. Default dry-run; --apply with guardrails (PG-floor 1000, 30min in-flight grace, --max-bytes cap). First run reclaimed 33,799 files / 26.92 GB (orphan .original pre-seamless artifacts).
323e1faf3eadb354c51f11be5ef744bb576dd407 · 2026-05-22 14:07:46 -0700 · Steve Abrams
Files touched
A scripts/sweep_orphan_generated.js
Diff
commit 323e1faf3eadb354c51f11be5ef744bb576dd407
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri May 22 14:07:46 2026 -0700
add scripts/sweep_orphan_generated.js — orphan-PNG sweeper for data/generated/. Default dry-run; --apply with guardrails (PG-floor 1000, 30min in-flight grace, --max-bytes cap). First run reclaimed 33,799 files / 26.92 GB (orphan .original pre-seamless artifacts).
---
scripts/sweep_orphan_generated.js | 128 ++++++++++++++++++++++++++++++++++++++
1 file changed, 128 insertions(+)
diff --git a/scripts/sweep_orphan_generated.js b/scripts/sweep_orphan_generated.js
new file mode 100644
index 0000000..c8322d4
--- /dev/null
+++ b/scripts/sweep_orphan_generated.js
@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+/**
+ * Orphan-PNG sweeper for ~/Projects/wallco-ai/data/generated/.
+ *
+ * "Orphan" = a file on disk that is NOT referenced by any row in
+ * `spoon_all_designs` (joined on basename(local_path)). Such files are leftover
+ * from failed gens, abandoned scrape runs, or rows that were deleted from PG
+ * without unlink.
+ *
+ * Default: DRY RUN — prints count, total bytes, sample 10 filenames, and a
+ * preview of how much disk would be reclaimed. NEVER deletes without --apply.
+ *
+ * Guardrails (apply mode):
+ * • PG reference set must be > 1000 (refuses to nuke when reference list is
+ * suspiciously small — protects against accidental PG outage during scan)
+ * • Skip any file modified in the last 30 minutes (in-flight from generator)
+ * • Print progress every 100 deletes
+ * • Exit if --max-bytes specified and we've freed past that cap
+ *
+ * Usage:
+ * node sweep_orphan_generated.js # dry run
+ * node sweep_orphan_generated.js --apply # actually delete
+ * node sweep_orphan_generated.js --apply --max-bytes 10737418240 # cap at 10 GB
+ *
+ * Idempotent: re-running after --apply just re-scans.
+ */
+
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const ROOT = path.join(__dirname, '..');
+const DIR = path.join(ROOT, 'data', 'generated');
+const DB = 'dw_unified';
+const APPLY = process.argv.includes('--apply');
+const MAX_BYTES_IDX = process.argv.indexOf('--max-bytes');
+const MAX_BYTES = MAX_BYTES_IDX > -1 ? Number(process.argv[MAX_BYTES_IDX + 1]) : Infinity;
+const SAFE_MIN_REF_COUNT = 1000;
+const IN_FLIGHT_GRACE_MS = 30 * 60 * 1000; // skip files modified in last 30 min
+
+function psqlList(sql) {
+ return execSync(`psql ${DB} -At -q -v ON_ERROR_STOP=1`, {
+ input: sql,
+ encoding: 'utf8',
+ maxBuffer: 1024 * 1024 * 256, // 256 MB — list is ~5 MB now, headroom for growth
+ }).trim().split('\n').filter(Boolean);
+}
+
+console.log(`scan dir: ${DIR}`);
+console.log(`mode: ${APPLY ? 'APPLY (will delete)' : 'DRY RUN'}`);
+if (MAX_BYTES !== Infinity) console.log(`max-bytes: ${(MAX_BYTES/1073741824).toFixed(1)} GB`);
+console.log('');
+
+// ── PG reference set ────────────────────────────────────────────────────────
+console.log('reading PG reference list...');
+const referencedPaths = psqlList(`
+ SELECT DISTINCT local_path FROM spoon_all_designs
+ WHERE local_path IS NOT NULL AND local_path LIKE '%/data/generated/%'
+`);
+const referencedNames = new Set(referencedPaths.map(p => path.basename(p)));
+console.log(` PG references ${referencedNames.size} distinct filenames in data/generated/`);
+if (referencedNames.size < SAFE_MIN_REF_COUNT) {
+ console.error(` ABORT: reference count ${referencedNames.size} < safety floor ${SAFE_MIN_REF_COUNT}`);
+ console.error(` This could indicate a PG outage. Refusing to proceed.`);
+ process.exit(2);
+}
+
+// ── on-disk walk ────────────────────────────────────────────────────────────
+console.log('walking data/generated/...');
+const files = fs.readdirSync(DIR);
+const now = Date.now();
+let orphanCount = 0;
+let orphanBytes = 0;
+let skippedInFlight = 0;
+let deleted = 0;
+let deletedBytes = 0;
+const sample = [];
+
+for (const name of files) {
+ if (referencedNames.has(name)) continue;
+ const full = path.join(DIR, name);
+ let st;
+ try { st = fs.statSync(full); }
+ catch { continue; }
+ if (!st.isFile()) continue;
+ if (now - st.mtimeMs < IN_FLIGHT_GRACE_MS) { skippedInFlight++; continue; }
+
+ orphanCount++;
+ orphanBytes += st.size;
+ if (sample.length < 10) sample.push({ name, size: st.size, mtime: new Date(st.mtimeMs).toISOString() });
+
+ if (APPLY) {
+ if (deletedBytes + st.size > MAX_BYTES) {
+ console.log(` hit --max-bytes cap, stopping at ${deleted} files`);
+ break;
+ }
+ try {
+ fs.unlinkSync(full);
+ deleted++;
+ deletedBytes += st.size;
+ if (deleted % 500 === 0) console.log(` deleted ${deleted} files (${(deletedBytes/1073741824).toFixed(2)} GB so far)`);
+ } catch (e) {
+ console.error(` failed to unlink ${name}: ${e.message}`);
+ }
+ }
+}
+
+// ── report ──────────────────────────────────────────────────────────────────
+console.log('');
+console.log('=== summary ===');
+console.log(` files on disk: ${files.length}`);
+console.log(` referenced by PG: ${referencedNames.size}`);
+console.log(` orphan candidates: ${orphanCount}`);
+console.log(` total orphan bytes: ${(orphanBytes/1073741824).toFixed(2)} GB`);
+console.log(` skipped (in-flight): ${skippedInFlight}`);
+if (APPLY) {
+ console.log(` ACTUALLY DELETED: ${deleted} files / ${(deletedBytes/1073741824).toFixed(2)} GB`);
+}
+if (!APPLY && sample.length) {
+ console.log('');
+ console.log('sample 10 orphans (would be deleted):');
+ for (const s of sample) {
+ console.log(` ${(s.size/1048576).toFixed(2).padStart(7)} MB ${s.mtime} ${s.name}`);
+ }
+ console.log('');
+ console.log('to actually delete: re-run with --apply');
+}
← d2d3d8e feat(admin): Crop & Fix tool — drag rect on any design, Gemi
·
back to Wallco Ai
·
refresh_designs_snapshot.py: emit local_path so /designs/img f9b2acc →