add dedupe-flagged-jsonl.js + watch-and-dedupe.sh

a7d17454c5791e95e55815e28cbac30ef9e7ee9d · 2026-05-24 01:02:58 -0700 · Steve Abrams

Post-scan cleanup. The current data/ghost-scan-flagged.jsonl is a polluted
mix:
  - 4,301 old-format entries (no 'unanimous' field) from a rogue
    scan-ghost-layer.js process (PID 11544) that started before the
    detector fixes landed
  - 337 new-format entries (with 'unanimous') from the current sweep

Plan: let the current scan finish, then drop the old-format entries and
dedupe by id (last-wins). Keeps the .pre-47f7a9c backup intact for
forensics; never touches it.

scripts/dedupe-flagged-jsonl.js — one-shot cleanup utility, dry-run by
default, refuses to truncate while a scan process is active.

scripts/watch-and-dedupe.sh — polls every 2 min, runs dedupe --apply when
the scan process exits.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files touched

A scripts/dedupe-flagged-jsonl.js
A scripts/watch-and-dedupe.sh

Diff

commit a7d17454c5791e95e55815e28cbac30ef9e7ee9d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun May 24 01:02:58 2026 -0700

    add dedupe-flagged-jsonl.js + watch-and-dedupe.sh
    
    Post-scan cleanup. The current data/ghost-scan-flagged.jsonl is a polluted
    mix:
      - 4,301 old-format entries (no 'unanimous' field) from a rogue
        scan-ghost-layer.js process (PID 11544) that started before the
        detector fixes landed
      - 337 new-format entries (with 'unanimous') from the current sweep
    
    Plan: let the current scan finish, then drop the old-format entries and
    dedupe by id (last-wins). Keeps the .pre-47f7a9c backup intact for
    forensics; never touches it.
    
    scripts/dedupe-flagged-jsonl.js — one-shot cleanup utility, dry-run by
    default, refuses to truncate while a scan process is active.
    
    scripts/watch-and-dedupe.sh — polls every 2 min, runs dedupe --apply when
    the scan process exits.
    
    Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 scripts/dedupe-flagged-jsonl.js | 92 +++++++++++++++++++++++++++++++++++++++++
 scripts/watch-and-dedupe.sh     | 26 ++++++++++++
 2 files changed, 118 insertions(+)

diff --git a/scripts/dedupe-flagged-jsonl.js b/scripts/dedupe-flagged-jsonl.js
new file mode 100644
index 0000000..0fc5c34
--- /dev/null
+++ b/scripts/dedupe-flagged-jsonl.js
@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+// dedupe-flagged-jsonl — clean up data/ghost-scan-flagged.jsonl after the
+// post-c4ac1f0 scan finishes.
+//
+// Two sources of pollution can land in this file:
+//   (a) rogue old-pipeline scans (no `unanimous` field) — these are SINGLE-call
+//       Gemini verdicts before the analyzeGhostLayerStable fix landed
+//   (b) duplicate IDs from overlapping scan processes
+//
+// Keep policy:
+//   - keep only entries WITH `unanimous` field (= new-pipeline 3-vote majority)
+//   - last-wins per id (the most recent verdict supersedes earlier ones)
+//   - preserve the pre-c4ac1f0 backup intact
+//
+// Default is dry-run. Pass --apply to actually overwrite the file.
+//
+// SAFETY: refuses to run if a scan-ghost-layer process is still active —
+// don't want to truncate a file the scan is appending to.
+
+const fs   = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const ROOT    = path.join(__dirname, '..');
+const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
+const APPLY   = process.argv.includes('--apply');
+const FORCE   = process.argv.includes('--force');
+
+function scanRunning() {
+  const r = spawnSync('pgrep', ['-f', 'node scripts/scan-ghost-layer.js'], { encoding: 'utf8' });
+  return r.stdout.trim().length > 0;
+}
+
+if (!fs.existsSync(FLAGGED)) {
+  console.error(`MISSING: ${FLAGGED}`);
+  process.exit(1);
+}
+
+if (APPLY && scanRunning() && !FORCE) {
+  console.error('REFUSING: a scan-ghost-layer.js process is still running.');
+  console.error('Wait for the scan to finish, or pass --force to truncate anyway.');
+  process.exit(2);
+}
+
+console.log(`[dedupe] reading ${FLAGGED}`);
+const lines = fs.readFileSync(FLAGGED, 'utf8').split('\n').filter(Boolean);
+console.log(`[dedupe] ${lines.length} total lines`);
+
+// Two-pass: count format buckets first
+let newFmt = 0, oldFmt = 0, badJson = 0;
+for (const ln of lines) {
+  try {
+    const j = JSON.parse(ln);
+    if ('unanimous' in j) newFmt++;
+    else                  oldFmt++;
+  } catch { badJson++; }
+}
+console.log(`[dedupe] new-format (with unanimous): ${newFmt}`);
+console.log(`[dedupe] old-format (no unanimous):   ${oldFmt}`);
+console.log(`[dedupe] bad JSON:                    ${badJson}`);
+
+// Build last-wins map of new-format entries
+const latestById = new Map();
+let lineNo = 0;
+for (const ln of lines) {
+  lineNo++;
+  try {
+    const j = JSON.parse(ln);
+    if (!('unanimous' in j)) continue;        // drop old-format
+    if (!Number.isFinite(j.id)) continue;     // drop malformed
+    // Preserve insertion order via Map; reinserting moves to end
+    latestById.set(j.id, ln);
+  } catch { /* skip */ }
+}
+console.log(`[dedupe] unique new-format IDs: ${latestById.size}`);
+console.log(`[dedupe] would write ${latestById.size} lines (was ${lines.length})`);
+
+if (!APPLY) {
+  console.log(`\nDRY RUN — pass --apply to overwrite ${FLAGGED}`);
+  process.exit(0);
+}
+
+// Backup + atomic write
+const ts = new Date().toISOString().replace(/[:.]/g, '-');
+const backupPath = FLAGGED + '.pre-dedupe-' + ts;
+fs.copyFileSync(FLAGGED, backupPath);
+console.log(`[dedupe] backup → ${backupPath}`);
+
+const tmp = FLAGGED + '.tmp-dedupe';
+fs.writeFileSync(tmp, [...latestById.values()].join('\n') + '\n');
+fs.renameSync(tmp, FLAGGED);
+console.log(`[dedupe] wrote ${FLAGGED} (${latestById.size} lines)`);
diff --git a/scripts/watch-and-dedupe.sh b/scripts/watch-and-dedupe.sh
new file mode 100755
index 0000000..67ae668
--- /dev/null
+++ b/scripts/watch-and-dedupe.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# watch-and-dedupe — poll until scan-ghost-layer.js finishes, then dedupe.
+#
+# Tail-logs to data/dedupe-watcher.log so progress is visible from another
+# terminal. Designed to be nohup'd:
+#   nohup bash scripts/watch-and-dedupe.sh > data/dedupe-watcher.log 2>&1 &
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+LOG=data/dedupe-watcher.log
+echo "[watcher] $(date '+%F %T') starting · pid=$$"
+ITER=0
+while pgrep -f "node scripts/scan-ghost-layer.js" > /dev/null 2>&1; do
+  ITER=$((ITER + 1))
+  if [ $((ITER % 15)) -eq 1 ]; then
+    # log every ~30 min so the file shows liveness
+    echo "[watcher] $(date '+%F %T') scan still running (iter=$ITER)"
+  fi
+  sleep 120
+done
+echo "[watcher] $(date '+%F %T') scan finished — running dedupe"
+echo "── dry-run preview ──"
+node scripts/dedupe-flagged-jsonl.js
+echo "── apply ──"
+node scripts/dedupe-flagged-jsonl.js --apply
+echo "[watcher] $(date '+%F %T') done"

← 06ad933 fix(regenerate-reverse): fall back to stored prompt when sou · back to Wallco Ai · fix(make_seamless): hard-cut mask threshold to eliminate gho 153962b →