`scripts/dedupe-flagged-jsonl.js`

93 lines
#!/usr/bin/env node
// dedupe-flagged-jsonl — clean up data/ghost-scan-flagged.jsonl after the
// post-c4ac1f0 scan finishes.
//
// Two sources of pollution can land in this file:
//   (a) rogue old-pipeline scans (no `unanimous` field) — these are SINGLE-call
//       Gemini verdicts before the analyzeGhostLayerStable fix landed
//   (b) duplicate IDs from overlapping scan processes
//
// Keep policy:
//   - keep only entries WITH `unanimous` field (= new-pipeline 3-vote majority)
//   - last-wins per id (the most recent verdict supersedes earlier ones)
//   - preserve the pre-c4ac1f0 backup intact
//
// Default is dry-run. Pass --apply to actually overwrite the file.
//
// SAFETY: refuses to run if a scan-ghost-layer process is still active —
// don't want to truncate a file the scan is appending to.

const fs   = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');

const ROOT    = path.join(__dirname, '..');
const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
const APPLY   = process.argv.includes('--apply');
const FORCE   = process.argv.includes('--force');

function scanRunning() {
  const r = spawnSync('pgrep', ['-f', 'node scripts/scan-ghost-layer.js'], { encoding: 'utf8' });
  return r.stdout.trim().length > 0;
}

if (!fs.existsSync(FLAGGED)) {
  console.error(`MISSING: ${FLAGGED}`);
  process.exit(1);
}

if (APPLY && scanRunning() && !FORCE) {
  console.error('REFUSING: a scan-ghost-layer.js process is still running.');
  console.error('Wait for the scan to finish, or pass --force to truncate anyway.');
  process.exit(2);
}

console.log(`[dedupe] reading ${FLAGGED}`);
const lines = fs.readFileSync(FLAGGED, 'utf8').split('\n').filter(Boolean);
console.log(`[dedupe] ${lines.length} total lines`);

// Two-pass: count format buckets first
let newFmt = 0, oldFmt = 0, badJson = 0;
for (const ln of lines) {
  try {
    const j = JSON.parse(ln);
    if ('unanimous' in j) newFmt++;
    else                  oldFmt++;
  } catch { badJson++; }
}
console.log(`[dedupe] new-format (with unanimous): ${newFmt}`);
console.log(`[dedupe] old-format (no unanimous):   ${oldFmt}`);
console.log(`[dedupe] bad JSON:                    ${badJson}`);

// Build last-wins map of new-format entries
const latestById = new Map();
let lineNo = 0;
for (const ln of lines) {
  lineNo++;
  try {
    const j = JSON.parse(ln);
    if (!('unanimous' in j)) continue;        // drop old-format
    if (!Number.isFinite(j.id)) continue;     // drop malformed
    // Preserve insertion order via Map; reinserting moves to end
    latestById.set(j.id, ln);
  } catch { /* skip */ }
}
console.log(`[dedupe] unique new-format IDs: ${latestById.size}`);
console.log(`[dedupe] would write ${latestById.size} lines (was ${lines.length})`);

if (!APPLY) {
  console.log(`\nDRY RUN — pass --apply to overwrite ${FLAGGED}`);
  process.exit(0);
}

// Backup + atomic write
const ts = new Date().toISOString().replace(/[:.]/g, '-');
const backupPath = FLAGGED + '.pre-dedupe-' + ts;
fs.copyFileSync(FLAGGED, backupPath);
console.log(`[dedupe] backup → ${backupPath}`);

const tmp = FLAGGED + '.tmp-dedupe';
fs.writeFileSync(tmp, [...latestById.values()].join('\n') + '\n');
fs.renameSync(tmp, FLAGGED);
console.log(`[dedupe] wrote ${FLAGGED} (${latestById.size} lines)`);