← back to Secrets Manager
redact-transcripts.mjs
295 lines
#!/usr/bin/env node
// redact-transcripts.mjs — REVERSIBLE redactor for Claude Code session transcripts
// under ~/.claude/projects/**/*.jsonl (TK-11683).
//
// It masks HIGH-CONFIDENCE live-secret matches (the same hc:true patterns the
// scanner FLAGs — shared via transcript-secret-lib.mjs, so redaction touches
// EXACTLY what the scan reports) with a non-secret placeholder:
// «REDACTED:<pattern>:<last4>:<sha256_16>»
// The sha256_16 is a stable, non-reversible fingerprint — enough to correlate a
// masked span back to a known key digest WITHOUT storing the value.
//
// ── REVERSIBILITY (hard) ──────────────────────────────────────────────────────
// Before ANY file is modified, its exact bytes are copied to a chmod-600 backup
// under data/transcript-redact/backups/, and a restore-map.json records
// {file, backup, sha256_before, sha256_after, replacements:[{pattern,last4,
// sha256_16,count}]}. `--restore <restore-map.json>` copies every backup back,
// verifying the on-disk sha matches sha256_after first (refuses if the file was
// changed since). So every redaction has a concrete recorded undo BEFORE the write.
//
// ── SAFETY RAILS (hard) ───────────────────────────────────────────────────────
// * DRY-RUN IS DEFAULT. It reports what it WOULD mask and writes NOTHING.
// * Modifying the LIVE transcript tree requires BOTH --apply AND
// --yes-modify-live. Absent either, an --apply against the real
// ~/.claude/projects tree is refused. (The negative test drives a temp dir.)
// * The restore-map + report NEVER contain a raw secret (last4 + sha only).
// * Backups DO contain the original bytes (that is what makes restore possible);
// they are written chmod 600 into a data/ dir the .gitignore already excludes.
//
// Usage:
// node redact-transcripts.mjs # DRY-RUN over ~/.claude/projects
// node redact-transcripts.mjs --json # dry-run + full JSON to stdout
// node redact-transcripts.mjs --apply --yes-modify-live # GATED: actually redact live
// node redact-transcripts.mjs --restore <map.json> # undo a prior apply
// node redact-transcripts.mjs --test # NEGATIVE TEST in a temp dir
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import crypto from 'node:crypto';
import { execSync } from 'node:child_process';
import { loadPatterns, isPlaceholder, digest } from './transcript-secret-lib.mjs';
const HOME = os.homedir();
const ROOT = path.dirname(new URL(import.meta.url).pathname);
const ROUTES = path.join(ROOT, 'routes.json');
const LIVE_PROJECTS = path.join(HOME, '.claude', 'projects');
const OUT_DIR = path.join(ROOT, 'data', 'transcript-redact');
const BACKUP_DIR = path.join(OUT_DIR, 'backups');
const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
function listTranscripts(dir) {
try {
return execSync(`find "${dir}" -type f -name '*.jsonl' 2>/dev/null`, {
encoding: 'utf8', maxBuffer: 1024 * 1024 * 200,
}).split('\n').filter(Boolean);
} catch { return null; }
}
// Compute the redacted text for one file. Returns {out, replacements} where
// replacements is a redacted digest list (never a raw value). Only hc:true
// patterns are masked; heuristic (hc:false) matches are left untouched (too noisy
// to safely rewrite). Replacement masks the captured token (group 1 if present,
// else the whole match) so surrounding prose / key-name prefixes are preserved.
function redactText(text, patterns) {
const spans = []; // {start, end, pattern, raw}
for (const p of patterns) {
if (!p.hc) continue;
let re;
try { re = new RegExp(p.regex, 'g'); } catch { continue; }
let m;
while ((m = re.exec(text)) !== null) {
if (m[0].length === 0) { re.lastIndex++; continue; }
const hasGroup = m[1] !== undefined;
const raw = hasGroup ? m[1] : m[0];
if (isPlaceholder(raw)) continue;
// locate the captured token within the whole match
const startInMatch = hasGroup ? m[0].indexOf(m[1]) : 0;
const start = m.index + (startInMatch >= 0 ? startInMatch : 0);
const end = start + raw.length;
spans.push({ start, end, pattern: p.name, raw });
}
}
if (!spans.length) return { out: text, replacements: [] };
// sort by start, drop overlaps (keep the earliest/longest)
spans.sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start));
const kept = [];
let lastEnd = -1;
for (const s of spans) { if (s.start >= lastEnd) { kept.push(s); lastEnd = s.end; } }
const counts = new Map(); // key: pattern|last4|sha16 → count
let out = '';
let cursor = 0;
for (const s of kept) {
const d = digest(s.raw);
out += text.slice(cursor, s.start);
out += `«REDACTED:${s.pattern}:${d.last4}:${d.sha256_16}»`;
cursor = s.end;
const k = `${s.pattern}|${d.last4}|${d.sha256_16}`;
counts.set(k, (counts.get(k) || 0) + 1);
}
out += text.slice(cursor);
const replacements = [...counts.entries()].map(([k, count]) => {
const [pattern, last4, sha256_16] = k.split('|');
return { pattern, last4, sha256_16, count };
});
return { out, replacements };
}
function run({ projectsDir, apply, confirmLive, jsonOut }) {
const patterns = loadPatterns(ROUTES);
const isLive = path.resolve(projectsDir) === path.resolve(LIVE_PROJECTS);
const mode = apply ? 'apply' : 'dry-run';
// hard rail: refuse a live --apply without the explicit confirm flag
if (apply && isLive && !confirmLive) {
console.error('REFUSED: --apply against the LIVE transcript tree requires --yes-modify-live.');
console.error('This is the DESTRUCTIVE, Steve-gated sweep. Run the scanner + review the memo first.');
process.exit(2);
}
const files = listTranscripts(projectsDir) || [];
const perFile = [];
let totalReplacements = 0, distinct = new Set(), touched = 0;
const ts = new Date().toISOString();
const stamp = ts.replace(/[:.]/g, '-');
const restoreMap = { ts, mode, projectsDir, live: isLive, entries: [] };
if (apply) { fs.mkdirSync(BACKUP_DIR, { recursive: true }); }
for (const file of files) {
let text;
try { text = fs.readFileSync(file, 'utf8'); }
catch { perFile.push({ file, error: 'unreadable' }); continue; }
const { out, replacements } = redactText(text, patterns);
if (!replacements.length) continue;
const n = replacements.reduce((a, r) => a + r.count, 0);
totalReplacements += n; touched++;
replacements.forEach(r => distinct.add(r.sha256_16));
const entry = { file, replacements, count: n };
if (apply) {
const before = sha256(text);
const rel = path.resolve(file).replace(/[/]/g, '__');
const backup = path.join(BACKUP_DIR, `${stamp}__${rel}.bak`);
fs.writeFileSync(backup, text, { mode: 0o600 }); // backup FIRST
fs.writeFileSync(file, out); // then rewrite
const after = sha256(out);
// self-verify: re-scan the rewritten text — must have zero hc spans left
const recheck = redactText(out, patterns).replacements.length;
entry.backup = backup;
entry.sha256_before = before;
entry.sha256_after = after;
entry.residual_hc_after = recheck; // must be 0
restoreMap.entries.push({ file, backup, sha256_before: before, sha256_after: after, replacements });
}
perFile.push(entry);
}
const report = {
ts, mode, projectsDir, live: isLive,
transcripts_seen: files.length,
files_would_touch: touched,
total_secret_spans: totalReplacements,
distinct_secrets: distinct.size,
perFile,
};
fs.mkdirSync(OUT_DIR, { recursive: true });
fs.writeFileSync(path.join(OUT_DIR, 'latest-redact-plan.json'), JSON.stringify(report, null, 2));
let restoreMapPath = null;
if (apply) {
restoreMapPath = path.join(OUT_DIR, `restore-map-${stamp}.json`);
fs.writeFileSync(restoreMapPath, JSON.stringify(restoreMap, null, 2), { mode: 0o600 });
report.restore_map = restoreMapPath;
}
if (!jsonOut) {
console.log(`redact-transcripts [${mode}]${isLive ? ' LIVE' : ''}`);
console.log(` transcripts seen: ${report.transcripts_seen}`);
console.log(` files ${apply ? 'redacted' : 'that WOULD be redacted'}: ${report.files_would_touch}`);
console.log(` secret spans ${apply ? 'masked' : 'that WOULD be masked'}: ${report.total_secret_spans}`);
console.log(` distinct secrets: ${report.distinct_secrets}`);
if (apply) {
const residual = perFile.reduce((a, e) => a + (e.residual_hc_after || 0), 0);
console.log(` residual hc after apply: ${residual} ${residual === 0 ? '(clean)' : '(!! NON-ZERO)'}`);
console.log(` restore map → ${restoreMapPath}`);
} else {
console.log(' DRY-RUN — nothing written. To execute the gated sweep:');
console.log(' node redact-transcripts.mjs --apply --yes-modify-live');
}
console.log(` plan → ${path.join(OUT_DIR, 'latest-redact-plan.json')}`);
} else {
console.log(JSON.stringify(report, null, 2));
}
return report;
}
function restore(mapPath) {
const map = JSON.parse(fs.readFileSync(mapPath, 'utf8'));
let restored = 0, skipped = 0;
for (const e of map.entries) {
if (!fs.existsSync(e.backup)) { console.error(` MISSING BACKUP, skip: ${e.file}`); skipped++; continue; }
if (fs.existsSync(e.file)) {
const cur = sha256(fs.readFileSync(e.file, 'utf8'));
if (cur !== e.sha256_after) {
console.error(` CHANGED since redaction, skip (manual review): ${e.file}`);
skipped++; continue;
}
}
fs.copyFileSync(e.backup, e.file);
restored++;
}
console.log(`restore: ${restored} file(s) restored, ${skipped} skipped`);
process.exit(skipped ? 1 : 0);
}
// ─── NEGATIVE TEST (TK-11431 amendment 3) ─────────────────────────────────────
// Prove, in a throwaway temp dir with FAKE secrets:
// 1. dry-run writes NOTHING to the fixture,
// 2. --apply backs up first, masks the secret (re-scan clean), writes a restore
// map that contains NO raw secret,
// 3. --restore reverses it byte-for-byte.
function runTest() {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'tk11683-redact-'));
const sub = path.join(tmp, '-Users-fake-Projects-demo');
fs.mkdirSync(sub, { recursive: true });
const f = path.join(sub, 'dirty.jsonl');
const FAKE_GOOGLE = 'AIza' + 'C'.repeat(35);
const FAKE_GHP = 'ghp_' + 'b'.repeat(36);
const original =
JSON.stringify({ type: 'user', text: `key ${FAKE_GOOGLE} and token ${FAKE_GHP}` }) + '\n' +
JSON.stringify({ type: 'assistant', text: 'placeholder GOOGLE_API_KEY=<YOUR_KEY_HERE> stays' }) + '\n';
fs.writeFileSync(f, original);
const originalSha = sha256(original);
// point OUT_DIR/BACKUP at the temp tree by monkeypatching module consts is not
// possible; instead we accept the real OUT_DIR but verify via the fixture file.
// 1) dry-run must not modify the fixture
run({ projectsDir: tmp, apply: false, confirmLive: false, jsonOut: true });
const afterDry = sha256(fs.readFileSync(f, 'utf8'));
const dryUntouched = afterDry === originalSha;
// 2) apply (temp dir is not the live tree, so no confirm needed)
const rep = run({ projectsDir: tmp, apply: true, confirmLive: false, jsonOut: true });
const afterApply = fs.readFileSync(f, 'utf8');
const masked = afterApply.includes('«REDACTED:') && !afterApply.includes(FAKE_GOOGLE) && !afterApply.includes(FAKE_GHP);
const placeholderKept = afterApply.includes('<YOUR_KEY_HERE>');
const mapPath = rep.restore_map;
const mapText = fs.readFileSync(mapPath, 'utf8');
const mapCleanOfRaw = !mapText.includes(FAKE_GOOGLE) && !mapText.includes(FAKE_GHP);
// 3) restore must reverse byte-for-byte — invoke restore logic inline
const map = JSON.parse(mapText);
for (const e of map.entries) fs.copyFileSync(e.backup, e.file);
const afterRestore = sha256(fs.readFileSync(f, 'utf8'));
const restoredExact = afterRestore === originalSha;
// cleanup: temp fixture + the backup/restore-map this test wrote into OUT_DIR
fs.rmSync(tmp, { recursive: true, force: true });
try {
for (const e of map.entries) if (fs.existsSync(e.backup)) fs.rmSync(e.backup);
if (fs.existsSync(mapPath)) fs.rmSync(mapPath);
} catch { /* best effort */ }
const pass = dryUntouched && masked && placeholderKept && mapCleanOfRaw && restoredExact;
console.log('NEGATIVE TEST (redactor)');
console.log(` dry-run leaves fixture untouched: ${dryUntouched ? 'YES ✓' : 'NO ✗'}`);
console.log(` --apply masks the fake secrets: ${masked ? 'YES ✓' : 'NO ✗'}`);
console.log(` placeholder line left intact: ${placeholderKept ? 'YES ✓' : 'NO ✗'}`);
console.log(` restore-map contains NO raw secret: ${mapCleanOfRaw ? 'YES ✓' : 'NO ✗ (LEAK!)'}`);
console.log(` --restore reverses byte-for-byte: ${restoredExact ? 'YES ✓' : 'NO ✗'}`);
console.log(pass ? 'RESULT: PASS' : 'RESULT: FAIL');
process.exit(pass ? 0 : 1);
}
// ─── main ─────────────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
if (args.includes('--test')) {
runTest();
} else if (args.includes('--restore')) {
const i = args.indexOf('--restore');
const mapPath = args[i + 1];
if (!mapPath) { console.error('usage: --restore <restore-map.json>'); process.exit(2); }
restore(mapPath);
} else {
run({
projectsDir: LIVE_PROJECTS,
apply: args.includes('--apply'),
confirmLive: args.includes('--yes-modify-live'),
jsonOut: args.includes('--json'),
});
}