← back to Wallco Ai
add ghost-rescore-flagged.js — quantify detector-change rescue rate
a3c2b00d06d395a8ad0ca731572695d81bffdb8d · 2026-05-23 23:49:59 -0700 · Steve Abrams
Diagnostic script that samples N previously-flagged IDs (optionally
filtered by category), resolves local_path via psql, re-runs through
analyzeGhostLayerStable, and reports the flip rate.
Validated Fix 47f7a9c on a 100-design damask sample:
- 52% rescued (flipped TRUE → FALSE, all unanimous 3/3)
- 48% still flagged TRUE (all unanimous, reasons cite "faded duplicate
motifs" — these are the real Python-pipeline ghost layers)
- 0 split votes, 0 errors
Pattern observed in sample: rescued IDs cluster low (1722, 3953, 4049,
4722, 4814), still-flagged IDs cluster high (36029, 38205, 38359,
38379, 38638). Consistent with the make_seamless.py regression theory —
older damasks are clean classical-engraving designs that the old single-
call detector mis-flagged; newer damasks have real seam-blending ghost
artifacts.
Detail JSONL written to data/ghost-rescore-<cat>-<ts>.jsonl per run.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Files touched
A scripts/ghost-rescore-flagged.js
Diff
commit a3c2b00d06d395a8ad0ca731572695d81bffdb8d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat May 23 23:49:59 2026 -0700
add ghost-rescore-flagged.js — quantify detector-change rescue rate
Diagnostic script that samples N previously-flagged IDs (optionally
filtered by category), resolves local_path via psql, re-runs through
analyzeGhostLayerStable, and reports the flip rate.
Validated Fix 47f7a9c on a 100-design damask sample:
- 52% rescued (flipped TRUE → FALSE, all unanimous 3/3)
- 48% still flagged TRUE (all unanimous, reasons cite "faded duplicate
motifs" — these are the real Python-pipeline ghost layers)
- 0 split votes, 0 errors
Pattern observed in sample: rescued IDs cluster low (1722, 3953, 4049,
4722, 4814), still-flagged IDs cluster high (36029, 38205, 38359,
38379, 38638). Consistent with the make_seamless.py regression theory —
older damasks are clean classical-engraving designs that the old single-
call detector mis-flagged; newer damasks have real seam-blending ghost
artifacts.
Detail JSONL written to data/ghost-rescore-<cat>-<ts>.jsonl per run.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
scripts/ghost-rescore-flagged.js | 148 +++++++++++++++++++++++++++++++++++++++
1 file changed, 148 insertions(+)
diff --git a/scripts/ghost-rescore-flagged.js b/scripts/ghost-rescore-flagged.js
new file mode 100644
index 0000000..88df386
--- /dev/null
+++ b/scripts/ghost-rescore-flagged.js
@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+// ghost-rescore-flagged — re-test previously-flagged designs through the
+// current ghost-detector pipeline. Diagnostic tool for validating detector
+// changes: if Fix N drops the false-positive rate, this script quantifies it.
+//
+// Usage:
+// node scripts/ghost-rescore-flagged.js --category damask --sample 100
+// node scripts/ghost-rescore-flagged.js --category damask --sample 100 --concurrency 4
+//
+// Reads previously-flagged IDs from data/ghost-scan-flagged.jsonl, samples N,
+// resolves local_path via psql, re-runs through analyzeGhostLayerStable, and
+// reports the flip rate (was TRUE under old detector → now FALSE under current).
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { analyzeGhostLayerStable } = require('../lib/ghost-detector');
+
+const ARGS = process.argv.slice(2);
+const arg = (n, d) => { const i = ARGS.indexOf('--' + n); return i >= 0 ? ARGS[i + 1] : d; };
+const CATEGORY = arg('category', null);
+const SAMPLE = parseInt(arg('sample', '100'), 10);
+const CONCURRENCY = Math.max(1, parseInt(arg('concurrency', '4'), 10));
+
+const ROOT = path.join(__dirname, '..');
+const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
+
+function psql(sql) {
+ const r = spawnSync('psql', ['dw_unified', '-At', '-q'], {
+ input: sql, encoding: 'utf8', maxBuffer: 50_000_000,
+ });
+ if (r.status !== 0) throw new Error(r.stderr || r.stdout || 'psql failed');
+ return r.stdout;
+}
+
+function loadFlaggedIds() {
+ const ids = [];
+ for (const line of fs.readFileSync(FLAGGED, 'utf8').split('\n')) {
+ if (!line.trim()) continue;
+ try {
+ const j = JSON.parse(line);
+ if (CATEGORY && j.category !== CATEGORY) continue;
+ ids.push(j.id);
+ } catch {}
+ }
+ return ids;
+}
+
+// Fisher-Yates sample-without-replacement
+function sample(arr, n) {
+ const out = arr.slice();
+ for (let i = out.length - 1; i > 0; i--) {
+ const j = Math.floor(Math.random() * (i + 1));
+ [out[i], out[j]] = [out[j], out[i]];
+ }
+ return out.slice(0, n);
+}
+
+function resolveLocalPaths(ids) {
+ if (!ids.length) return new Map();
+ const inList = ids.join(',');
+ const raw = psql(`SELECT id, category, local_path
+ FROM spoon_all_designs
+ WHERE id IN (${inList}) AND local_path IS NOT NULL`);
+ const map = new Map();
+ for (const line of raw.split('\n')) {
+ if (!line.trim()) continue;
+ const [id, category, local_path] = line.split('|');
+ map.set(parseInt(id, 10), { category, local_path });
+ }
+ return map;
+}
+
+async function runPool(items, workers, fn) {
+ let cursor = 0;
+ const out = [];
+ const next = () => (cursor < items.length ? items[cursor++] : null);
+ const loop = async () => {
+ while (true) {
+ const it = next();
+ if (!it) return;
+ try { out.push(await fn(it)); }
+ catch (e) { out.push({ id: it.id, error: e.message.slice(0, 200) }); }
+ }
+ };
+ await Promise.all(Array.from({ length: workers }, loop));
+ return out;
+}
+
+(async () => {
+ console.log(`[rescore] category=${CATEGORY || 'any'} sample=${SAMPLE} concurrency=${CONCURRENCY}`);
+ const all = loadFlaggedIds();
+ console.log(`[rescore] previously-flagged pool: ${all.length}`);
+ if (all.length === 0) { console.log('nothing to rescore'); process.exit(0); }
+
+ const ids = sample(all, Math.min(SAMPLE, all.length));
+ const meta = resolveLocalPaths(ids);
+ const rows = ids
+ .map(id => ({ id, ...(meta.get(id) || {}) }))
+ .filter(r => r.local_path && fs.existsSync(r.local_path));
+ console.log(`[rescore] resolved local paths: ${rows.length}/${ids.length}`);
+
+ const t0 = Date.now();
+ let done = 0;
+ const results = await runPool(rows, CONCURRENCY, async (r) => {
+ const v = await analyzeGhostLayerStable(r.local_path, { category: r.category });
+ done++;
+ if (done % 10 === 0) {
+ const rate = done / ((Date.now() - t0) / 1000);
+ console.log(`[rescore] ${done}/${rows.length} · ${rate.toFixed(1)}/s`);
+ }
+ return { id: r.id, category: r.category, ...v };
+ });
+
+ const ok = results.filter(r => !r.error);
+ const errored = results.filter(r => r.error);
+ const stillTrue = ok.filter(r => r.hasGhostLayer);
+ const flipped = ok.filter(r => !r.hasGhostLayer);
+ const unanimousTrue = stillTrue.filter(r => r.unanimous).length;
+ const unanimousFalse = flipped.filter(r => r.unanimous).length;
+ const splitVotes = ok.filter(r => !r.unanimous).length;
+
+ console.log('\n=== rescore summary ===');
+ console.log(`tested : ${ok.length}`);
+ console.log(`errored : ${errored.length}`);
+ console.log(`still flagged TRUE: ${stillTrue.length} (unanimous: ${unanimousTrue})`);
+ console.log(`now FALSE (flipped): ${flipped.length} (unanimous: ${unanimousFalse})`);
+ console.log(`split votes : ${splitVotes}`);
+ console.log(`rescue rate : ${((flipped.length / ok.length) * 100).toFixed(1)}% of previously-flagged now read clean`);
+
+ if (stillTrue.length > 0) {
+ console.log('\nsample IDs still flagged TRUE (review these — likely real ghost layers):');
+ for (const r of stillTrue.slice(0, 5)) {
+ console.log(` id=${r.id} consensus=${r.confidence.toFixed(2)} reason=${r.reason}`);
+ }
+ }
+ if (flipped.length > 0) {
+ console.log('\nsample IDs flipped to FALSE (rescued false-positives):');
+ for (const r of flipped.slice(0, 5)) {
+ console.log(` id=${r.id} consensus=${r.confidence.toFixed(2)} reason=${r.reason}`);
+ }
+ }
+
+ // Write detail file for follow-up
+ const outFile = path.join(ROOT, 'data', `ghost-rescore-${CATEGORY || 'all'}-${Date.now()}.jsonl`);
+ fs.writeFileSync(outFile, results.map(r => JSON.stringify(r)).join('\n') + '\n');
+ console.log(`\nfull results: ${outFile}`);
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
← 7031abf ghost-review: magic wand — click motif to isolate, then reco
·
back to Wallco Ai
·
admin: live before/after Fixes Feed for every repair event 080cc68 →