← back to Wallco Ai
geometric pilot: bulk-verify ghost-detector (stable 3-vote) — false-fail 100%→52.6%
e0387ff8ce280666307bd902fa0c6aaf325d4101 · 2026-05-26 10:02:53 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Files touched
A scripts/verify-geometric-pilot.js
Diff
commit e0387ff8ce280666307bd902fa0c6aaf325d4101
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue May 26 10:02:53 2026 -0700
geometric pilot: bulk-verify ghost-detector (stable 3-vote) — false-fail 100%→52.6%
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
scripts/verify-geometric-pilot.js | 129 ++++++++++++++++++++++++++++++++++++++
1 file changed, 129 insertions(+)
diff --git a/scripts/verify-geometric-pilot.js b/scripts/verify-geometric-pilot.js
new file mode 100644
index 0000000..b9b6ffd
--- /dev/null
+++ b/scripts/verify-geometric-pilot.js
@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+// verify-geometric-pilot — bulk-run the STABLE ghost detector across every
+// `category='geometric-block-leaves'` design and report the false-fail rate.
+//
+// Context: the OLD single-call ghost detector false-flagged 20/20 geometric
+// pilot designs as hasGhostLayer (memory project_geometric_block_leaves_20260513).
+// The 3-vote stable detector (lib/ghost-detector.js analyzeGhostLayerStable,
+// 2026-05-23) is supposed to fix that. This script measures the new rate.
+//
+// These designs are intentionally flat single-ink geometric leaf repeats —
+// the ground truth is hasGhostLayer=FALSE for all of them. So every TRUE the
+// detector returns is a FALSE-FAIL.
+//
+// Batch-style (script loops, not the LLM) so cost is fixed regardless of count.
+// node scripts/verify-geometric-pilot.js
+// node scripts/verify-geometric-pilot.js --concurrency 4
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { analyzeGhostLayerStable } = require('../lib/ghost-detector');
+
+const ARGS = process.argv.slice(2);
+const arg = (n, d) => { const i = ARGS.indexOf('--' + n); return i >= 0 ? ARGS[i + 1] : d; };
+const CONCURRENCY = Math.max(1, parseInt(arg('concurrency', '4'), 10));
+const CATEGORY = 'geometric-block-leaves';
+
+const ROOT = path.join(__dirname, '..');
+
+function psql(sql) {
+ const onLinux = process.platform === 'linux';
+ const cmd = onLinux ? 'sudo' : 'psql';
+ const a = onLinux
+ ? ['-n', '-u', 'postgres', 'psql', 'dw_unified', '-At', '-F', '|', '-q']
+ : ['dw_unified', '-At', '-F', '|', '-q'];
+ const r = spawnSync(cmd, a, { input: sql, encoding: 'utf8', maxBuffer: 50_000_000 });
+ if (r.status !== 0) throw new Error(r.stderr || r.stdout || 'psql failed');
+ return r.stdout;
+}
+
+function fetchTargets() {
+ const sql = `SELECT id, local_path, COALESCE(kind,'')
+ FROM spoon_all_designs
+ WHERE category = '${CATEGORY}'
+ ORDER BY id`;
+ const rows = [];
+ for (const line of psql(sql).split('\n')) {
+ if (!line.trim()) continue;
+ const [id, local_path, kind] = line.split('|');
+ rows.push({ id: parseInt(id, 10), local_path, kind });
+ }
+ return rows;
+}
+
+async function mapPool(items, n, fn) {
+ const out = new Array(items.length);
+ let idx = 0;
+ await Promise.all(Array.from({ length: n }, async () => {
+ while (idx < items.length) {
+ const i = idx++;
+ out[i] = await fn(items[i], i);
+ }
+ }));
+ return out;
+}
+
+(async () => {
+ const targets = fetchTargets();
+ console.error(`[verify] ${targets.length} ${CATEGORY} designs · stable 3-vote ghost detector · concurrency ${CONCURRENCY}`);
+
+ const results = await mapPool(targets, CONCURRENCY, async (d) => {
+ if (!d.local_path || !fs.existsSync(d.local_path)) {
+ return { ...d, error: 'file missing', hasGhostLayer: null };
+ }
+ try {
+ const r = await analyzeGhostLayerStable(d.local_path, { category: CATEGORY });
+ process.stderr.write(r.hasGhostLayer ? (r.unanimous ? 'X' : 'x') : '.');
+ return {
+ id: d.id, kind: d.kind,
+ hasGhostLayer: r.hasGhostLayer,
+ unanimous: r.unanimous,
+ consensus: r.confidence,
+ votes: r.votes.map(v => v.hasGhost ? 'G' : '-').join(''),
+ errored: r.errored,
+ reason: r.reason,
+ cost: r.cost_estimate,
+ };
+ } catch (e) {
+ process.stderr.write('!');
+ return { id: d.id, kind: d.kind, error: e.message, hasGhostLayer: null };
+ }
+ });
+ process.stderr.write('\n');
+
+ const ok = results.filter(r => r.hasGhostLayer !== null);
+ const errs = results.filter(r => r.hasGhostLayer === null);
+ const flagged = ok.filter(r => r.hasGhostLayer); // any majority TRUE
+ const flaggedUnan = flagged.filter(r => r.unanimous); // unanimous 3/3 TRUE — what the gate blocks on
+ const split = ok.filter(r => !r.unanimous); // 2/1 borderline
+ const totalCost = ok.reduce((s, r) => s + (r.cost || 0), 0);
+
+ const N = ok.length;
+ const pct = (x) => N ? (100 * x / N).toFixed(1) + '%' : 'n/a';
+
+ const summary = {
+ category: CATEGORY,
+ total_designs: results.length,
+ scored: N,
+ errored: errs.length,
+ ground_truth: 'hasGhostLayer=FALSE for all (flat single-ink geometric leaf repeats) — every TRUE is a false-fail',
+ new_detector: {
+ flagged_any_majority: flagged.length,
+ flagged_unanimous_3of3: flaggedUnan.length,
+ split_2of1: split.length,
+ false_fail_rate_majority: pct(flagged.length),
+ false_fail_rate_gate_blocking: pct(flaggedUnan.length),
+ },
+ old_detector_baseline: '20/20 (100%) false-fail (single-call, pre-2026-05-23)',
+ cost_usd_estimate: Number(totalCost.toFixed(4)),
+ };
+
+ const out = { generated_at: new Date().toISOString(), summary, results };
+ const date = new Date().toISOString().slice(0, 10);
+ const jsonPath = path.join(ROOT, 'data', `geometric-pilot-verification-${date}.json`);
+ fs.writeFileSync(jsonPath, JSON.stringify(out, null, 2));
+ console.error(`[verify] wrote ${jsonPath}`);
+
+ console.log(JSON.stringify(summary, null, 2));
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
← 66addf6 Broaden seamless-comfy guard to kind==='seamless_tile' (cove
·
back to Wallco Ai
·
edge_seam_heal: combined mids+edges offset-heal for the edge 9ca0509 →