← back to Wallco Ai
ghost-detector: 429 retry-with-backoff + triage-ghost-flagged.js
788de56156a4dd8e2d33dc79b932b892cd379ed0 · 2026-05-23 22:38:22 -0700 · Steve Abrams
Two pieces:
1. lib/ghost-detector.js — wrap the Gemini fetch in retry-on-429/5xx with
exponential backoff (1s/2s/4s, max 3 retries, ±500ms jitter). The triage
sweep was hitting 15-25% 429 rate-limit errors at 54-concurrent calls;
short pause + retry recovers without manual re-runs. Same behavior for
the pre-publish gate (verifyNoGhostLayer) — no more spurious gate
failures under burst load.
2. scripts/triage-ghost-flagged.js — re-verify each row in
data/ghost-scan-flagged.jsonl using analyzeGhostLayerStable (3x votes,
majority rule). Buckets into CONFIRMED (3/3 true), BORDERLINE (2/1
split), CLEARED (3/3 false). Writes to data/ghost-scan-triage.jsonl
(resumable). Image resolution handles both pg-source (PG local_path
lookup) and json-source (designs.json filename lookup) rows.
Sample of 300 (100 each from damask, drunk-animals, designer-zoo-calm):
~76-85% CONFIRMED, 0% CLEARED, rest borderline/error. The single-vote scan
was conservative but accurate on the images it flagged — damask false-
positive theory disproved.
Files touched
M lib/ghost-detector.jsA scripts/triage-ghost-flagged.js
Diff
commit 788de56156a4dd8e2d33dc79b932b892cd379ed0
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat May 23 22:38:22 2026 -0700
ghost-detector: 429 retry-with-backoff + triage-ghost-flagged.js
Two pieces:
1. lib/ghost-detector.js — wrap the Gemini fetch in retry-on-429/5xx with
exponential backoff (1s/2s/4s, max 3 retries, ±500ms jitter). The triage
sweep was hitting 15-25% 429 rate-limit errors at 54-concurrent calls;
short pause + retry recovers without manual re-runs. Same behavior for
the pre-publish gate (verifyNoGhostLayer) — no more spurious gate
failures under burst load.
2. scripts/triage-ghost-flagged.js — re-verify each row in
data/ghost-scan-flagged.jsonl using analyzeGhostLayerStable (3x votes,
majority rule). Buckets into CONFIRMED (3/3 true), BORDERLINE (2/1
split), CLEARED (3/3 false). Writes to data/ghost-scan-triage.jsonl
(resumable). Image resolution handles both pg-source (PG local_path
lookup) and json-source (designs.json filename lookup) rows.
Sample of 300 (100 each from damask, drunk-animals, designer-zoo-calm):
~76-85% CONFIRMED, 0% CLEARED, rest borderline/error. The single-vote scan
was conservative but accurate on the images it flagged — damask false-
positive theory disproved.
---
lib/ghost-detector.js | 24 +++--
scripts/triage-ghost-flagged.js | 218 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 235 insertions(+), 7 deletions(-)
diff --git a/lib/ghost-detector.js b/lib/ghost-detector.js
index 1dadb1f..67cf5dd 100644
--- a/lib/ghost-detector.js
+++ b/lib/ghost-detector.js
@@ -118,14 +118,24 @@ async function analyzeGhostLayer(imagePath, opts = {}) {
},
};
- const r = await fetch(url, {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify(body),
- });
+ // Retry-on-429 (and 5xx) with exponential backoff. Gemini Flash returns
+ // 429 "Resource exhausted" under high parallel load; one short pause is
+ // usually enough. Max 3 retries: 1s, 2s, 4s.
+ let r, lastTxt = '';
+ for (let attempt = 0; attempt < 4; attempt++) {
+ r = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ if (r.ok) break;
+ lastTxt = await r.text().catch(() => '');
+ if (r.status !== 429 && r.status < 500) break; // non-retryable
+ if (attempt === 3) break;
+ await new Promise(res => setTimeout(res, 1000 * Math.pow(2, attempt) + Math.random() * 500));
+ }
if (!r.ok) {
- const txt = await r.text().catch(() => '');
- throw new Error(`gemini ${r.status}: ${txt.slice(0, 200)}`);
+ throw new Error(`gemini ${r.status}: ${lastTxt.slice(0, 200)}`);
}
const j = await r.json();
const text = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
diff --git a/scripts/triage-ghost-flagged.js b/scripts/triage-ghost-flagged.js
new file mode 100755
index 0000000..8c79441
--- /dev/null
+++ b/scripts/triage-ghost-flagged.js
@@ -0,0 +1,218 @@
+#!/usr/bin/env node
+// triage-ghost-flagged — re-verify each row in data/ghost-scan-flagged.jsonl
+// using analyzeGhostLayerStable (3x parallel votes, majority rule).
+//
+// The original scan used single-vote analyzeGhostLayer which has a ~20-30%
+// flip rate on borderline cases. Re-running with the stable variant tells us
+// which flags are real (CONFIRMED), which are borderline (need human eyes),
+// and which are noise from the single-vote era (CLEARED).
+//
+// Resumable — skips ids already in data/ghost-scan-triage.jsonl.
+//
+// Usage:
+// node scripts/triage-ghost-flagged.js # all flagged
+// node scripts/triage-ghost-flagged.js --limit 100 # sample
+// node scripts/triage-ghost-flagged.js --category damask # one category
+// node scripts/triage-ghost-flagged.js --concurrency 6 # parallelism
+//
+// Output schema (per id):
+// { id, ts, bucket: 'CONFIRMED'|'BORDERLINE'|'CLEARED',
+// stable: { hasGhostLayer, confidence, unanimous, votes, reason },
+// original_confidence, category, source }
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { analyzeGhostLayerStable } = require('../lib/ghost-detector');
+
+const ARGS = process.argv.slice(2);
+function arg(name, def) {
+ const i = ARGS.indexOf('--' + name);
+ return i >= 0 ? ARGS[i + 1] : def;
+}
+const LIMIT = parseInt(arg('limit', '0'), 10);
+const CONCURRENCY = Math.max(1, parseInt(arg('concurrency', '4'), 10));
+const CATEGORY = arg('category', null);
+
+const ROOT = path.join(__dirname, '..');
+const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
+const TRIAGE = path.join(ROOT, 'data', 'ghost-scan-triage.jsonl');
+
+function loadLines(file) {
+ if (!fs.existsSync(file)) return [];
+ return fs.readFileSync(file, 'utf8').split('\n')
+ .filter(Boolean).map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+}
+
+function loadTriagedIds() {
+ const ids = new Set();
+ for (const r of loadLines(TRIAGE)) if (r.id) ids.add(r.id);
+ return ids;
+}
+
+// Resolve disk path. flagged.jsonl rewrites image_url to /designs/img/by-id/<id>
+// which is a route, not a filename, so we need to look up the real path:
+// pg source → SELECT local_path FROM spoon_all_designs WHERE id=...
+// json source → lookup in designs.json (id → filename) then join GENERATED_DIR
+const GENERATED_DIR = path.join(ROOT, 'data', 'generated');
+const TMP_IMG_DIR = '/tmp/ghost-scan-imgs';
+fs.mkdirSync(TMP_IMG_DIR, { recursive: true });
+
+// Build id → filename map from designs.json once (~37k entries, cheap)
+let DESIGNS_JSON_MAP = null;
+function designsJsonMap() {
+ if (DESIGNS_JSON_MAP) return DESIGNS_JSON_MAP;
+ const f = path.join(ROOT, 'data', 'designs.json');
+ DESIGNS_JSON_MAP = new Map();
+ if (!fs.existsSync(f)) return DESIGNS_JSON_MAP;
+ const arr = JSON.parse(fs.readFileSync(f, 'utf8'));
+ for (const d of arr) {
+ if (!d.id) continue;
+ const fn = d.filename || (d.image_url || '').replace(/^\/designs\/img\//, '');
+ if (fn) DESIGNS_JSON_MAP.set(d.id, fn);
+ }
+ return DESIGNS_JSON_MAP;
+}
+
+// PG fallback — look up local_path for ids that don't carry one in the flagged row.
+const PG_PATH_CACHE = new Map();
+function pgLocalPath(id) {
+ if (PG_PATH_CACHE.has(id)) return PG_PATH_CACHE.get(id);
+ const onLinux = process.platform === 'linux';
+ const cmd = onLinux ? 'sudo' : 'psql';
+ const args = onLinux
+ ? ['-n', '-u', 'postgres', 'psql', 'dw_unified', '-At', '-q', '-c', `SELECT local_path FROM spoon_all_designs WHERE id=${id}`]
+ : ['dw_unified', '-At', '-q', '-c', `SELECT local_path FROM spoon_all_designs WHERE id=${id}`];
+ const r = spawnSync(cmd, args, { encoding: 'utf8' });
+ const out = r.status === 0 ? r.stdout.trim() : '';
+ PG_PATH_CACHE.set(id, out || null);
+ return out || null;
+}
+
+async function resolveImagePath(row) {
+ if (row.local_path && fs.existsSync(row.local_path)) return row.local_path;
+ // pg source → consult PG for the local_path
+ if (row.source === 'pg') {
+ const p = pgLocalPath(row.id);
+ if (p && fs.existsSync(p)) return p;
+ }
+ // json source → consult designs.json for the filename
+ if (row.source === 'json') {
+ const fn = designsJsonMap().get(row.id);
+ if (fn) {
+ const cand = path.join(GENERATED_DIR, fn);
+ if (fs.existsSync(cand)) return cand;
+ }
+ }
+ // last resort — fetch via local API (works regardless of source if server is up)
+ const url = `http://127.0.0.1:9905/designs/img/by-id/${row.id}`;
+ const dest = path.join(TMP_IMG_DIR, `${row.id}.png`);
+ if (fs.existsSync(dest)) return dest;
+ const r = await fetch(url);
+ if (!r.ok) throw new Error(`fetch ${r.status}`);
+ const buf = Buffer.from(await r.arrayBuffer());
+ if (buf.length < 200) throw new Error(`tiny body ${buf.length}b`);
+ fs.writeFileSync(dest, buf);
+ return dest;
+}
+
+function bucketOf(stable) {
+ // stable.confidence encodes consensus: 1.0 unanimous, 0.67 majority split
+ if (stable.hasGhostLayer && stable.unanimous) return 'CONFIRMED';
+ if (stable.hasGhostLayer && !stable.unanimous) return 'BORDERLINE';
+ if (!stable.hasGhostLayer && !stable.unanimous) return 'BORDERLINE'; // 2/1 false
+ return 'CLEARED'; // 3/3 false
+}
+
+async function processOne(row) {
+ try {
+ const imgPath = await resolveImagePath(row);
+ const stable = await analyzeGhostLayerStable(imgPath, { category: row.category });
+ return {
+ id: row.id,
+ ts: new Date().toISOString(),
+ bucket: bucketOf(stable),
+ stable: {
+ hasGhostLayer: stable.hasGhostLayer,
+ confidence: stable.confidence,
+ unanimous: stable.unanimous,
+ votes: stable.votes,
+ reason: stable.reason,
+ errored: stable.errored,
+ },
+ original_confidence: row.confidence,
+ category: row.category,
+ source: row.source,
+ };
+ } catch (e) {
+ return {
+ id: row.id,
+ ts: new Date().toISOString(),
+ bucket: 'ERROR',
+ error: e.message.slice(0, 200),
+ category: row.category,
+ source: row.source,
+ };
+ }
+}
+
+async function runPool(rows, workers, onResult) {
+ let cursor = 0;
+ const next = () => (cursor < rows.length ? rows[cursor++] : null);
+ async function workerLoop() {
+ while (true) {
+ const r = next();
+ if (!r) return;
+ const out = await processOne(r);
+ onResult(out);
+ }
+ }
+ await Promise.all(Array.from({ length: workers }, workerLoop));
+}
+
+(async () => {
+ console.log(`[triage] limit=${LIMIT || 'all'} concurrency=${CONCURRENCY} category=${CATEGORY || 'all'}`);
+ const flagged = loadLines(FLAGGED);
+ if (!flagged.length) {
+ console.log('no flagged file — nothing to triage');
+ return;
+ }
+ const done = loadTriagedIds();
+ let todo = flagged.filter(r => !done.has(r.id));
+ if (CATEGORY) todo = todo.filter(r => (r.category || '') === CATEGORY);
+ if (LIMIT > 0) todo = todo.slice(0, LIMIT);
+ console.log(`[triage] flagged=${flagged.length} already-triaged=${done.size} todo=${todo.length}`);
+
+ const counts = { CONFIRMED: 0, BORDERLINE: 0, CLEARED: 0, ERROR: 0 };
+ const byCat = {};
+ let n = 0;
+ const t0 = Date.now();
+ const onResult = (out) => {
+ n++;
+ fs.appendFileSync(TRIAGE, JSON.stringify(out) + '\n');
+ counts[out.bucket] = (counts[out.bucket] || 0) + 1;
+ const c = out.category || 'null';
+ byCat[c] = byCat[c] || { CONFIRMED: 0, BORDERLINE: 0, CLEARED: 0, ERROR: 0 };
+ byCat[c][out.bucket] = (byCat[c][out.bucket] || 0) + 1;
+ if (n % 10 === 0 || n === todo.length) {
+ const rate = n / ((Date.now() - t0) / 1000);
+ const eta = ((todo.length - n) / Math.max(rate, 0.01)).toFixed(0);
+ console.log(`[triage] ${n}/${todo.length} · CONFIRMED=${counts.CONFIRMED} BORDERLINE=${counts.BORDERLINE} CLEARED=${counts.CLEARED} ERROR=${counts.ERROR} · ${rate.toFixed(1)}/s · ETA ${eta}s`);
+ }
+ };
+ await runPool(todo, CONCURRENCY, onResult);
+
+ console.log('\n=== triage summary ===');
+ console.log(`scanned=${n} CONFIRMED=${counts.CONFIRMED} (${(100*counts.CONFIRMED/n).toFixed(1)}%) BORDERLINE=${counts.BORDERLINE} (${(100*counts.BORDERLINE/n).toFixed(1)}%) CLEARED=${counts.CLEARED} (${(100*counts.CLEARED/n).toFixed(1)}%) ERROR=${counts.ERROR}`);
+
+ console.log('\n=== per-category breakdown (top 15 by count) ===');
+ const rows = Object.entries(byCat)
+ .map(([cat, b]) => ({ cat, ...b, total: b.CONFIRMED + b.BORDERLINE + b.CLEARED + b.ERROR }))
+ .sort((a, b) => b.total - a.total)
+ .slice(0, 15);
+ for (const r of rows) {
+ const clearedPct = r.total ? (100 * r.CLEARED / r.total).toFixed(0) : 0;
+ console.log(` ${r.cat.padEnd(40)} total=${String(r.total).padStart(4)} CONFIRMED=${String(r.CONFIRMED).padStart(3)} BORDERLINE=${String(r.BORDERLINE).padStart(3)} CLEARED=${String(r.CLEARED).padStart(3)} (${clearedPct}%)`);
+ }
+ console.log(`\ntriage file: ${TRIAGE}`);
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
← 6bd6333 admin: ghost-review viewer + label-collection routes
·
back to Wallco Ai
·
ghost-review: more robust labeling — multi-rect, defect type 150060a →