← back to Wallco Ai
ghost-layer: detector + scanner + pre-publish gate + fix-queuer
49e0ab6f437a128b030bf02fd5165a2ac26fb7a4 · 2026-05-22 06:02:54 -0700 · Steve Abrams
3 pieces wired up:
1. lib/ghost-detector.js — shared classifier using Gemini 2.0 Flash vision.
Returns { hasGhostLayer, confidence 0-1, reason }. Strict JSON output.
Two exports — analyzeGhostLayer (info) and verifyNoGhostLayer (gate that
throws GHOST_LAYER_DETECTED on conf >= 0.5).
2. scripts/scan-ghost-layer.js — sweeps published catalog (spoon_all_designs
is_published=TRUE + local_path NOT NULL), classifies each design, appends
results to data/ghost-scan-results.jsonl + flagged subset to
data/ghost-scan-flagged.jsonl. Resumable (skips ids already in results
file). --limit / --concurrency / --category flags.
3. scripts/queue-ghost-fixes.js — reads ghost-scan-flagged.jsonl and POSTs
each id to /api/design/:id/ghosting?action=update-sku which regenerates
in-place w/ anti-ghost prompt addendum AND preserves the DIG number.
Rate-limited (8s default) so Replicate doesn't spike. Resumable via
data/ghost-scan-queued.jsonl.
4. scripts/generate_designs.js — added a pre-publish gate that runs
verifyNoGhostLayer after every successful gen, retries once with a
harder anti-ghost prompt addendum if defective, skips if still defective
on retry. Gated by WALLCO_GHOST_GATE env (set to '0' to disable).
main() promoted to async to support the await.
Pair w/ feedback_sdxl_antiprompts_unreliable.md (the foundational note —
anti-prompts don't enforce, output must be visually verified).
Triggered by Steve's call on design 26518 (cactus pattern with obvious
ghost layer), already hard-deleted + regen queued via the existing
/api/design/:id/ghosting endpoint.
Files touched
A lib/ghost-detector.jsM scripts/generate_designs.jsA scripts/queue-ghost-fixes.jsA scripts/scan-ghost-layer.js
Diff
commit 49e0ab6f437a128b030bf02fd5165a2ac26fb7a4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri May 22 06:02:54 2026 -0700
ghost-layer: detector + scanner + pre-publish gate + fix-queuer
3 pieces wired up:
1. lib/ghost-detector.js — shared classifier using Gemini 2.0 Flash vision.
Returns { hasGhostLayer, confidence 0-1, reason }. Strict JSON output.
Two exports — analyzeGhostLayer (info) and verifyNoGhostLayer (gate that
throws GHOST_LAYER_DETECTED on conf >= 0.5).
2. scripts/scan-ghost-layer.js — sweeps published catalog (spoon_all_designs
is_published=TRUE + local_path NOT NULL), classifies each design, appends
results to data/ghost-scan-results.jsonl + flagged subset to
data/ghost-scan-flagged.jsonl. Resumable (skips ids already in results
file). --limit / --concurrency / --category flags.
3. scripts/queue-ghost-fixes.js — reads ghost-scan-flagged.jsonl and POSTs
each id to /api/design/:id/ghosting?action=update-sku which regenerates
in-place w/ anti-ghost prompt addendum AND preserves the DIG number.
Rate-limited (8s default) so Replicate doesn't spike. Resumable via
data/ghost-scan-queued.jsonl.
4. scripts/generate_designs.js — added a pre-publish gate that runs
verifyNoGhostLayer after every successful gen, retries once with a
harder anti-ghost prompt addendum if defective, skips if still defective
on retry. Gated by WALLCO_GHOST_GATE env (set to '0' to disable).
main() promoted to async to support the await.
Pair w/ feedback_sdxl_antiprompts_unreliable.md (the foundational note —
anti-prompts don't enforce, output must be visually verified).
Triggered by Steve's call on design 26518 (cactus pattern with obvious
ghost layer), already hard-deleted + regen queued via the existing
/api/design/:id/ghosting endpoint.
---
lib/ghost-detector.js | 104 ++++++++++++++++++++++++++++++
scripts/generate_designs.js | 31 ++++++++-
scripts/queue-ghost-fixes.js | 87 +++++++++++++++++++++++++
scripts/scan-ghost-layer.js | 147 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 368 insertions(+), 1 deletion(-)
diff --git a/lib/ghost-detector.js b/lib/ghost-detector.js
new file mode 100644
index 0000000..3b017ea
--- /dev/null
+++ b/lib/ghost-detector.js
@@ -0,0 +1,104 @@
+// Ghost-layer detector (Gemini 2.0 Flash vision).
+//
+// A "ghost layer" = multiple opacity tiers in a single design — solid
+// foreground motifs PLUS faded/translucent copies of the same motifs sitting
+// behind. SDXL produces these reliably despite our anti-prompts (see memory
+// note feedback_sdxl_antiprompts_unreliable.md). Anti-prompts alone don't
+// catch it; we have to verify visually after generation.
+//
+// Two exports:
+// analyzeGhostLayer(imagePath) → { hasGhostLayer, confidence, reason, cost_estimate }
+// verifyNoGhostLayer(imagePath) → throws if hasGhostLayer === true (use as a gate)
+
+const fs = require('fs');
+const path = require('path');
+
+const GEMINI_MODEL = process.env.GHOST_DETECTOR_MODEL || 'gemini-2.0-flash';
+const GEMINI_BASE = 'https://generativelanguage.googleapis.com/v1beta/models';
+
+const PROMPT = `You are inspecting a single wallpaper / surface-design image for a manufacturing defect called a "ghost layer."
+
+A ghost layer means the design has MULTIPLE OPACITY TIERS in the SAME illustration — e.g. solid sharp-edged foreground motifs PLUS faded / translucent / watercolor-washed copies of the same motifs sitting behind them at lower opacity. The intended aesthetic for these designs is FLAT, SINGLE-INK, single-opacity hand-blocked silhouette. Any secondary motifs at <100% opacity = defect.
+
+Decide:
+ hasGhostLayer: true if you see ANY of:
+ - lighter / faded copies of motifs sitting behind solid ones
+ - semi-transparent secondary layer (the same motifs at ~30-70% opacity)
+ - atmospheric perspective / depth haze
+ - watercolor-wash secondary layer
+ - "halo" pale silhouettes behind sharp ones
+ hasGhostLayer: false if the design is uniformly flat single-ink-on-background with no secondary opacity tier.
+
+ confidence: 0.0-1.0 how confident you are
+
+ reason: ONE short sentence (max 25 words) describing what you saw
+
+Reply with ONLY a JSON object exactly in this shape:
+{"hasGhostLayer": true|false, "confidence": <number>, "reason": "<short sentence>"}
+No other text. No markdown. No prose.`;
+
+function readKey() {
+ if (process.env.GEMINI_API_KEY) return process.env.GEMINI_API_KEY;
+ // .env fallback when script run outside pm2
+ try {
+ const env = fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8');
+ const m = env.match(/^GEMINI_API_KEY=(.+)$/m);
+ if (m) return m[1].trim();
+ } catch {}
+ throw new Error('GEMINI_API_KEY missing');
+}
+
+async function analyzeGhostLayer(imagePath, opts = {}) {
+ const KEY = readKey();
+ const bytes = fs.readFileSync(imagePath);
+ const mime = imagePath.toLowerCase().endsWith('.png') ? 'image/png' : 'image/jpeg';
+ const url = `${GEMINI_BASE}/${GEMINI_MODEL}:generateContent?key=${KEY}`;
+
+ const body = {
+ contents: [{
+ parts: [
+ { text: PROMPT },
+ { inline_data: { mime_type: mime, data: bytes.toString('base64') } },
+ ],
+ }],
+ generationConfig: { temperature: 0.0, response_mime_type: 'application/json' },
+ };
+
+ const r = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ if (!r.ok) {
+ const txt = await r.text().catch(() => '');
+ throw new Error(`gemini ${r.status}: ${txt.slice(0, 200)}`);
+ }
+ const j = await r.json();
+ const text = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
+ let parsed;
+ try { parsed = JSON.parse(text); }
+ catch (e) { throw new Error(`gemini returned non-JSON: ${text.slice(0, 200)}`); }
+ if (typeof parsed.hasGhostLayer !== 'boolean') {
+ throw new Error(`gemini missing hasGhostLayer field: ${JSON.stringify(parsed)}`);
+ }
+ return {
+ hasGhostLayer: parsed.hasGhostLayer,
+ confidence: Number(parsed.confidence) || 0,
+ reason: String(parsed.reason || '').slice(0, 250),
+ cost_estimate: 0.0005, // approx — gemini-2.0-flash vision
+ };
+}
+
+// Gate variant — for the pre-publish pipeline. Throws if defective.
+async function verifyNoGhostLayer(imagePath) {
+ const r = await analyzeGhostLayer(imagePath);
+ if (r.hasGhostLayer && r.confidence >= 0.5) {
+ const err = new Error(`ghost-layer detected (conf=${r.confidence.toFixed(2)}): ${r.reason}`);
+ err.code = 'GHOST_LAYER_DETECTED';
+ err.detector_result = r;
+ throw err;
+ }
+ return r;
+}
+
+module.exports = { analyzeGhostLayer, verifyNoGhostLayer, GEMINI_MODEL };
diff --git a/scripts/generate_designs.js b/scripts/generate_designs.js
index ed516ec..0a758f9 100644
--- a/scripts/generate_designs.js
+++ b/scripts/generate_designs.js
@@ -423,7 +423,7 @@ VALUES (
// ---- main -----------------------------------------------------------------
-function main() {
+async function main() {
const opt = args();
const widthFixed = (opt.width != null);
console.log(`Generating ${opt.n} × ${opt.kind} (${widthFixed ? `${opt.width}"×${opt.height||opt.width}"` : `width mixed from [${WIDTH_POOL.join(',')}]`}, category=${opt.category}, backend=${BACKEND})`);
@@ -465,6 +465,35 @@ function main() {
} catch (e) {
console.warn(' ⚠ watermark error:', e.message);
}
+
+ // ── Pre-publish ghost-layer gate (2026-05-22) ────────────────────────
+ // Anti-prompts don't actually prevent SDXL from emitting multi-opacity
+ // layers (feedback_sdxl_antiprompts_unreliable.md). Verify visually here.
+ // If detector says it's defective, retry ONCE with a stronger anti-ghost
+ // prompt; if still defective, skip this design entirely.
+ try {
+ const { verifyNoGhostLayer } = require('../lib/ghost-detector');
+ if (process.env.WALLCO_GHOST_GATE !== '0') {
+ try {
+ await verifyNoGhostLayer(outPath);
+ } catch (gerr) {
+ if (gerr.code === 'GHOST_LAYER_DETECTED') {
+ console.warn(` ⚠ ghost-layer detected — regenerating once · ${gerr.detector_result.reason}`);
+ const retryPrompt = prompt + '\n\nABSOLUTE RULE — ONE FLAT INK COLOR ONLY. No ghost layer. No translucent secondary motifs. No faded background copies. No depth haze. No watercolor wash. Single-opacity hand-blocked silhouette only. If you produce any partial-opacity layer the image will be rejected.';
+ generate(retryPrompt, seed + 1, outPath, { kind: opt.kind, category: opt.category });
+ const r2 = await verifyNoGhostLayer(outPath).catch(e => e);
+ if (r2 && r2.code === 'GHOST_LAYER_DETECTED') {
+ console.warn(` ✗ ghost-layer persists after retry — skipping #${seed}`);
+ continue;
+ }
+ console.log(` ✓ retry clean`);
+ } else { throw gerr; }
+ }
+ }
+ } catch (e) {
+ console.warn(' ⚠ ghost-gate non-fatal:', e.message);
+ }
+
const id = persistDesign({
kind: opt.kind, prompt, seed, localPath: outPath,
width: thisWidth, height: thisHeight, panels: opt.panels,
diff --git a/scripts/queue-ghost-fixes.js b/scripts/queue-ghost-fixes.js
new file mode 100644
index 0000000..b5657c1
--- /dev/null
+++ b/scripts/queue-ghost-fixes.js
@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+// queue-ghost-fixes — read data/ghost-scan-flagged.jsonl and POST each design
+// to /api/design/:id/ghosting?action=update-sku which regenerates in-place
+// with the anti-ghost prompt addendum and PRESERVES the DIG number.
+//
+// Resumable — skips ids already in data/ghost-scan-queued.jsonl.
+// Rate-limited (default 1 fix every 8s — Replicate SDXL takes ~5-15s and
+// charges per generation, so don't fan out too hard).
+//
+// Usage:
+// node scripts/queue-ghost-fixes.js # all flagged
+// node scripts/queue-ghost-fixes.js --limit 50 # first 50
+// node scripts/queue-ghost-fixes.js --interval-ms 6000 # tune throttle
+
+const fs = require('fs');
+const path = require('path');
+
+const ARGS = process.argv.slice(2);
+function arg(name, def) {
+ const i = ARGS.indexOf('--' + name);
+ return i >= 0 ? ARGS[i + 1] : def;
+}
+const LIMIT = parseInt(arg('limit', '0'), 10);
+const INTERVAL_MS = parseInt(arg('interval-ms', '8000'), 10);
+const ADMIN_TOKEN = process.env.ADMIN_TOKEN || readEnv('ADMIN_TOKEN');
+const HOST = process.env.WALLCO_HOST || 'https://wallco.ai';
+
+function readEnv(key) {
+ try {
+ const env = fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8');
+ const m = env.match(new RegExp(`^${key}=(.+)$`, 'm'));
+ return m ? m[1].trim() : '';
+ } catch { return ''; }
+}
+
+const ROOT = path.join(__dirname, '..');
+const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
+const QUEUED = path.join(ROOT, 'data', 'ghost-scan-queued.jsonl');
+
+function loadLines(file) {
+ if (!fs.existsSync(file)) return [];
+ return fs.readFileSync(file, 'utf8').split('\n')
+ .filter(Boolean).map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+}
+
+async function main() {
+ if (!ADMIN_TOKEN) throw new Error('ADMIN_TOKEN missing — set in env or .env');
+ if (!fs.existsSync(FLAGGED)) {
+ console.log('no flagged file yet — run scan-ghost-layer.js first');
+ return;
+ }
+
+ const flagged = loadLines(FLAGGED);
+ const queued = new Set(loadLines(QUEUED).map(r => r.id));
+ const todo = flagged.filter(r => !queued.has(r.id)).slice(0, LIMIT || flagged.length);
+
+ console.log(`[queue-fixes] flagged=${flagged.length} queued=${queued.size} todo=${todo.length} interval=${INTERVAL_MS}ms`);
+
+ let ok = 0, fail = 0;
+ for (let i = 0; i < todo.length; i++) {
+ const row = todo[i];
+ const url = `${HOST}/api/design/${row.id}/ghosting?admin=${encodeURIComponent(ADMIN_TOKEN)}`;
+ let result;
+ try {
+ const r = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ action: 'update-sku', reason: 'auto-queued from ghost-scan-flagged.jsonl', detector_confidence: row.confidence }),
+ });
+ result = {
+ id: row.id, ts: new Date().toISOString(),
+ ok: r.ok, status: r.status,
+ body: r.ok ? null : (await r.text().catch(() => '')).slice(0, 200),
+ };
+ } catch (e) {
+ result = { id: row.id, ts: new Date().toISOString(), ok: false, error: e.message };
+ }
+ fs.appendFileSync(QUEUED, JSON.stringify(result) + '\n');
+ if (result.ok) { ok++; } else { fail++; }
+ console.log(` ${i + 1}/${todo.length} ${result.ok ? '✓' : '✗'} #${row.id} ${result.body || result.error || 'queued'}`);
+ if (i < todo.length - 1) await new Promise(r => setTimeout(r, INTERVAL_MS));
+ }
+
+ console.log(`\n[queue-fixes] done · ok=${ok} fail=${fail}`);
+}
+
+main().catch(e => { console.error('FATAL', e); process.exit(1); });
diff --git a/scripts/scan-ghost-layer.js b/scripts/scan-ghost-layer.js
new file mode 100644
index 0000000..209a5f3
--- /dev/null
+++ b/scripts/scan-ghost-layer.js
@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+// scan-ghost-layer — sweep the catalog for ghost-layer defects.
+//
+// Iterates spoon_all_designs (published, has local_path), classifies each
+// image via lib/ghost-detector, appends one JSONL row per design to
+// data/ghost-scan-results.jsonl. Resumable — skips IDs already in the
+// results file so SIGINT + restart is safe.
+//
+// Usage:
+// node scripts/scan-ghost-layer.js # all published designs
+// node scripts/scan-ghost-layer.js --limit 200 # first 200 (smoke test)
+// node scripts/scan-ghost-layer.js --concurrency 8 # parallel workers (default 4)
+// node scripts/scan-ghost-layer.js --category cactus # restrict by category
+//
+// Output:
+// data/ghost-scan-results.jsonl — one line per design:
+// { id, ts, hasGhostLayer, confidence, reason, image_url, category }
+// data/ghost-scan-flagged.jsonl — append-only flagged subset (confidence ≥ 0.5)
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { analyzeGhostLayer } = require('../lib/ghost-detector');
+
+// ── args ────────────────────────────────────────────────────────────────
+const ARGS = process.argv.slice(2);
+function arg(name, def) {
+ const i = ARGS.indexOf('--' + name);
+ return i >= 0 ? ARGS[i + 1] : def;
+}
+const LIMIT = parseInt(arg('limit', '0'), 10);
+const CONCURRENCY = Math.max(1, parseInt(arg('concurrency', '4'), 10));
+const CATEGORY = arg('category', null);
+
+const ROOT = path.join(__dirname, '..');
+const RESULTS = path.join(ROOT, 'data', 'ghost-scan-results.jsonl');
+const FLAGGED = path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl');
+
+// ── PG query (use the same psql sudo trick the server uses) ─────────────
+function psql(sql) {
+ const onLinux = process.platform === 'linux';
+ const cmd = onLinux ? 'sudo' : 'psql';
+ const args = onLinux
+ ? ['-n', '-u', 'postgres', 'psql', 'dw_unified', '-At', '-q']
+ : ['dw_unified', '-At', '-q'];
+ const r = spawnSync(cmd, args, { input: sql, encoding: 'utf8', maxBuffer: 50_000_000 });
+ if (r.status !== 0) throw new Error(r.stderr || r.stdout || 'psql failed');
+ return r.stdout;
+}
+
+// ── load already-scanned ids so we can resume ───────────────────────────
+function loadScanned() {
+ if (!fs.existsSync(RESULTS)) return new Set();
+ const ids = new Set();
+ for (const line of fs.readFileSync(RESULTS, 'utf8').split('\n')) {
+ if (!line.trim()) continue;
+ try { ids.add(JSON.parse(line).id); } catch {}
+ }
+ return ids;
+}
+
+// ── fetch target ids from PG ────────────────────────────────────────────
+function fetchTargets() {
+ const where = [
+ 'is_published = TRUE',
+ 'user_removed IS NOT TRUE',
+ 'local_path IS NOT NULL',
+ ];
+ if (CATEGORY) where.push(`category = '${CATEGORY.replace(/'/g, "''")}'`);
+ const sql = `SELECT id, category, local_path
+ FROM spoon_all_designs
+ WHERE ${where.join(' AND ')}
+ ORDER BY id DESC
+ ${LIMIT > 0 ? `LIMIT ${LIMIT}` : ''}`;
+ const raw = psql(sql);
+ const rows = [];
+ for (const line of raw.split('\n')) {
+ if (!line.trim()) continue;
+ const [id, category, local_path] = line.split('|');
+ rows.push({ id: parseInt(id, 10), category, local_path });
+ }
+ return rows;
+}
+
+// ── worker pool ─────────────────────────────────────────────────────────
+async function processOne(row) {
+ if (!row.local_path || !fs.existsSync(row.local_path)) {
+ return { id: row.id, ts: new Date().toISOString(), error: 'file_missing', local_path: row.local_path };
+ }
+ try {
+ const r = await analyzeGhostLayer(row.local_path);
+ return {
+ id: row.id, ts: new Date().toISOString(),
+ hasGhostLayer: r.hasGhostLayer,
+ confidence: r.confidence,
+ reason: r.reason,
+ category: row.category,
+ image_url: `/designs/img/by-id/${row.id}`,
+ };
+ } catch (e) {
+ return { id: row.id, ts: new Date().toISOString(), error: e.message.slice(0, 200), category: row.category };
+ }
+}
+
+async function runPool(rows, workers, onResult) {
+ let cursor = 0;
+ const next = () => (cursor < rows.length ? rows[cursor++] : null);
+ async function workerLoop() {
+ while (true) {
+ const r = next();
+ if (!r) return;
+ const out = await processOne(r);
+ onResult(out);
+ }
+ }
+ await Promise.all(Array.from({ length: workers }, workerLoop));
+}
+
+// ── main ────────────────────────────────────────────────────────────────
+(async () => {
+ console.log(`[scan-ghost] starting · limit=${LIMIT || 'all'} concurrency=${CONCURRENCY} category=${CATEGORY || 'all'}`);
+ const scanned = loadScanned();
+ console.log(`[scan-ghost] already-scanned: ${scanned.size}`);
+ const all = fetchTargets();
+ const todo = all.filter(r => !scanned.has(r.id));
+ console.log(`[scan-ghost] eligible: ${all.length}, todo: ${todo.length}`);
+
+ let n = 0, flagged = 0, errored = 0;
+ const t0 = Date.now();
+ const onResult = (out) => {
+ n++;
+ fs.appendFileSync(RESULTS, JSON.stringify(out) + '\n');
+ if (out.hasGhostLayer && (out.confidence || 0) >= 0.5) {
+ flagged++;
+ fs.appendFileSync(FLAGGED, JSON.stringify(out) + '\n');
+ }
+ if (out.error) errored++;
+ if (n % 25 === 0 || n === todo.length) {
+ const rate = n / ((Date.now() - t0) / 1000);
+ const eta = ((todo.length - n) / rate).toFixed(0);
+ console.log(`[scan-ghost] ${n}/${todo.length} · ${flagged} flagged · ${errored} errored · ${rate.toFixed(1)}/s · ETA ${eta}s`);
+ }
+ };
+ await runPool(todo, CONCURRENCY, onResult);
+ console.log(`[scan-ghost] done · scanned=${n} flagged=${flagged} errored=${errored}`);
+ console.log(`[scan-ghost] flagged log: ${FLAGGED}`);
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
← f587ed9 vendor-scrub: remove all references to de Gournay / Gracie /
·
back to Wallco Ai
·
feat(scenic): hospitality style-pack sub-filter — Aman / Aub 2bd3823 →