← back to Wallco Ai
lib/subject-detector — Gemini vision gate for "is the prompted subject visible"
caabe863400831ec305dd2a4f712422b1e305b44 · 2026-05-25 01:57:49 -0700 · Steve Abrams
Catches the broader class of failure where SDXL/Gemini drops the subject
silently and ships only the background. Modeled after the
bullfrog-bluebonnets prompt that produced flower-only output (design
#28066) and was rescued by gen-from-stored-prompt to #41536.
API matches the lib/composition-detector and lib/ghost-detector shape:
- analyzeSubjectPresence(input, { subjects: [...] })
- analyzeSubjectPresenceStable(input, opts) — 3-vote majority
- gateSubjectPresence(input, opts) — { ok, result }
- extractSubjectsFromPrompt(prompt) — regex-only noun-phrase
extractor (no LLM cost) for callers with raw generation prompts
Cost ≈ $0.0005 per call (gemini-2.0-flash vision).
Validated on the bullfrog rescue family:
#28066 (broken — flowers only): all 3 subjects (bullfrog, champagne
bottle, bluebonnets) flagged MISSING, confidence 1.0,
"white flowers and dark green leaves on a blue background"
#41536 (rescue): all 3 subjects PRESENT, confidence 1.0,
"bullfrogs wearing bluebonnets and holding champagne bottles"
Optional post-gen-vision check for gen-luxe.js (T6 on the overnight
backlog) — wires this in as a publish gate when --verify-subjects is set.
Files touched
A lib/subject-detector.js
Diff
commit caabe863400831ec305dd2a4f712422b1e305b44
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon May 25 01:57:49 2026 -0700
lib/subject-detector — Gemini vision gate for "is the prompted subject visible"
Catches the broader class of failure where SDXL/Gemini drops the subject
silently and ships only the background. Modeled after the
bullfrog-bluebonnets prompt that produced flower-only output (design
#28066) and was rescued by gen-from-stored-prompt to #41536.
API matches the lib/composition-detector and lib/ghost-detector shape:
- analyzeSubjectPresence(input, { subjects: [...] })
- analyzeSubjectPresenceStable(input, opts) — 3-vote majority
- gateSubjectPresence(input, opts) — { ok, result }
- extractSubjectsFromPrompt(prompt) — regex-only noun-phrase
extractor (no LLM cost) for callers with raw generation prompts
Cost ≈ $0.0005 per call (gemini-2.0-flash vision).
Validated on the bullfrog rescue family:
#28066 (broken — flowers only): all 3 subjects (bullfrog, champagne
bottle, bluebonnets) flagged MISSING, confidence 1.0,
"white flowers and dark green leaves on a blue background"
#41536 (rescue): all 3 subjects PRESENT, confidence 1.0,
"bullfrogs wearing bluebonnets and holding champagne bottles"
Optional post-gen-vision check for gen-luxe.js (T6 on the overnight
backlog) — wires this in as a publish gate when --verify-subjects is set.
---
lib/subject-detector.js | 216 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 216 insertions(+)
diff --git a/lib/subject-detector.js b/lib/subject-detector.js
new file mode 100644
index 0000000..937a90a
--- /dev/null
+++ b/lib/subject-detector.js
@@ -0,0 +1,216 @@
+// subject-detector — verifies the subjects named in a generation prompt
+// are actually visible in the produced image. Built to catch the broader
+// class of failure where SDXL/Gemini silently drops the subject and ships
+// only the background (e.g. the bullfrog-among-bluebonnets prompt that
+// produced bluebonnets only, no frog — design #28066 → #41536 rescue).
+//
+// Sister to lib/composition-detector and lib/ghost-detector — same Gemini
+// vision approach, different failure class.
+//
+// API:
+// analyzeSubjectPresence(input, { subjects: ['bullfrog', 'champagne bottle'] })
+// → { all_present, missing, present, confidence, what_you_see, cost_estimate }
+//
+// analyzeSubjectPresenceStable(input, opts) — 3-vote majority variant
+//
+// gateSubjectPresence(input, opts) — boolean gate { ok, result }
+//
+// extractSubjectsFromPrompt(prompt) — best-effort noun-phrase extractor
+// (regex-only, no LLM cost) for callers that have a raw generation
+// prompt but no curated subjects list
+//
+// Cost: ~$0.0005 per analyzeSubjectPresence call (one Gemini Flash vision
+// call). Stable variant runs 3 in parallel, ~$0.0015.
+
+const fs = require('fs');
+const path = require('path');
+
+const GEMINI_MODEL = process.env.SUBJECT_DETECTOR_MODEL || 'gemini-2.0-flash';
+const GEMINI_BASE = 'https://generativelanguage.googleapis.com/v1beta/models';
+
+// Anti-prompt qualifiers we strip from extracted subject phrases — these
+// are commonly attached to the noun in wallco generation prompts.
+const QUALIFIER_STRIP = /\b(pink-cheeked|sleepy|tipsy|drunken|sloshed|wobbly|inebriated|starry-eyed|long-lashed|reticulated|plastered|stoned|silly|happy|sad|dignified|elegant|hand-painted|naturalist|cartoon|chibi)\b/gi;
+
+function readKey() {
+ if (process.env.GEMINI_API_KEY) return process.env.GEMINI_API_KEY;
+ try {
+ const env = fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8');
+ const m = env.match(/^GEMINI_API_KEY=(.+)$/m);
+ if (m) return m[1].trim();
+ } catch {}
+ throw new Error('GEMINI_API_KEY missing');
+}
+
+function toBase64(input) {
+ if (Buffer.isBuffer(input)) return { b64: input.toString('base64'), mime: 'image/png' };
+ if (typeof input !== 'string') throw new Error('subject-detector: input must be path|b64|buffer');
+ if (input.startsWith('/') || /\.(png|jpg|jpeg)$/i.test(input)) {
+ const bytes = fs.readFileSync(input);
+ const mime = /\.png$/i.test(input) ? 'image/png' : 'image/jpeg';
+ return { b64: bytes.toString('base64'), mime };
+ }
+ return { b64: input, mime: 'image/png' };
+}
+
+// Best-effort subject extraction from a free-text generation prompt.
+// Returns up to 3 noun phrases — the main subject(s) most likely to have
+// been dropped. Doesn't try to be perfect; callers that need precision
+// should pass an explicit subjects array.
+//
+// Heuristic: take the first comma-delimited fragment (usually the subject
+// in wallco prompts: "pink-cheeked drunken bullfrog, champagne bottle,
+// ..."), strip qualifier adjectives, split on " and ", take the last 1-3
+// noun-ish tokens.
+function extractSubjectsFromPrompt(prompt) {
+ if (!prompt || typeof prompt !== 'string') return [];
+ const parts = prompt.split(',').map(p => p.trim()).filter(Boolean);
+ const subjects = [];
+ for (const part of parts.slice(0, 3)) {
+ const cleaned = part.replace(QUALIFIER_STRIP, '').replace(/\s+/g, ' ').trim();
+ if (!cleaned || cleaned.length < 3) continue;
+ // Take the last 1-3 tokens (the noun phrase head)
+ const tokens = cleaned.split(/\s+/).filter(t => t.length > 1);
+ if (!tokens.length) continue;
+ const head = tokens.slice(-3).join(' ').toLowerCase();
+ if (head.length < 3 || head.length > 60) continue;
+ if (!subjects.includes(head)) subjects.push(head);
+ if (subjects.length >= 3) break;
+ }
+ return subjects;
+}
+
+function buildPrompt(subjects) {
+ const list = subjects.map((s, i) => `${i + 1}. "${s}"`).join('\n');
+ return `You are inspecting a WALLPAPER TILE image. Check whether each of the following SUBJECTS is visibly present in the image — not implied, not described, VISIBLY DRAWN/RENDERED.
+
+Subjects to check:
+${list}
+
+For each subject, mark TRUE if you can point to it in the image (even if rendered abstractly, as a silhouette, or as a small repeating element). Mark FALSE if it's missing entirely or replaced by a different motif.
+
+Reply ONLY in this JSON shape:
+{
+ "subjects": [
+ {"name": "<subject name>", "present": true|false, "notes": "<one short phrase, max 12 words>"}
+ ],
+ "what_you_see": "<one sentence describing the image, max 30 words>",
+ "confidence": <0.0-1.0>
+}
+
+Be strict — if a subject is named but the image shows only the BACKGROUND or a DIFFERENT subject, mark it FALSE. The image is a 24" wallpaper tile from a designer wallcovering catalog.`;
+}
+
+async function analyzeSubjectPresence(input, opts = {}) {
+ const subjects = Array.isArray(opts.subjects) ? opts.subjects.filter(s => s && typeof s === 'string').slice(0, 8) : [];
+ if (!subjects.length) {
+ return {
+ all_present: true,
+ present: [],
+ missing: [],
+ confidence: 1.0,
+ what_you_see: 'No subjects supplied — vacuously passing.',
+ cost_estimate: 0,
+ no_subjects: true,
+ };
+ }
+ const KEY = readKey();
+ const { b64, mime } = toBase64(input);
+ const url = `${GEMINI_BASE}/${GEMINI_MODEL}:generateContent?key=${KEY}`;
+ const body = {
+ contents: [{ parts: [
+ { text: buildPrompt(subjects) },
+ { inline_data: { mime_type: mime, data: b64 } },
+ ]}],
+ generationConfig: {
+ temperature: 0.0,
+ topP: 1.0,
+ topK: 1,
+ candidateCount: 1,
+ response_mime_type: 'application/json',
+ },
+ };
+ const r = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ if (!r.ok) {
+ const txt = await r.text().catch(() => '');
+ throw new Error(`gemini ${r.status}: ${txt.slice(0, 200)}`);
+ }
+ const j = await r.json();
+ const text = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
+ let parsed;
+ try { parsed = JSON.parse(text); }
+ catch { throw new Error(`gemini returned non-JSON: ${text.slice(0, 200)}`); }
+
+ const items = Array.isArray(parsed.subjects) ? parsed.subjects : [];
+ const present = [];
+ const missing = [];
+ // Match Gemini's per-subject answers back to the input list — fall back
+ // to "missing" if Gemini didn't return a row for a subject we asked about
+ for (const s of subjects) {
+ const row = items.find(r => r && typeof r.name === 'string' && r.name.toLowerCase().includes(s.toLowerCase().slice(0, 12)));
+ if (row && row.present === true) present.push(s);
+ else missing.push(s);
+ }
+ return {
+ all_present: missing.length === 0,
+ present,
+ missing,
+ confidence: Number(parsed.confidence) || 0,
+ what_you_see: String(parsed.what_you_see || '').slice(0, 250),
+ cost_estimate: 0.0005,
+ raw_per_subject: items,
+ };
+}
+
+// 3-vote stable variant — majority on per-subject present flag.
+async function analyzeSubjectPresenceStable(input, opts = {}) {
+ const N = 3;
+ const settled = await Promise.allSettled(
+ Array.from({ length: N }, () => analyzeSubjectPresence(input, opts))
+ );
+ const votes = settled.filter(s => s.status === 'fulfilled').map(s => s.value);
+ if (!votes.length) {
+ throw new Error('all 3 vision calls failed: ' + settled.map(s => s.reason?.message || '?').join(' / '));
+ }
+ // Per-subject majority
+ const subjects = Array.isArray(opts.subjects) ? opts.subjects : [];
+ const present = [];
+ const missing = [];
+ for (const s of subjects) {
+ const yes = votes.filter(v => v.present.includes(s)).length;
+ if (yes >= Math.ceil(votes.length / 2)) present.push(s);
+ else missing.push(s);
+ }
+ const avgConf = votes.reduce((a, v) => a + (v.confidence || 0), 0) / votes.length;
+ return {
+ all_present: missing.length === 0,
+ present, missing,
+ confidence: avgConf,
+ consensus: votes.every(v => v.missing.length === missing.length && v.present.length === present.length) ? 1.0 : (votes.length - 1) / votes.length,
+ votes_count: votes.length,
+ errored: N - votes.length,
+ what_you_see: votes[0].what_you_see,
+ cost_estimate: votes.length * 0.0005,
+ };
+}
+
+// Convenience gate — returns { ok, result } so callers can pattern-match
+// the same way as gateComposition / gateGhost.
+async function gateSubjectPresence(input, opts = {}) {
+ const stable = opts.stable === true;
+ const result = await (stable
+ ? analyzeSubjectPresenceStable(input, opts)
+ : analyzeSubjectPresence(input, opts));
+ return { ok: result.all_present, result };
+}
+
+module.exports = {
+ analyzeSubjectPresence,
+ analyzeSubjectPresenceStable,
+ gateSubjectPresence,
+ extractSubjectsFromPrompt,
+};
← 45dcfb7 revert luxe-c blanket rollout + add curator-mode 4-up picker
·
back to Wallco Ai
·
T2: subject-mismatch scan + gallery — flags dropped-subject e7bc7e1 →