← back to Wallco Ai
scripts/audit-cohort-generic.js
199 lines
#!/usr/bin/env node
/**
* audit-cohort-generic.js — Parameterized Gemini vision audit for any
* subject-bearing cohort. Takes CATEGORY + INTENT via env vars.
*
* AUDIT_CATEGORY=monterey-mural AUDIT_INTENT='California coastal mural — \
* pelicans, sea otters, cypress trees, ocean scenes' \
* node scripts/audit-cohort-generic.js
*
* Same shape as the drunk/stoned audit scripts:
* - Reads queue from data/designs.json (filter category + is_published)
* - Per design: fetch image → Gemini vision → 3-line response → JS verdict
* - Output: /tmp/<category>-audit-{results.jsonl,kill-list.json,keep-list.json}
*
* The prompt asks Gemini to identify the visual subject + judge match to intent.
* The collection-specific bit is INTENT, baked into the prompt at runtime.
*/
'use strict';
require('dotenv').config();
const fs = require('fs');
const path = require('path');
const http = require('http');
const https = require('https');
const API_KEY = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
if (!API_KEY) { console.error('GEMINI_API_KEY not set'); process.exit(1); }
const CATEGORY = process.env.AUDIT_CATEGORY;
const INTENT = process.env.AUDIT_INTENT;
if (!CATEGORY || !INTENT) {
console.error('AUDIT_CATEGORY + AUDIT_INTENT both required');
console.error(' example: AUDIT_CATEGORY=monterey-mural AUDIT_INTENT="California coastal mural with pelicans, sea otters, cypress" node ' + __filename);
process.exit(2);
}
const MODEL = process.env.AUDIT_MODEL || 'gemini-2.5-flash';
const SERVER = process.env.AUDIT_SERVER || 'http://127.0.0.1:9905';
const SLUG = CATEGORY.replace(/[^a-z0-9_-]+/gi, '-').replace(/^-|-$/g, '');
const OUT_JSONL = `/tmp/${SLUG}-audit-results.jsonl`;
const KILL_OUT = `/tmp/${SLUG}-audit-kill-list.json`;
const KEEP_OUT = `/tmp/${SLUG}-audit-keep-list.json`;
const CONCURRENCY = parseInt(process.env.AUDIT_CONCURRENCY, 10) || 6;
const MAX_AUDIT = parseInt(process.env.AUDIT_MAX, 10) || 0; // 0 = no cap
const PROMPT = `Describe what you see in this wallpaper pattern. The intended motif of this collection:
${INTENT}
Answer in EXACTLY this format on three lines, nothing else:
SUBJECT: name what you actually see in the image (one short phrase — e.g. "sea otter and pelican", "abstract geometric shapes", "cypress tree", "indistinct smeared shapes"). Be honest — if the figure is AI-stretched, dismembered, smeared, or amorphous, say "indistinct".
MATCHES_INTENT: pick one word — "yes" if the subject clearly matches the intended motif above, "partial" if it's related but a stretch, "no" if the subject is unrelated to the intent, "indistinct" if no clear subject.
QUALITY: pick one word — "clean" if the illustration reads as a clear intentional pattern, "stretched" if subjects look AI-stretched/dismembered/oddly-elongated across the tile, "abstract" if dominated by formless shapes.`;
// Verdict logic: KILL on (matches=no) OR (matches=indistinct) OR (quality in [stretched, abstract])
function computeVerdict(subject, matches, quality) {
const s = (subject || '').toLowerCase();
const m = (matches || '').toLowerCase();
const q = (quality || '').toLowerCase();
if (!s || s === 'indistinct') return { verdict: 'KILL', reason: 'subject indistinct' };
if (m === 'no' || m === 'indistinct') return { verdict: 'KILL', reason: 'matches:' + m };
if (q === 'stretched' || q === 'abstract' || q.includes('stretched') || q.includes('abstract')) {
return { verdict: 'KILL', reason: 'quality:' + q };
}
return { verdict: 'KEEP', reason: '' };
}
function loadQueue() {
const snap = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'data', 'designs.json'), 'utf8'));
let q = snap.filter(d => d.category === CATEGORY && d.is_published !== false && !d.user_removed)
.map(d => ({ id: d.id, title: d.title || '', created_at: d.created_at || null }));
if (MAX_AUDIT > 0) q = q.slice(0, MAX_AUDIT);
return q;
}
function loadDone() {
if (!fs.existsSync(OUT_JSONL)) return new Set();
const lines = fs.readFileSync(OUT_JSONL, 'utf8').split('\n').filter(Boolean);
const done = new Set();
for (const l of lines) { try { const r = JSON.parse(l); if (r.id) done.add(r.id); } catch {} }
return done;
}
function fetchImage(id) {
return new Promise((resolve, reject) => {
const req = http.get(`${SERVER}/designs/img/by-id/${id}`, res => {
if (res.statusCode !== 200) { res.resume(); return reject(new Error(`img HTTP ${res.statusCode}`)); }
const c = []; res.on('data', x => c.push(x));
res.on('end', () => resolve({ buf: Buffer.concat(c), mime: res.headers['content-type'] || 'image/png' }));
res.on('error', reject);
});
req.on('error', reject);
req.setTimeout(20000, () => req.destroy(new Error('img timeout')));
});
}
function callGemini(imgB64, mime) {
return new Promise((resolve, reject) => {
const body = JSON.stringify({
contents: [{ parts: [
{ inline_data: { mime_type: mime.split(';')[0].trim(), data: imgB64 } },
{ text: PROMPT },
]}],
generationConfig: { temperature: 0.1, maxOutputTokens: 100 },
});
const req = https.request({
hostname: 'generativelanguage.googleapis.com',
path: `/v1beta/models/${MODEL}:generateContent?key=${API_KEY}`,
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
}, res => {
const c = []; res.on('data', x => c.push(x));
res.on('end', () => {
const raw = Buffer.concat(c).toString('utf8');
if (res.statusCode !== 200) return reject(new Error(`gemini HTTP ${res.statusCode}: ${raw.slice(0,200)}`));
try {
const j = JSON.parse(raw);
const text = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
resolve({ text, usage: j?.usageMetadata });
} catch (e) { reject(new Error('gemini parse: ' + e.message)); }
});
});
req.on('error', reject);
req.setTimeout(45000, () => req.destroy(new Error('gemini timeout')));
req.write(body); req.end();
});
}
function parseResponse(text) {
const subject = (text.match(/SUBJECT:\s*([^\n]+)/i) || [, ''])[1].trim().slice(0, 100);
const matches = (text.match(/MATCHES_INTENT:\s*([^\n]+)/i) || [, ''])[1].trim().slice(0, 30);
const quality = (text.match(/QUALITY:\s*([^\n]+)/i) || [, ''])[1].trim().slice(0, 30);
const mClean = matches.replace(/\s*\(.+\)$/, '').replace(/\s*[—-].*$/, '').trim();
const qClean = quality.replace(/\s*\(.+\)$/, '').replace(/\s*[—-].*$/, '').trim();
const { verdict, reason } = computeVerdict(subject, mClean, qClean);
return { verdict, kill_reason: reason, subject, matches: mClean, quality: qClean };
}
async function auditOne(item) {
const t0 = Date.now();
try {
const { buf, mime } = await fetchImage(item.id);
const { text, usage } = await callGemini(buf.toString('base64'), mime);
const v = parseResponse(text);
return { id: item.id, title: item.title, ...v, raw: text.slice(0, 200),
tokens_in: usage?.promptTokenCount || 0, tokens_out: usage?.candidatesTokenCount || 0,
ms: Date.now() - t0 };
} catch (e) {
return { id: item.id, title: item.title, verdict: 'ERROR', error: String(e.message || e).slice(0, 160), ms: Date.now() - t0 };
}
}
async function main() {
const queue = loadQueue();
const done = loadDone();
const work = queue.filter(it => !done.has(it.id));
console.log(`category: ${CATEGORY}, queue: ${queue.length}, done: ${done.size}, to do: ${work.length}`);
if (!work.length) { console.log(' nothing to audit'); return; }
const out = fs.createWriteStream(OUT_JSONL, { flags: 'a' });
let idx = 0, processed = 0;
const total = work.length;
const tally = { KEEP: 0, KILL: 0, ERROR: 0 };
let tin = 0, tout = 0;
const t0 = Date.now();
async function worker() {
while (true) {
const i = idx++;
if (i >= work.length) break;
const r = await auditOne(work[i]);
tally[r.verdict] = (tally[r.verdict] || 0) + 1;
tin += r.tokens_in || 0; tout += r.tokens_out || 0;
out.write(JSON.stringify(r) + '\n');
processed++;
if (processed % 25 === 0 || processed === total) {
const elapsed = ((Date.now() - t0) / 1000).toFixed(0);
const rate = (processed / Math.max(1, (Date.now() - t0) / 1000)).toFixed(2);
const cost = (tin * 0.10 / 1e6) + (tout * 0.40 / 1e6);
console.log(` [${processed}/${total}] ${elapsed}s @ ${rate}/s — KEEP=${tally.KEEP} KILL=${tally.KILL} ERR=${tally.ERROR} | $${cost.toFixed(4)}`);
}
}
}
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
out.end();
const allRows = fs.readFileSync(OUT_JSONL, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
const kill = allRows.filter(r => r.verdict === 'KILL');
const keep = allRows.filter(r => r.verdict === 'KEEP');
fs.writeFileSync(KILL_OUT, JSON.stringify(kill, null, 2));
fs.writeFileSync(KEEP_OUT, JSON.stringify(keep, null, 2));
console.log(`\nDONE — KEEP=${keep.length} KILL=${kill.length} ERR=${allRows.filter(r=>r.verdict==='ERROR').length}`);
console.log(`cost: $${((tin*0.10/1e6)+(tout*0.40/1e6)).toFixed(4)}`);
console.log(`kill: ${KILL_OUT}`);
console.log(`keep: ${KEEP_OUT}`);
}
main().catch(e => { console.error('FATAL:', e); process.exit(1); });