`scripts/audit-dogs-breed.js`

219 lines
#!/usr/bin/env node
/**
 * audit-dogs-breed.js — Gemini vision pass over the 220 dog designs.
 * Each design is in category `dogs · <breed-slug>` and is supposed to depict
 * THAT SPECIFIC breed. Two failure modes:
 *   1. Not a recognizable dog at all (AI-stretched / abstract)
 *   2. Recognizable dog but wrong breed (Frenchie image in labrador-retriever slot)
 *
 * Per-design the prompt injects the claimed breed so Gemini can do a focused
 * yes/no on breed match. The verdict logic is computed in JS so we can tune.
 *
 * Output:
 *   /tmp/dogs-audit-results.jsonl
 *   /tmp/dogs-audit-kill-list.json
 *   /tmp/dogs-audit-keep-list.json
 *   /tmp/dogs-audit-breed-drift.json (recognizable dog but wrong breed)
 *
 * Cost: ~220 * 1400 in-tokens = ~$0.03 on gemini-2.5-flash.
 */
'use strict';

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const http = require('http');
const https = require('https');

const API_KEY = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
if (!API_KEY) { console.error('GEMINI_API_KEY not set'); process.exit(1); }

const MODEL = process.env.AUDIT_MODEL || 'gemini-2.5-flash';
const SERVER = process.env.AUDIT_SERVER || 'http://127.0.0.1:9905';
const OUT_JSONL = '/tmp/dogs-audit-results.jsonl';
const KILL_OUT = '/tmp/dogs-audit-kill-list.json';
const KEEP_OUT = '/tmp/dogs-audit-keep-list.json';
const DRIFT_OUT = '/tmp/dogs-audit-breed-drift.json';
const CONCURRENCY = parseInt(process.env.AUDIT_CONCURRENCY, 10) || 6;

// Pretty-print the breed slug for the prompt (labrador-retriever -> Labrador Retriever).
const BREED_PRETTY = {
  'beagle': 'Beagle',
  'bulldog': 'English Bulldog',
  'dachshund': 'Dachshund',
  'french-bulldog': 'French Bulldog',
  'german-shepherd': 'German Shepherd',
  'german-shorthaired-pointer': 'German Shorthaired Pointer',
  'golden-retriever': 'Golden Retriever',
  'labrador-retriever': 'Labrador Retriever',
  'pit-lab-mix': 'Pit Bull / Labrador mix (Labrabull)',
  'poodle': 'Poodle',
  'rottweiler': 'Rottweiler',
};

function buildPrompt(breed) {
  return `Describe what you see in this wallpaper pattern. The collection is "dogs" — each design is supposed to depict the breed: **${breed}**.

Answer in EXACTLY this format on three lines, nothing else:
SHAPE: is the main subject a clearly recognizable DOG (any breed)? Answer "dog" or "indistinct" (if it's smeared/stretched/abstract shapes, not clearly canine).
BREED: name the dog breed you most see in the image. Be specific (golden retriever, beagle, dachshund, poodle, french bulldog, german shepherd, rottweiler, etc.). Say "indistinct" if you can't tell, "mixed/generic" if it could be any dog.
MATCHES: does the breed you see match the claimed breed (${breed})? Answer "yes" if it clearly does, "close" if it's plausibly the same breed family (e.g. lab vs golden), or "no" if it's a different breed entirely.`;
}

function computeVerdict(shape, breed, matches) {
  const sh = (shape || '').toLowerCase();
  const m  = (matches || '').toLowerCase();
  if (sh === 'indistinct' || sh === 'none' || !sh.includes('dog')) {
    return { verdict: 'KILL', reason: 'shape:' + (shape || 'empty') };
  }
  if (m === 'no' || m.startsWith('no')) {
    return { verdict: 'BREED_DRIFT', reason: 'wrong-breed' };
  }
  // 'close' and 'yes' both keep
  return { verdict: 'KEEP', reason: '' };
}

function loadQueue() {
  const snap = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'data', 'designs.json'), 'utf8'));
  return snap.filter(d => (d.category || '').startsWith('dogs · ') && !d.user_removed)
             .map(d => ({
               id: d.id,
               title: d.title || '',
               category: d.category,
               breed_slug: d.category.replace('dogs · ', ''),
               is_published: d.is_published !== false,
             }));
}

function loadDone() {
  if (!fs.existsSync(OUT_JSONL)) return new Set();
  const lines = fs.readFileSync(OUT_JSONL, 'utf8').split('\n').filter(Boolean);
  const done = new Set();
  for (const l of lines) {
    try { const r = JSON.parse(l); if (r.id) done.add(r.id); } catch {}
  }
  return done;
}

function fetchImage(id) {
  return new Promise((resolve, reject) => {
    const req = http.get(`${SERVER}/designs/img/by-id/${id}`, res => {
      if (res.statusCode !== 200) { res.resume(); return reject(new Error(`img HTTP ${res.statusCode}`)); }
      const c = []; res.on('data', x => c.push(x));
      res.on('end', () => resolve({ buf: Buffer.concat(c), mime: res.headers['content-type'] || 'image/png' }));
      res.on('error', reject);
    });
    req.on('error', reject);
    req.setTimeout(15000, () => req.destroy(new Error('img timeout')));
  });
}

function callGemini(imgB64, mime, prompt) {
  return new Promise((resolve, reject) => {
    const body = JSON.stringify({
      contents: [{ parts: [
        { inline_data: { mime_type: mime.split(';')[0].trim(), data: imgB64 } },
        { text: prompt },
      ]}],
      generationConfig: { temperature: 0.1, maxOutputTokens: 80 },
    });
    const req = https.request({
      hostname: 'generativelanguage.googleapis.com',
      path: `/v1beta/models/${MODEL}:generateContent?key=${API_KEY}`,
      method: 'POST',
      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
    }, res => {
      const c = []; res.on('data', x => c.push(x));
      res.on('end', () => {
        const raw = Buffer.concat(c).toString('utf8');
        if (res.statusCode !== 200) return reject(new Error(`gemini HTTP ${res.statusCode}: ${raw.slice(0,200)}`));
        try {
          const j = JSON.parse(raw);
          const text = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
          resolve({ text, usage: j?.usageMetadata });
        } catch (e) { reject(new Error('gemini parse: ' + e.message)); }
      });
    });
    req.on('error', reject);
    req.setTimeout(45000, () => req.destroy(new Error('gemini timeout')));
    req.write(body); req.end();
  });
}

function parseResponse(text) {
  const shape   = (text.match(/SHAPE:\s*([^\n]+)/i)   || [, ''])[1].trim().slice(0, 60);
  const breed   = (text.match(/BREED:\s*([^\n]+)/i)   || [, ''])[1].trim().slice(0, 80);
  const matches = (text.match(/MATCHES:\s*([^\n]+)/i) || [, ''])[1].trim().slice(0, 40);
  const matchesClean = matches.replace(/\s*\(.+\)$/, '').replace(/\s*[—-].*$/, '').trim();
  const { verdict, reason } = computeVerdict(shape, breed, matchesClean);
  return { verdict, kill_reason: reason, shape, gemini_breed: breed, matches: matchesClean };
}

async function auditOne(item) {
  const t0 = Date.now();
  try {
    const { buf, mime } = await fetchImage(item.id);
    const breedPretty = BREED_PRETTY[item.breed_slug] || item.breed_slug;
    const { text, usage } = await callGemini(buf.toString('base64'), mime, buildPrompt(breedPretty));
    const v = parseResponse(text);
    return { id: item.id, title: item.title, breed_slug: item.breed_slug, breed_claimed: breedPretty,
             is_published: item.is_published, ...v, raw: text.slice(0, 200),
             tokens_in: usage?.promptTokenCount || 0, tokens_out: usage?.candidatesTokenCount || 0,
             ms: Date.now() - t0 };
  } catch (e) {
    return { id: item.id, title: item.title, breed_slug: item.breed_slug,
             verdict: 'ERROR', error: String(e.message || e).slice(0, 160), ms: Date.now() - t0 };
  }
}

async function main() {
  const queue = loadQueue();
  const done = loadDone();
  const work = queue.filter(it => !done.has(it.id));
  console.log(`queue: ${queue.length} total, ${done.size} done, ${work.length} to process`);
  console.log(`model: ${MODEL}, concurrency: ${CONCURRENCY}`);

  const out = fs.createWriteStream(OUT_JSONL, { flags: 'a' });
  let idx = 0, processed = 0;
  const total = work.length;
  const tally = { KEEP: 0, KILL: 0, BREED_DRIFT: 0, ERROR: 0 };
  let tin = 0, tout = 0;
  const t0 = Date.now();

  async function worker() {
    while (true) {
      const i = idx++;
      if (i >= work.length) break;
      const r = await auditOne(work[i]);
      tally[r.verdict] = (tally[r.verdict] || 0) + 1;
      tin += r.tokens_in || 0; tout += r.tokens_out || 0;
      out.write(JSON.stringify(r) + '\n');
      processed++;
      if (processed % 20 === 0 || processed === total) {
        const elapsed = ((Date.now() - t0) / 1000).toFixed(0);
        const rate = (processed / Math.max(1, (Date.now() - t0) / 1000)).toFixed(2);
        const cost = (tin * 0.10 / 1e6) + (tout * 0.40 / 1e6);
        console.log(`  [${processed}/${total}] ${elapsed}s @ ${rate}/s — KEEP=${tally.KEEP} DRIFT=${tally.BREED_DRIFT} KILL=${tally.KILL} ERR=${tally.ERROR} | $${cost.toFixed(4)}`);
      }
    }
  }
  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
  out.end();

  const allRows = fs.readFileSync(OUT_JSONL, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
  const kill = allRows.filter(r => r.verdict === 'KILL');
  const keep = allRows.filter(r => r.verdict === 'KEEP');
  const drift = allRows.filter(r => r.verdict === 'BREED_DRIFT');
  fs.writeFileSync(KILL_OUT, JSON.stringify(kill, null, 2));
  fs.writeFileSync(KEEP_OUT, JSON.stringify(keep, null, 2));
  fs.writeFileSync(DRIFT_OUT, JSON.stringify(drift, null, 2));
  console.log(`\n=== DONE ===`);
  console.log(`KILL (not-a-dog): ${kill.length}`);
  console.log(`BREED_DRIFT (wrong breed): ${drift.length}`);
  console.log(`KEEP (matches claimed breed): ${keep.length}`);
  console.log(`errors: ${allRows.filter(r => r.verdict === 'ERROR').length}`);
  console.log(`cost: $${((tin * 0.10 / 1e6) + (tout * 0.40 / 1e6)).toFixed(4)}`);
}

main().catch(e => { console.error('FATAL:', e); process.exit(1); });