`agents/instagram-agent/vision-classify.mjs`

87 lines
#!/usr/bin/env node
/**
 * vision-classify.mjs — binary Gemini 2.0 Flash classifier for TK-11367.
 * For each local image, asks: is this a styled ROOM SETTING / interior scene,
 * and/or a JEWELRY DISPLAY CASE? Returns strict JSON per image.
 *
 *   node vision-classify.mjs <img1.jpg> [img2.jpg ...]        # ad-hoc
 *   node vision-classify.mjs --manifest <file.jsonl>          # batch (see below)
 *
 * Manifest lines: {"id": "...", "path": "/abs/img.jpg"}  ->  appends a result
 * line {"id","room","jewelry","raw"} to <manifest>.results.jsonl, idempotent
 * (skips ids already in the results file).
 */
import fs from 'fs';
import path from 'path';

const ENV = fs.readFileSync(path.join(process.env.HOME, 'Projects/secrets-manager/.env'), 'utf8');
const KEY = (ENV.match(/^GEMINI_API_KEY=(.+)$/m) || [])[1]?.trim().replace(/"/g, '');
if (!KEY) { console.error('no GEMINI_API_KEY'); process.exit(1); }
const MODEL = process.env.GEMINI_MODEL || 'gemini-3.6-flash';
const ENDPOINT = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${KEY}`;

const PROMPT = `You are auditing a wallpaper/wallcovering brand's Instagram images.
Classify THIS image on two independent yes/no questions:

1. ROOM: Is this a styled ROOM SETTING / interior scene — i.e. the wallcovering
   is shown installed on a wall as part of a decorated room or vignette, with
   furniture, decor, lamps, plants, or architectural context (bedroom, living
   room, lobby, office, dining, hallway, etc.)? A flat swatch, a rolled bolt,
   a folded fabric drape, a close-up texture, a pattern tile, or a product-on-
   white-background shot is NOT a room setting -> ROOM: no.

2. JEWELRY: Does this image show a JEWELRY DISPLAY CASE, jewelry counter, or
   jewelry-store glass display presenting the product? -> yes/no.

Reply with ONLY a compact JSON object, no prose, no markdown fences:
{"room": true|false, "jewelry": true|false, "scene": "<3-6 word description>"}`;

async function classify(imgPath) {
  const b64 = fs.readFileSync(imgPath).toString('base64');
  const mime = imgPath.toLowerCase().endsWith('.png') ? 'image/png' : 'image/jpeg';
  const body = {
    contents: [{ parts: [{ text: PROMPT }, { inline_data: { mime_type: mime, data: b64 } }] }],
    generationConfig: { temperature: 0, maxOutputTokens: 120 },
  };
  for (let attempt = 0; attempt < 4; attempt++) {
    try {
      const res = await fetch(ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
      if (res.status === 429 || res.status >= 500) { await new Promise(r => setTimeout(r, 1500 * (attempt + 1))); continue; }
      const j = await res.json();
      const txt = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
      const m = txt.match(/\{[\s\S]*\}/);
      if (!m) return { room: null, jewelry: null, scene: 'PARSE_FAIL', raw: txt.slice(0, 120) };
      const parsed = JSON.parse(m[0]);
      return { room: !!parsed.room, jewelry: !!parsed.jewelry, scene: parsed.scene || '' };
    } catch (e) {
      await new Promise(r => setTimeout(r, 1500 * (attempt + 1)));
    }
  }
  return { room: null, jewelry: null, scene: 'API_FAIL' };
}

const args = process.argv.slice(2);
if (args[0] === '--manifest') {
  const manifest = args[1];
  const outFile = manifest.replace(/\.jsonl$/, '') + '.results.jsonl';
  const done = new Set(fs.existsSync(outFile)
    ? fs.readFileSync(outFile, 'utf8').trim().split('\n').filter(Boolean).map(l => { try { return JSON.parse(l).id; } catch { return null; } }).filter(Boolean)
    : []);
  const rows = fs.readFileSync(manifest, 'utf8').trim().split('\n').filter(Boolean).map(l => JSON.parse(l)).filter(r => !done.has(r.id));
  console.error(`manifest: ${rows.length} to classify (${done.size} already done) -> ${outFile}`);
  let n = 0, hits = 0;
  for (const r of rows) {
    if (!fs.existsSync(r.path)) { fs.appendFileSync(outFile, JSON.stringify({ id: r.id, room: null, jewelry: null, scene: 'NO_FILE' }) + '\n'); continue; }
    const v = await classify(r.path);
    fs.appendFileSync(outFile, JSON.stringify({ id: r.id, ...v }) + '\n');
    n++; if (v.room || v.jewelry) { hits++; console.error(`  HIT[${v.room ? 'R' : ''}${v.jewelry ? 'J' : ''}] ${r.id} — ${v.scene}`); }
    if (n % 100 === 0) console.error(`  … ${n}/${rows.length} classified, ${hits} hits so far`);
  }
  console.error(`DONE — ${n} classified, ${hits} room/jewelry hits.`);
} else {
  for (const p of args) {
    const v = await classify(p);
    console.log(path.basename(p), '->', JSON.stringify(v));
  }
}