`scripts/closet-vision.js`

111 lines
/**
 * Closet vision worker — scans data/closet for un-parsed photos and pipes
 * each to Ollama llava to extract garment bounding boxes + category +
 * color guesses. Updates the closet row in-place.
 *
 * Run on demand:  node scripts/closet-vision.js
 * Scheduled via ~/Library/LaunchAgents/com.steve.wms-closet-vision.plist
 * (every 120s).
 */
const Database = require('better-sqlite3');
const path = require('path');
const fs = require('fs');
const fetch = require('node-fetch');

const OLLAMA = process.env.OLLAMA_URL || 'http://127.0.0.1:11434';
const MODEL  = process.env.OLLAMA_VISION_MODEL || 'llava:latest';

// Two-pass extraction:
//   Pass 1 — classify the photo as "single garment" or "closet/outfit"
//   Pass 2 — extract items, capped by classification
// On a single-garment Shopify product shot llava previously hallucinated up
// to 12 alternating top/bottom rows. The classifier collapses that to 1.
async function classifyScene(imagePath) {
  const b64 = fs.readFileSync(imagePath).toString('base64');
  const prompt = `Look at this photo. Answer with ONE word only.
Reply "single" if the photo shows exactly one garment, accessory, shoe, or bag (e.g. a product shot on a model or flat-lay).
Reply "multi" only if the photo shows a wardrobe / closet / outfit with several distinct garments at once.
Reply "none" if you see no clothing.
Word only:`;
  try {
    const r = await fetch(`${OLLAMA}/api/generate`, {
      method: 'POST',
      headers: { 'content-type': 'application/json' },
      body: JSON.stringify({ model: MODEL, prompt, images: [b64], stream: false, options: { temperature: 0 } }),
    });
    const j = await r.json();
    const w = (j.response || '').toLowerCase().match(/single|multi|none/);
    return w ? w[0] : 'single';
  } catch { return 'single'; }
}

async function visionDescribe(imagePath, maxItems) {
  const buf = fs.readFileSync(imagePath);
  const b64 = buf.toString('base64');
  const prompt = maxItems === 1
    ? `You see a photograph of ONE garment, shoe, bag, or accessory. Return STRICT JSON only:
{"items":[{"category":"top|bottom|dress|outerwear|shoes|bag|accessory","color":"...","pattern":"solid|stripe|floral|plaid|...","material_guess":"cotton|silk|wool|denim|leather|other","vendor_guess":"unknown|<brand>","bbox":{"x":0,"y":0,"w":1,"h":1}}]}
Return EXACTLY ONE item. No prose. JSON only.`
    : `You see a photograph of a closet or outfit with multiple garments. Return STRICT JSON only:
{"items":[{"category":"top|bottom|dress|outerwear|shoes|bag|accessory","color":"...","pattern":"solid|stripe|floral|plaid|...","material_guess":"cotton|silk|wool|denim|leather|other","vendor_guess":"unknown|<brand>","bbox":{"x":0,"y":0,"w":1,"h":1}}]}
Return up to ${maxItems} distinct items. No prose. JSON only.`;
  const r = await fetch(`${OLLAMA}/api/generate`, {
    method: 'POST',
    headers: { 'content-type': 'application/json' },
    body: JSON.stringify({ model: MODEL, prompt, images: [b64], stream: false, options: { temperature: 0.2 } }),
  });
  const j = await r.json();
  const raw = j.response || '';
  const m = raw.match(/\{[\s\S]*\}/);
  if (!m) return { items: [] };
  try { return JSON.parse(m[0]); } catch { return { items: [], _raw: raw }; }
}

async function processOne(db, row) {
  if (!row.photo_path || !fs.existsSync(row.photo_path)) return;
  // /api/closet/photo is a single-file endpoint — user uploads one garment
  // at a time. Llava's "multi" classification on Shopify product shots is
  // unreliable (returns 12 alternating top/bottom rows). For now hard-cap
  // at 1. A future /api/closet/photo/bulk scan-my-closet flow can opt in
  // via source_meta.allow_multi.
  let meta = {}; try { meta = JSON.parse(row.source_meta || '{}'); } catch {}
  const allowMulti = !!meta.allow_multi;
  const scene = allowMulti ? await classifyScene(row.photo_path) : 'single';
  const cap = (allowMulti && scene === 'multi') ? 12 : 1;
  console.log(`[vision] ${row.id} ${row.photo_path}  scene=${scene} cap=${cap}`);
  if (scene === 'none') {
    db.prepare(`UPDATE closet SET source_meta=? WHERE id=?`)
      .run(JSON.stringify({ scene: 'none', note: 'no clothing detected' }), row.id);
    return;
  }
  const out = await visionDescribe(row.photo_path, cap);
  const items = (out.items || []).slice(0, cap);
  if (!items.length) return;
  const first = items[0];
  db.prepare(`UPDATE closet SET category=?, color=?, pattern=?, material_guess=?, vendor_guess=?, bbox=?, source_meta=? WHERE id=?`)
    .run(first.category, first.color, first.pattern, first.material_guess, first.vendor_guess,
         JSON.stringify(first.bbox || {}), JSON.stringify({ count: items.length, scene }), row.id);
  // Only fan out additional rows when the photo truly is a multi-garment shot.
  if (cap > 1 && items.length > 1) {
    const ins = db.prepare(`INSERT INTO closet (user_id, photo_path, category, color, pattern, material_guess, vendor_guess, bbox, acquired_via, source_meta)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'photo', ?)`);
    for (let i = 1; i < items.length; i++) {
      const it = items[i];
      ins.run(row.user_id, row.photo_path, it.category, it.color, it.pattern, it.material_guess, it.vendor_guess,
              JSON.stringify(it.bbox || {}), JSON.stringify({ from_photo: row.id, idx: i, scene }));
    }
  }
}

async function main() {
  const dbPath = path.join(__dirname, '..', 'data', 'whatsmystyle.db');
  const db = new Database(dbPath);
  const rows = db.prepare(`SELECT * FROM closet WHERE category IS NULL AND photo_path IS NOT NULL LIMIT 25`).all();
  console.log(`[vision] ${rows.length} pending`);
  for (const r of rows) {
    try { await processOne(db, r); } catch (e) { console.error('[vision] err', r.id, e.message); }
  }
}

if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });