[object Object]

← back to Whatsmystyle

closet vision — wire launchd drain + cap to 1 item per single-garment shot

4904ff45ceed0634d01bdbc5516a760e86676aee · 2026-05-12 17:46:28 -0700 · SteveStudio2

- Add com.steve.wms-closet-vision launchd plist (120s cadence, mirrors
  wms-embed-drainer pattern)
- ecosystem.config.js: pin OLLAMA_VISION_MODEL=llava:latest (only model
  loaded locally; default was llava:13b which isn't on this machine)
- closet-vision.js: cap items=1 unless source_meta.allow_multi is set,
  because /api/closet/photo is single-file and llava:7b hallucinates 12
  alternating top/bottom rows on Shopify product shots
- Validated end-to-end: 3 garment uploads → 3 closet rows (was 13)
  with correct categories (top/top/outerwear)

Files touched

Diff

commit 4904ff45ceed0634d01bdbc5516a760e86676aee
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Tue May 12 17:46:28 2026 -0700

    closet vision — wire launchd drain + cap to 1 item per single-garment shot
    
    - Add com.steve.wms-closet-vision launchd plist (120s cadence, mirrors
      wms-embed-drainer pattern)
    - ecosystem.config.js: pin OLLAMA_VISION_MODEL=llava:latest (only model
      loaded locally; default was llava:13b which isn't on this machine)
    - closet-vision.js: cap items=1 unless source_meta.allow_multi is set,
      because /api/closet/photo is single-file and llava:7b hallucinates 12
      alternating top/bottom rows on Shopify product shots
    - Validated end-to-end: 3 garment uploads → 3 closet rows (was 13)
      with correct categories (top/top/outerwear)
---
 ecosystem.config.js          |  2 ++
 scripts/closet-vision.js     | 82 +++++++++++++++++++++++++++++++++-----------
 scripts/e2e-random-photos.js |  5 +--
 3 files changed, 67 insertions(+), 22 deletions(-)

diff --git a/ecosystem.config.js b/ecosystem.config.js
index ae18281..2493bc7 100644
--- a/ecosystem.config.js
+++ b/ecosystem.config.js
@@ -6,6 +6,8 @@ module.exports = {
     env: {
       NODE_ENV: 'production',
       PORT: 9777,
+      OLLAMA_URL: 'http://127.0.0.1:11434',
+      OLLAMA_VISION_MODEL: 'llava:latest',
     },
     autorestart: true,
     max_memory_restart: '500M',
diff --git a/scripts/closet-vision.js b/scripts/closet-vision.js
index 323ecd5..94af848 100644
--- a/scripts/closet-vision.js
+++ b/scripts/closet-vision.js
@@ -1,10 +1,11 @@
 /**
  * Closet vision worker — scans data/closet for un-parsed photos and pipes
- * each to Ollama llava:13b to extract garment bounding boxes + category +
+ * each to Ollama llava to extract garment bounding boxes + category +
  * color guesses. Updates the closet row in-place.
  *
  * Run on demand:  node scripts/closet-vision.js
- * Or via launchd every 5 min (added in a later tick).
+ * Scheduled via ~/Library/LaunchAgents/com.steve.wms-closet-vision.plist
+ * (every 120s).
  */
 const Database = require('better-sqlite3');
 const path = require('path');
@@ -12,16 +13,42 @@ const fs = require('fs');
 const fetch = require('node-fetch');
 
 const OLLAMA = process.env.OLLAMA_URL || 'http://127.0.0.1:11434';
-const MODEL  = process.env.OLLAMA_VISION_MODEL || 'llava:13b';
+const MODEL  = process.env.OLLAMA_VISION_MODEL || 'llava:latest';
 
-async function visionDescribe(imagePath) {
+// Two-pass extraction:
+//   Pass 1 — classify the photo as "single garment" or "closet/outfit"
+//   Pass 2 — extract items, capped by classification
+// On a single-garment Shopify product shot llava previously hallucinated up
+// to 12 alternating top/bottom rows. The classifier collapses that to 1.
+async function classifyScene(imagePath) {
+  const b64 = fs.readFileSync(imagePath).toString('base64');
+  const prompt = `Look at this photo. Answer with ONE word only.
+Reply "single" if the photo shows exactly one garment, accessory, shoe, or bag (e.g. a product shot on a model or flat-lay).
+Reply "multi" only if the photo shows a wardrobe / closet / outfit with several distinct garments at once.
+Reply "none" if you see no clothing.
+Word only:`;
+  try {
+    const r = await fetch(`${OLLAMA}/api/generate`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ model: MODEL, prompt, images: [b64], stream: false, options: { temperature: 0 } }),
+    });
+    const j = await r.json();
+    const w = (j.response || '').toLowerCase().match(/single|multi|none/);
+    return w ? w[0] : 'single';
+  } catch { return 'single'; }
+}
+
+async function visionDescribe(imagePath, maxItems) {
   const buf = fs.readFileSync(imagePath);
   const b64 = buf.toString('base64');
-  const prompt = `You see a photograph of a closet, a single garment, or an outfit.
-Return STRICT JSON only with this shape:
+  const prompt = maxItems === 1
+    ? `You see a photograph of ONE garment, shoe, bag, or accessory. Return STRICT JSON only:
 {"items":[{"category":"top|bottom|dress|outerwear|shoes|bag|accessory","color":"...","pattern":"solid|stripe|floral|plaid|...","material_guess":"cotton|silk|wool|denim|leather|other","vendor_guess":"unknown|<brand>","bbox":{"x":0,"y":0,"w":1,"h":1}}]}
-If you see ONE garment, return one item. If you see a closet, return up to 12 items.
-No prose. JSON only.`;
+Return EXACTLY ONE item. No prose. JSON only.`
+    : `You see a photograph of a closet or outfit with multiple garments. Return STRICT JSON only:
+{"items":[{"category":"top|bottom|dress|outerwear|shoes|bag|accessory","color":"...","pattern":"solid|stripe|floral|plaid|...","material_guess":"cotton|silk|wool|denim|leather|other","vendor_guess":"unknown|<brand>","bbox":{"x":0,"y":0,"w":1,"h":1}}]}
+Return up to ${maxItems} distinct items. No prose. JSON only.`;
   const r = await fetch(`${OLLAMA}/api/generate`, {
     method: 'POST',
     headers: { 'content-type': 'application/json' },
@@ -36,22 +63,37 @@ No prose. JSON only.`;
 
 async function processOne(db, row) {
   if (!row.photo_path || !fs.existsSync(row.photo_path)) return;
-  console.log(`[vision] ${row.id} ${row.photo_path}`);
-  const out = await visionDescribe(row.photo_path);
-  const items = (out.items || []).slice(0, 12);
+  // /api/closet/photo is a single-file endpoint — user uploads one garment
+  // at a time. Llava's "multi" classification on Shopify product shots is
+  // unreliable (returns 12 alternating top/bottom rows). For now hard-cap
+  // at 1. A future /api/closet/photo/bulk scan-my-closet flow can opt in
+  // via source_meta.allow_multi.
+  let meta = {}; try { meta = JSON.parse(row.source_meta || '{}'); } catch {}
+  const allowMulti = !!meta.allow_multi;
+  const scene = allowMulti ? await classifyScene(row.photo_path) : 'single';
+  const cap = (allowMulti && scene === 'multi') ? 12 : 1;
+  console.log(`[vision] ${row.id} ${row.photo_path}  scene=${scene} cap=${cap}`);
+  if (scene === 'none') {
+    db.prepare(`UPDATE closet SET source_meta=? WHERE id=?`)
+      .run(JSON.stringify({ scene: 'none', note: 'no clothing detected' }), row.id);
+    return;
+  }
+  const out = await visionDescribe(row.photo_path, cap);
+  const items = (out.items || []).slice(0, cap);
   if (!items.length) return;
-  // Update the parent row with first item's broad attrs
   const first = items[0];
   db.prepare(`UPDATE closet SET category=?, color=?, pattern=?, material_guess=?, vendor_guess=?, bbox=?, source_meta=? WHERE id=?`)
     .run(first.category, first.color, first.pattern, first.material_guess, first.vendor_guess,
-         JSON.stringify(first.bbox || {}), JSON.stringify({ count: items.length }), row.id);
-  // Append additional items as new closet rows linked to same photo
-  const ins = db.prepare(`INSERT INTO closet (user_id, photo_path, category, color, pattern, material_guess, vendor_guess, bbox, acquired_via, source_meta)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'photo', ?)`);
-  for (let i = 1; i < items.length; i++) {
-    const it = items[i];
-    ins.run(row.user_id, row.photo_path, it.category, it.color, it.pattern, it.material_guess, it.vendor_guess,
-            JSON.stringify(it.bbox || {}), JSON.stringify({ from_photo: row.id, idx: i }));
+         JSON.stringify(first.bbox || {}), JSON.stringify({ count: items.length, scene }), row.id);
+  // Only fan out additional rows when the photo truly is a multi-garment shot.
+  if (cap > 1 && items.length > 1) {
+    const ins = db.prepare(`INSERT INTO closet (user_id, photo_path, category, color, pattern, material_guess, vendor_guess, bbox, acquired_via, source_meta)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'photo', ?)`);
+    for (let i = 1; i < items.length; i++) {
+      const it = items[i];
+      ins.run(row.user_id, row.photo_path, it.category, it.color, it.pattern, it.material_guess, it.vendor_guess,
+              JSON.stringify(it.bbox || {}), JSON.stringify({ from_photo: row.id, idx: i, scene }));
+    }
   }
 }
 
diff --git a/scripts/e2e-random-photos.js b/scripts/e2e-random-photos.js
index 567070c..fa13238 100644
--- a/scripts/e2e-random-photos.js
+++ b/scripts/e2e-random-photos.js
@@ -220,9 +220,10 @@ async function main() {
   // wires it). For an honest e2e test we trigger it inline rather than
   // waiting for a cron that doesn't exist.
   try {
-    const { execSync } = require('child_process');
+    const { spawnSync } = require('child_process');
     const env = { ...process.env, OLLAMA_VISION_MODEL: process.env.OLLAMA_VISION_MODEL || 'llava:latest' };
-    execSync(`node ${path.join(__dirname, 'closet-vision.js')}`, { env, stdio: 'inherit', timeout: 180000 });
+    const r = spawnSync('node', [path.join(__dirname, 'closet-vision.js')], { env, stdio: 'inherit', timeout: 300000 });
+    if (r.status !== 0) console.log(`  ⚠ worker exit ${r.status} signal=${r.signal || ''}`);
   } catch (e) {
     console.log(`  ⚠ worker run failed: ${e.message}`);
   }

← 8af7af1 scripts/e2e-random-photos.js — real-photo E2E harness  ·  back to Whatsmystyle  ·  tryon-worker: hook Gemini call into cost-tracker (logGemini) 63fc189 →