feat: visual identify — Gemini compares the ACTUAL user photo against candidate catalog IMAGES (not typed attributes) and floats the true pixel-match to the front, or honestly reports no-match. Adds fetchImageB64 + geminiVisualMatch; /api/recognize returns a visual{} block. ~$0.0036/recognize (6 imgs). Correctly rejected 5 non-matching grasscloths in test.

873541c55f33fbacfe69056c7b150455f129e68a · 2026-07-06 18:47:15 -0700 · Steve Abrams
Files touched

M server.js
Diff

commit 873541c55f33fbacfe69056c7b150455f129e68a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 6 18:47:15 2026 -0700

    feat: visual identify — Gemini compares the ACTUAL user photo against candidate catalog IMAGES (not typed attributes) and floats the true pixel-match to the front, or honestly reports no-match. Adds fetchImageB64 + geminiVisualMatch; /api/recognize returns a visual{} block. ~$0.0036/recognize (6 imgs). Correctly rejected 5 non-matching grasscloths in test.
---
 server.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/server.js b/server.js
index ec5476e..a913a10 100644
--- a/server.js
+++ b/server.js
@@ -542,6 +542,49 @@ function geminiVision(b64, prompt, timeoutMs) {
   });
 }
 
+// Fetch a catalog image URL → base64 (bounded), so it can be shown to the vision model. $0.
+function fetchImageB64(url, maxBytes) {
+  return new Promise(resolve => {
+    let u; try { u = new URL(url); } catch (e) { return resolve(null); }
+    if (u.protocol !== 'https:' && u.protocol !== 'http:') return resolve(null);
+    // ask Shopify CDN for a small variant to keep it light
+    if (/shopify/i.test(u.hostname) && !/[?&]width=/.test(u.href)) u.href += (u.search ? '&' : '?') + 'width=400';
+    const lib = u.protocol === 'https:' ? https : http;
+    const rq = lib.get(u.href, { timeout: 6000 }, res => {
+      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { res.resume(); return resolve(fetchImageB64(res.headers.location, maxBytes)); }
+      if (res.statusCode !== 200) { res.resume(); return resolve(null); }
+      const chunks = []; let n = 0;
+      res.on('data', c => { n += c.length; if (n > (maxBytes || 3e6)) { rq.destroy(); resolve(null); } else chunks.push(c); });
+      res.on('end', () => resolve(Buffer.concat(chunks).toString('base64')));
+    });
+    rq.on('error', () => resolve(null)); rq.on('timeout', () => { rq.destroy(); resolve(null); });
+  });
+}
+
+// VISION MATCH: show Gemini the user's ACTUAL photo + candidate catalog images and have it pick the
+// one that is the SAME pattern by pixels (not typed attributes). images = [b64,...]; returns
+// { best: <0-based idx into images, or -1>, confidence }.
+function geminiVisualMatch(userB64, images) {
+  return new Promise(resolve => {
+    if (!GEMINI_API_KEY || !images.length) return resolve(null);
+    const parts = [{ text: `Image 1 is a PHOTO of a physical wallcovering/fabric SAMPLE. The remaining images (2 to ${images.length + 1}) are catalog product photos. Which ONE catalog image is the SAME pattern/material as the sample in image 1? Judge only the actual visual pattern, texture and colors — ignore crop, lighting, angle, and background. Reply ONLY compact JSON: {"match": <the catalog image number 2-${images.length + 1}, or 0 if none clearly match>, "confidence": <0-1>}.` },
+      { inline_data: { mime_type: 'image/jpeg', data: userB64 } }];
+    images.forEach(im => parts.push({ inline_data: { mime_type: 'image/jpeg', data: im } }));
+    let target; try { target = new URL(`https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_VISION_MODEL}:generateContent?key=${encodeURIComponent(GEMINI_API_KEY)}`); }
+    catch (e) { return resolve(null); }
+    const payload = JSON.stringify({ contents: [{ parts }], generationConfig: { temperature: 0, responseMimeType: 'application/json' } });
+    const rq = https.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: 45000 },
+      res => { let d = ''; res.on('data', c => d += c); res.on('end', () => {
+        try { const j = JSON.parse(d);
+          const t = ((j.candidates && j.candidates[0] && j.candidates[0].content && j.candidates[0].content.parts) || []).map(x => x.text || '').join('');
+          const o = JSON.parse(t); const num = parseInt(o.match, 10);
+          resolve({ best: (num >= 2 ? num - 2 : -1), confidence: (o.confidence ?? null) });
+        } catch (e) { resolve(null); } }); });
+    rq.on('error', () => resolve(null)); rq.on('timeout', () => { rq.destroy(); resolve(null); });
+    rq.write(payload); rq.end();
+  });
+}
+
 // Google Cloud Vision DOCUMENT_TEXT_DETECTION (OCR only — no reasoning). Returns { text, error }.
 function gcvOcr(b64, timeoutMs) {
   return new Promise(resolve => {
@@ -904,13 +947,32 @@ const appHandler = (req, res) => {
       const terms = similarTerms(a);
       let items = []; try { items = await unifiedSimilar(terms); } catch (e) { items = []; }
       const code = (a.code || '').toString().toUpperCase().replace(/\s+/g, '');
+      // VISION MATCH: don't stop at typed attributes — show Gemini the ACTUAL photo + the top
+      // candidate IMAGES and let it pick the true pixel match, then float it to the front.
+      let visual = null;
+      if (items.length && GEMINI_API_KEY && p.visualMatch !== false) {
+        try {
+          const top = items.filter(x => x.image).slice(0, 5);
+          const fetched = (await Promise.all(top.map(x => fetchImageB64(x.image, 2e6))))
+            .map((bb, i) => ({ bb, item: top[i] })).filter(x => x.bb);
+          if (fetched.length) {
+            const vm = await geminiVisualMatch(b64, fetched.map(x => x.bb));
+            const cost = 0.0006 * (fetched.length + 1);
+            if (vm && vm.best >= 0 && vm.best < fetched.length) {
+              const chosen = fetched[vm.best].item;
+              items = [chosen, ...items.filter(x => x !== chosen)];
+              visual = { matched_sku: chosen.dw_sku || null, matched_mfr: chosen.mfr || null, matched_title: chosen.title || null, matched_image: chosen.image || null, confidence: vm.confidence, via: 'gemini-visual', compared: fetched.length, cost_usd: cost };
+            } else { visual = { matched_sku: null, confidence: vm && vm.confidence, via: 'gemini-visual', note: 'no confident visual match', compared: fetched.length, cost_usd: cost }; }
+          }
+        } catch (e) { visual = { error: e.message }; }
+      }
       return send(res, 200, { ok: !r.error, model: (engine === 'macvision' ? OLLAMA_VISION_MODEL : GEMINI_VISION_MODEL), engine,
         recognized: {
           description: a.description || null, motif: a.motif || null, style: a.style || null,
           material: a.material || null, colors: Array.isArray(a.colors) ? a.colors.slice(0, 6) : [],
           background: a.background || null, scale: a.scale || null, code: code || null
         },
-        terms, total: items.length, items, err: r.error || null });
+        visual, terms, total: items.length, items, err: r.error || null });
     });
     return;
   }
← 11691f4 feat: Option B — Mac-side sticker watcher (use both). sticke · back to Dw Photo Capture · feat: CLIP visual search + multi-photo front/back fusion ide 023d1b2 →