Multi-photo visual-ID fusion: identify-multi now fuses up to 5 views (fronts[]) — a product matching across multiple photos gets a corroboration boost and outranks spurious single-view matches; Add/Update flows send all captured photos. Independent lever on the 28% single-shot problem (complements the augmentation retrain)

6cdd9d9b26c5584d1519d518a21bc91ecbafea4c · 2026-07-07 13:02:32 -0700 · Steve Abrams

Files touched

M public/index.html
M server.js

Diff

commit 6cdd9d9b26c5584d1519d518a21bc91ecbafea4c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jul 7 13:02:32 2026 -0700

    Multi-photo visual-ID fusion: identify-multi now fuses up to 5 views (fronts[]) — a product matching across multiple photos gets a corroboration boost and outranks spurious single-view matches; Add/Update flows send all captured photos. Independent lever on the 28% single-shot problem (complements the augmentation retrain)
---
 public/index.html |  4 ++--
 server.js         | 22 ++++++++++++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/public/index.html b/public/index.html
index 0c15180..f7c6588 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1556,7 +1556,7 @@ async function bindProduct(mfr,sku,confirmed){ const qs=mfr?('mfr='+encodeURICom
 // visual-ID). If it strongly matches an existing product, WARN (non-blocking) + offer "Update instead".
 async function addDedupCheck(){ const mySession=_addSession;   // guard against a stale result landing in a reopened modal
   const photos=addPhotosList(); if(!photos.length || _addMode!=='add') return;
-  let r; try{ r=await(await fetch('/api/identify-multi',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({front:photos[photos.length-1]})})).json(); }catch(e){ return; }
+  let r; try{ r=await(await fetch('/api/identify-multi',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({fronts:photos})})).json(); }catch(e){ return; }
   const prim=r&&r.primary; if(!prim || !r.confidence || r.confidence==='low') return;   // only warn on a confident match
   if(_addMode!=='add' || _addSession!==mySession) return;   // modal switched/reopened mid-request — discard
   const label=prim.pattern || prim.dw_sku || prim.mfr_sku || 'an existing item';
@@ -1583,7 +1583,7 @@ async function addResolveUpdate(){ const mfr=$('#addMfr').value.trim(); if(!mfr)
 async function visualIdentify(){ const photos=addPhotosList(); if(!photos.length) return;
   $('#addTarget').innerHTML='🔍 Identifying by pattern…';
   let r; try{ r=await(await fetch('/api/identify-multi',{method:'POST',headers:{'Content-Type':'application/json'},
-    body:JSON.stringify({back:photos[0], front:photos[photos.length-1]})})).json(); }catch(e){ $('#addTarget').textContent='Visual ID failed.'; return; }
+    body:JSON.stringify({back:photos[0], fronts:photos})})).json(); }catch(e){ $('#addTarget').textContent='Visual ID failed.'; return; }
   const cands=(r.visual||[]).slice(0,4), prim=r.primary; let bound=null;
   // AUTO-bind ONLY a confident primary. Low-confidence or candidate matches require an explicit
   // tap — never silently bind a live customer product off an ambiguous pattern photo.
diff --git a/server.js b/server.js
index 0144eae..b652cfd 100644
--- a/server.js
+++ b/server.js
@@ -1240,9 +1240,27 @@ const appHandler = (req, res) => {
           for (const t of toks) { codeId = await identifyUnified(t).catch(() => null); if (codeId) { code = t; qrUsed = true; break; } }
         }
         const backVendor = (backOcr && backOcr.vendor) || null;   // vendor name off the full back label
-        // FRONT → CLIP visual search (nearest catalog products by actual pixels)
+        // FRONT → CLIP visual search. MULTI-PHOTO FUSION: real handheld photos are noisy (28% single-shot
+        // top-1), so if several views of the same sample are sent, fuse them — a product matching across
+        // multiple views is corroborated and outranks a spurious single-view match.
+        const fronts = (Array.isArray(p.fronts) && p.fronts.length ? p.fronts : (p.front ? [p.front] : [])).slice(0, 5).map(strip).filter(Boolean);
         let visual = [];
-        if (p.front) visual = await visualSearch(strip(p.front), 8);
+        if (fronts.length === 1) {
+          visual = await visualSearch(fronts[0], 8);
+        } else if (fronts.length > 1) {
+          const lists = await Promise.all(fronts.map(f => visualSearch(f, 8).catch(() => [])));
+          const agg = new Map();
+          for (const list of lists) for (const it of list) {
+            const key = it.dw_sku || it.vc_id || it.mfr_sku || it.image || JSON.stringify(it);
+            const e = agg.get(key) || { it, scores: [] }; e.scores.push(it.score || 0); agg.set(key, e);
+          }
+          const nViews = lists.length;
+          visual = Array.from(agg.values()).map(e => {
+            const mean = e.scores.reduce((a, b) => a + b, 0) / e.scores.length;
+            const bonus = 0.05 * ((e.scores.length - 1) / Math.max(1, nViews - 1));   // multi-view corroboration
+            return Object.assign({}, e.it, { score: Math.min(1, mean + bonus), views: e.scores.length });
+          }).sort((a, b) => b.score - a.score).slice(0, 8);
+        }
         // vendor clue: with no exact code, prefer visual matches from the vendor printed on the label
         if (!codeId && backVendor && visual.length) {
           const vlow = backVendor.toLowerCase().split(/\s+/)[0];

← 724673e auto-save: 2026-07-07T13:00:27 (1 files) — visual-search/tra · back to Dw Photo Capture · robustness_fusion: note CDN rate-limiting makes the A/B unre de2aca3 →