← back to Dw Photo Capture
feat: CLIP visual search + multi-photo front/back fusion identify
023d1b23767dc724318d76b26ea422920a1dc78e · 2026-07-06 19:01:32 -0700 · Steve Abrams
- visual-search/embed_catalog.py: batch CLIP-embeds (ViT-B-32/laion2b) the 316k catalog images →
image_embeddings (bytea), resumable, ~19 img/s. Full batch running.
- visual-search/search_service.py: in-RAM brute-force cosine service (no pgvector needed), pm2
dwphoto-vsearch :9914 — photo → visually-nearest catalog products.
- /api/identify-multi: fuses ALL clues — BACK label OCR→code→crossref (exact) + FRONT pattern CLIP
visual search; returns primary + confidence + corroborated flag when both agree.
- Fixed Node→python Content-Length (chunked body read as empty). Verified: WDW2310 code + grasscloth
visual 0.91 live.
Files touched
M server.jsA visual-search/embed_catalog.pyA visual-search/search_service.py
Diff
commit 023d1b23767dc724318d76b26ea422920a1dc78e
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 6 19:01:32 2026 -0700
feat: CLIP visual search + multi-photo front/back fusion identify
- visual-search/embed_catalog.py: batch CLIP-embeds (ViT-B-32/laion2b) the 316k catalog images →
image_embeddings (bytea), resumable, ~19 img/s. Full batch running.
- visual-search/search_service.py: in-RAM brute-force cosine service (no pgvector needed), pm2
dwphoto-vsearch :9914 — photo → visually-nearest catalog products.
- /api/identify-multi: fuses ALL clues — BACK label OCR→code→crossref (exact) + FRONT pattern CLIP
visual search; returns primary + confidence + corroborated flag when both agree.
- Fixed Node→python Content-Length (chunked body read as empty). Verified: WDW2310 code + grasscloth
visual 0.91 live.
---
server.js | 53 ++++++++++++++++++++++++++
visual-search/embed_catalog.py | 83 +++++++++++++++++++++++++++++++++++++++++
visual-search/search_service.py | 77 ++++++++++++++++++++++++++++++++++++++
3 files changed, 213 insertions(+)
diff --git a/server.js b/server.js
index a913a10..1a2470f 100644
--- a/server.js
+++ b/server.js
@@ -25,6 +25,7 @@ const FM_ENABLED = () => !!(FM && process.env.FM_CLARIS_EMAIL && process.env.FM_
// Today's date as MM/DD/YYYY in the business timezone (Kamatera runs UTC; stamp must be Pacific
// so an evening scan never records tomorrow's date on the FileMaker record). Override via FM_TZ.
function todayMDY() { return new Date().toLocaleDateString('en-US', { timeZone: process.env.FM_TZ || 'America/Los_Angeles', year: 'numeric', month: '2-digit', day: '2-digit' }); }
+const VSEARCH_URL = process.env.VSEARCH_URL || 'http://127.0.0.1:9914'; // CLIP visual-search service
const ROOT = __dirname;
const DATA = path.join(ROOT, 'data');
@@ -1106,6 +1107,45 @@ const appHandler = (req, res) => {
return;
}
+ // Multi-photo identify: fuse ALL clues — BACK label (OCR code → exact crossref) + FRONT pattern
+ // (CLIP visual search over the catalog). Body: { back, front } dataUrls (either optional).
+ if (u.pathname === '/api/identify-multi' && req.method === 'POST') {
+ let body = ''; req.on('data', c => { body += c; if (body.length > 30 * 1024 * 1024) req.destroy(); });
+ req.on('end', async () => {
+ let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
+ const strip = s => (s || '').replace(/^data:image\/\w+;base64,/, '');
+ try {
+ // BACK → OCR the code → resolve to a canonical identity (exact)
+ let code = null, codeId = null, backOcr = null;
+ if (p.back) {
+ const buf = Buffer.from(strip(p.back), 'base64');
+ backOcr = await runOcr('gemini', buf).catch(() => null);
+ const cands = [backOcr && backOcr.barcode, backOcr && backOcr.top, ...((backOcr && backOcr.candidates) || [])].filter(Boolean);
+ for (const c of cands) { codeId = await identifyUnified(c).catch(() => null); if (codeId) { code = c; break; } }
+ }
+ // FRONT → CLIP visual search (nearest catalog products by actual pixels)
+ let visual = [];
+ if (p.front) visual = await visualSearch(strip(p.front), 8);
+ // FUSE the clues
+ let primary = null, confidence = 'low', source = null, corroborated = false;
+ if (codeId) {
+ primary = { dw_sku: codeId.internal_sku, mfr_sku: codeId.mfr_sku, mfr_code: codeId.mfr_code, vendor: codeId.vendor, pattern: codeId.pattern, image: codeId.image };
+ source = 'code'; confidence = 'high';
+ corroborated = visual.some(v => (v.dw_sku && codeId.internal_sku && String(v.dw_sku).toUpperCase() === String(codeId.internal_sku).toUpperCase()) || (v.mfr_sku && codeId.mfr_sku && v.mfr_sku === codeId.mfr_sku));
+ if (corroborated) confidence = 'very-high';
+ } else if (visual.length) {
+ const v0 = visual[0];
+ primary = { dw_sku: v0.dw_sku, mfr_sku: v0.mfr_sku, vendor: v0.vendor, pattern: v0.pattern, image: v0.image };
+ source = 'visual'; confidence = v0.score >= 0.92 ? 'high' : v0.score >= 0.85 ? 'medium' : 'low';
+ }
+ send(res, 200, { ok: true, found: !!primary, primary, confidence, source, corroborated,
+ code, code_identity: codeId, visual,
+ back_read: backOcr && { top: backOcr.top, candidates: backOcr.candidates, vendor: backOcr.vendor, fields: backOcr.fields } });
+ } catch (e) { send(res, 500, { err: e.message }); }
+ });
+ return;
+ }
+
// Sample-request lookup: BEST-ID the scanned item against the unified catalog, then find the
// most-recent OPEN sample request for it in FileMaker (client-ordered filled + vendor-ordered
// filled + WP-sample-sent EMPTY). Returns identity + client name + dates for the print popup.
@@ -1760,6 +1800,19 @@ function identifyUnified(code) {
});
}
+// Query the CLIP visual-search service (front/pattern photo → visually-nearest catalog products).
+function visualSearch(b64, k) {
+ return new Promise(resolve => {
+ let target; try { target = new URL(VSEARCH_URL + '/search'); } catch (e) { return resolve([]); }
+ const lib = target.protocol === 'https:' ? https : http;
+ const payload = JSON.stringify({ image: b64, k: k || 8 });
+ const rq = lib.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }, timeout: 20000 },
+ res => { let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve((JSON.parse(d).results) || []); } catch (e) { resolve([]); } }); });
+ rq.on('error', () => resolve([])); rq.on('timeout', () => { rq.destroy(); resolve([]); });
+ rq.write(payload); rq.end();
+ });
+}
+
server.listen(PORT, '0.0.0.0', () => {
console.log(`DW Photo Capture on http://0.0.0.0:${PORT} (Shopify push: ${TOKEN ? 'ON' : 'OFF — no token'}, sheet GRS: ${SHEET.length})`);
rebuildIndex(); // sheet-only items searchable immediately
diff --git a/visual-search/embed_catalog.py b/visual-search/embed_catalog.py
new file mode 100644
index 0000000..79fc5aa
--- /dev/null
+++ b/visual-search/embed_catalog.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# Batch CLIP-embed the dw_unified catalog images for visual product search.
+# Downloads each vendor_catalog.image_url, embeds with open_clip ViT-B-32 (laion2b), stores the
+# L2-normalized float32 vector in image_embeddings. Resumable: rows already in image_embeddings
+# (success OR permanent-fail placeholder with NULL embedding) are skipped. $0 (local CPU).
+#
+# Usage: DW_UNIFIED_DB=<uri> python3 embed_catalog.py [--limit N] [--batch 32] [--workers 24]
+import os, sys, io, time, struct, argparse, concurrent.futures as cf
+import urllib.request
+import numpy as np, psycopg2
+from PIL import Image
+import torch, open_clip
+
+DB = os.environ.get("DW_UNIFIED_DB") or sys.exit("set DW_UNIFIED_DB")
+ap = argparse.ArgumentParser()
+ap.add_argument("--limit", type=int, default=0) # 0 = all remaining
+ap.add_argument("--batch", type=int, default=32) # CLIP forward-pass batch
+ap.add_argument("--workers", type=int, default=24) # concurrent downloads
+ap.add_argument("--chunk", type=int, default=2000) # rows fetched per DB round
+args = ap.parse_args()
+
+print("loading CLIP ViT-B-32 (laion2b)…", flush=True)
+model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="laion2b_s34b_b79k")
+model.eval()
+torch.set_num_threads(max(1, os.cpu_count() - 2))
+DIM = model.visual.output_dim
+
+def fetch(row):
+ vc_id, dw_sku, mfr, vc, pat, url = row
+ try:
+ req = urllib.request.Request(url, headers={"User-Agent": "dw-visual/1.0"})
+ with urllib.request.urlopen(req, timeout=8) as r:
+ data = r.read(6_000_000)
+ img = Image.open(io.BytesIO(data)).convert("RGB")
+ return (vc_id, dw_sku, mfr, vc, pat, url, preprocess(img))
+ except Exception:
+ return (vc_id, dw_sku, mfr, vc, pat, url, None) # permanent-fail placeholder
+
+def main():
+ conn = psycopg2.connect(DB); conn.autocommit = False
+ total_done = 0; t0 = time.time()
+ while True:
+ cur = conn.cursor()
+ cur.execute("""select v.id, v.dw_sku, v.mfr_sku, v.vendor_code, v.pattern_name, v.image_url
+ from vendor_catalog v left join image_embeddings e on e.vc_id=v.id
+ where v.image_url is not null and v.image_url<>'' and e.vc_id is null
+ limit %s""", (args.chunk,))
+ rows = cur.fetchall(); cur.close()
+ if not rows: break
+ # download concurrently
+ got = []
+ with cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
+ got = list(ex.map(fetch, rows))
+ ok = [g for g in got if g[6] is not None]
+ bad = [g for g in got if g[6] is None]
+ # embed in batches
+ recs = []
+ for i in range(0, len(ok), args.batch):
+ b = ok[i:i+args.batch]
+ with torch.no_grad():
+ t = torch.stack([g[6] for g in b])
+ v = model.encode_image(t)
+ v = (v / v.norm(dim=-1, keepdim=True)).cpu().numpy().astype("float32")
+ for g, emb in zip(b, v):
+ recs.append((g[0], g[1], g[2], g[3], g[4], g[5], DIM, psycopg2.Binary(emb.tobytes())))
+ # write successes + permanent-fail placeholders (NULL embedding) so they aren't retried
+ cur = conn.cursor()
+ if recs:
+ cur.executemany("""insert into image_embeddings (vc_id,dw_sku,mfr_sku,vendor_code,pattern_name,image_url,dim,embedding)
+ values (%s,%s,%s,%s,%s,%s,%s,%s) on conflict (vc_id) do nothing""", recs)
+ if bad:
+ cur.executemany("""insert into image_embeddings (vc_id,image_url,embedding) values (%s,%s,NULL)
+ on conflict (vc_id) do nothing""", [(g[0], g[5]) for g in bad])
+ conn.commit(); cur.close()
+ total_done += len(rows)
+ rate = total_done / max(1e-6, time.time()-t0)
+ print(f" +{len(rows)} (ok={len(ok)} fail={len(bad)}) · {total_done} done · {rate:.1f}/s", flush=True)
+ if args.limit and total_done >= args.limit: break
+ conn.close()
+ print(f"done: {total_done} processed in {time.time()-t0:.0f}s", flush=True)
+
+if __name__ == "__main__":
+ main()
diff --git a/visual-search/search_service.py b/visual-search/search_service.py
new file mode 100644
index 0000000..9279439
--- /dev/null
+++ b/visual-search/search_service.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# Visual product search service. Holds the catalog CLIP embeddings in RAM and answers nearest-image
+# queries. dwphoto (Node) POSTs a photo → this returns the visually-closest catalog products.
+# No pgvector needed: brute-force cosine over the normalized matrix (numpy, ~ms for hundreds of k).
+#
+# DW_UNIFIED_DB=<uri> VS_PORT=9914 python3 search_service.py
+import os, io, json, base64, threading, time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import numpy as np, psycopg2
+from PIL import Image
+import torch, open_clip
+
+DB = os.environ["DW_UNIFIED_DB"]
+PORT = int(os.environ.get("VS_PORT", "9914"))
+
+print("loading CLIP…", flush=True)
+model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="laion2b_s34b_b79k")
+model.eval(); torch.set_num_threads(max(1, os.cpu_count() - 2))
+
+LOCK = threading.Lock()
+STATE = {"M": None, "meta": [], "loaded_at": 0, "n": 0}
+
+def load():
+ conn = psycopg2.connect(DB); cur = conn.cursor()
+ cur.execute("""select vc_id, dw_sku, mfr_sku, vendor_code, pattern_name, image_url, embedding
+ from image_embeddings where embedding is not null""")
+ vecs, meta = [], []
+ for vc_id, dw_sku, mfr, vc, pat, url, emb in cur:
+ vecs.append(np.frombuffer(bytes(emb), dtype="float32"))
+ meta.append({"vc_id": vc_id, "dw_sku": dw_sku, "mfr_sku": mfr, "vendor": vc, "pattern": pat, "image": url})
+ cur.close(); conn.close()
+ M = np.vstack(vecs).astype("float32") if vecs else np.zeros((0, 512), "float32")
+ with LOCK:
+ STATE["M"] = M; STATE["meta"] = meta; STATE["loaded_at"] = time.time(); STATE["n"] = len(meta)
+ print(f"loaded {len(meta)} embeddings", flush=True)
+
+def embed_photo(b64):
+ img = Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
+ with torch.no_grad():
+ v = model.encode_image(preprocess(img).unsqueeze(0))
+ v = (v / v.norm(dim=-1, keepdim=True)).cpu().numpy().astype("float32")[0]
+ return v
+
+class H(BaseHTTPRequestHandler):
+ def log_message(self, *a): pass
+ def _send(self, code, obj):
+ b = json.dumps(obj).encode(); self.send_response(code)
+ self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(b)))
+ self.end_headers(); self.wfile.write(b)
+ def do_GET(self):
+ if self.path.startswith("/health"): return self._send(200, {"ok": True, "n": STATE["n"], "loaded_at": STATE["loaded_at"]})
+ if self.path.startswith("/reload"):
+ try: load(); return self._send(200, {"ok": True, "n": STATE["n"]})
+ except Exception as e: return self._send(500, {"ok": False, "err": str(e)})
+ self._send(404, {"err": "not found"})
+ def do_POST(self):
+ if not self.path.startswith("/search"): return self._send(404, {"err": "not found"})
+ try:
+ n = int(self.headers.get("Content-Length", 0)); p = json.loads(self.rfile.read(n) or b"{}")
+ b64 = (p.get("image") or "").split(",")[-1]
+ k = int(p.get("k", 8))
+ if not b64: return self._send(400, {"err": "image required"})
+ q = embed_photo(b64)
+ with LOCK: M, meta = STATE["M"], STATE["meta"]
+ if M is None or not len(M): return self._send(200, {"ok": True, "results": [], "n": 0})
+ sims = M @ q # cosine (both L2-normalized)
+ idx = np.argpartition(-sims, min(k, len(sims)-1))[:k]
+ idx = idx[np.argsort(-sims[idx])]
+ res = [{**meta[i], "score": round(float(sims[i]), 4)} for i in idx]
+ self._send(200, {"ok": True, "n": STATE["n"], "results": res})
+ except Exception as e:
+ self._send(500, {"ok": False, "err": str(e)})
+
+if __name__ == "__main__":
+ load()
+ print(f"visual-search on :{PORT} · {STATE['n']} embeddings", flush=True)
+ ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()
← 873541c feat: visual identify — Gemini compares the ACTUAL user phot
·
back to Dw Photo Capture
·
feat: front+back identify UI + QR/vendor clues. #fbModal cap ac1d6d0 →