← back to Dw Photo Capture
Add 🎨 Recognize Pattern: VLM reads a swatch, finds this + visually-similar items across the whole unified DB
edd68a9bf24bfc227f5556c8ac301a10f7c75ee8 · 2026-06-25 23:51:26 -0700 · Steve Abrams
Files touched
M public/index.htmlM server.js
Diff
commit edd68a9bf24bfc227f5556c8ac301a10f7c75ee8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 23:51:26 2026 -0700
Add 🎨 Recognize Pattern: VLM reads a swatch, finds this + visually-similar items across the whole unified DB
---
public/index.html | 61 ++++++++++++++++++++++++++--
server.js | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 174 insertions(+), 6 deletions(-)
diff --git a/public/index.html b/public/index.html
index 9a6930c..795831d 100644
--- a/public/index.html
+++ b/public/index.html
@@ -360,14 +360,64 @@ async function doShopSearch(){
}catch(e){ toast('Shopify search failed'); }
}
// 🎨 Recognize a PATTERN (the swatch itself) → this + similar across the whole unified DB.
-let SIMILAR=[], RECOGNIZED=null;
+let SIMILAR=[], RECOGNIZED=null, QSIG=null, VRANK='', _rerankSeq=0;
+const _SIGCACHE=new Map();
+// Perceptual "thumbnail fingerprint": average the image down to an 8×8 RGB grid (192 nums,
+// 0-1). Two patterns that look alike — similar colors in similar places — land close together.
+function imgSig(src){
+ const N=8, c=document.createElement('canvas'); c.width=N; c.height=N;
+ const x=c.getContext('2d',{willReadFrequently:true});
+ try{ x.drawImage(src,0,0,N,N); }catch(e){ return null; }
+ let d; try{ d=x.getImageData(0,0,N,N).data; }catch(e){ return null; } // tainted canvas → null
+ const sig=new Float32Array(N*N*3);
+ for(let i=0;i<N*N;i++){ sig[i*3]=d[i*4]/255; sig[i*3+1]=d[i*4+1]/255; sig[i*3+2]=d[i*4+2]/255; }
+ return sig;
+}
+function sigDist(a,b){ if(!a||!b)return 1e9; let s=0; for(let i=0;i<a.length;i++){const e=a[i]-b[i]; s+=e*e;} return s; }
+function sigFromDataUrl(dataUrl){ return new Promise(res=>{ const im=new Image(); im.onload=()=>res(imgSig(im)); im.onerror=()=>res(null); im.src=dataUrl; }); }
+// Fingerprint a candidate's ACTUAL catalog image (tiny 72px, fetched same-origin through the
+// server proxy so Shopify's no-CORS image doesn't taint the canvas). Cached per URL.
+function candSig(url){
+ if(_SIGCACHE.has(url)) return Promise.resolve(_SIGCACHE.get(url));
+ return new Promise(res=>{
+ const small=url+(url.includes('?')?'&':'?')+'width=72';
+ const im=new Image(); im.crossOrigin='anonymous';
+ im.onload=()=>{ const s=imgSig(im); _SIGCACHE.set(url,s); res(s); };
+ im.onerror=()=>{ _SIGCACHE.set(url,null); res(null); };
+ im.src='/api/imgproxy?url='+encodeURIComponent(small);
+ });
+}
+const _vd=x=>(x._vdist==null?1e9:x._vdist);
+// Re-rank the tag-prefiltered candidates by VISUAL distance to the photographed swatch —
+// "sort by the actual pattern image." Runs in the background, repainting as results land.
+async function visualRerank(){
+ if(!QSIG || !SIMILAR.length){ VRANK=''; return; }
+ const seq=++_rerankSeq, items=SIMILAR.filter(x=>x.image);
+ VRANK='running'; if(filter==='similar') render();
+ let idx=0, done=0;
+ async function worker(){
+ while(idx<items.length){
+ const x=items[idx++];
+ const s=await candSig(x.image);
+ if(seq!==_rerankSeq) return; // a newer recognize superseded us
+ x._vdist=sigDist(QSIG,s); done++;
+ if(done%8===0 && filter==='similar') render(); // progressive repaint
+ }
+ }
+ await Promise.all(Array.from({length:6},()=>worker())); // 6-way concurrency
+ if(seq!==_rerankSeq) return;
+ VRANK='done'; if(filter==='similar') render();
+}
async function recognizeDataUrl(dataUrl){
toast('🎨 Recognizing pattern…'); $('#count').textContent='🎨 Reading pattern…';
+ QSIG=null; VRANK=''; _rerankSeq++;
+ sigFromDataUrl(dataUrl).then(s=>{ QSIG=s; if(SIMILAR.length) visualRerank(); }); // fingerprint the photo
try{
const r=await fetch('/api/recognize',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({dataUrl})});
const d=await r.json();
- RECOGNIZED=d.recognized||null; SIMILAR=d.items||[];
+ RECOGNIZED=d.recognized||null; SIMILAR=(d.items||[]).map(x=>({...x,_vdist:null}));
collapsed=false; chipTo('similar'); $('#q').value=''; render();
+ if(QSIG) visualRerank(); // sort by the actual pattern image
if(d.err && !SIMILAR.length){ toast('Recognize failed: '+d.err); }
else if(RECOGNIZED && RECOGNIZED.code){ toast('🏷 Code seen: '+RECOGNIZED.code+' · '+SIMILAR.length+' similar'); }
else { toast('🎨 '+SIMILAR.length+' similar pattern'+(SIMILAR.length===1?'':'s')+' found'); }
@@ -427,7 +477,7 @@ function render(){
});
list.sort((a,b)=> sort==='title'?(a.title||'').localeCompare(b.title||'')
: sort==='sku'?(a.dw_sku||'').localeCompare(b.dw_sku||'')
- : recog?((b.score||0)-(a.score||0))||(priced(b)-priced(a)) // keep similarity ranking
+ : recog?(_vd(a)-_vd(b))||((b.score||0)-(a.score||0)) // visual (actual-image) distance, then tag score
: (priced(b)-priced(a))||(a.title||'').localeCompare(b.title||''));
if(liveSearch){
const typed=($('#q').value||'').trim();
@@ -448,7 +498,10 @@ function render(){
} else if(recog){
$('#prog').style.width='0%';
$('#count').textContent = RECOGNIZED ? `${list.length} similar match${list.length===1?'':'es'}` : '🎨 Find Similar';
- $('#remain').textContent = RECOGNIZED ? 'ranked by pattern · whole catalog' : '📷 photo a swatch to start';
+ $('#remain').textContent = !RECOGNIZED ? '📷 photo a swatch to start'
+ : VRANK==='running' ? '🎨 ranking by image…'
+ : VRANK==='done' ? '✓ sorted by actual pattern image'
+ : 'ranked by pattern · whole catalog';
} else if(curated){
$('#prog').style.width='0%';
$('#count').textContent = filter==='fav' ? `${list.length} ⭐ favorite${list.length===1?'':'s'}` : `${list.length} recently updated`;
diff --git a/server.js b/server.js
index 74cb529..1da602a 100644
--- a/server.js
+++ b/server.js
@@ -383,13 +383,15 @@ rebuildLexicon();
// Reads a STYLIZED logo/wordmark + typesetting that the deterministic OCR can't, to name
// the vendor when no brand text was cleanly recognized. An on-lock assist, not per-frame.
const OLLAMA_VISION_MODEL = process.env.OLLAMA_VISION_MODEL || 'qwen2.5vl:7b'; // reuses OLLAMA_URL declared with ollamaAsk
-function ollamaVision(b64, prompt) {
+function ollamaVision(b64, prompt, timeoutMs) {
return new Promise(resolve => {
let target; try { target = new URL('/api/generate', OLLAMA_URL); } catch (e) { return resolve({ error: 'bad OLLAMA_URL' }); }
const lib = target.protocol === 'https:' ? https : http;
const payload = JSON.stringify({ model: OLLAMA_VISION_MODEL, prompt, images: [b64],
stream: false, format: 'json', options: { temperature: 0 } });
- const r = lib.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: 35000 },
+ // default 35s is fine once the model is warm; callers expecting a possible COLD model
+ // load (first call after idle ≈ 50s here) pass a larger budget so they don't false-timeout.
+ const r = lib.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: timeoutMs || 35000 },
res => { let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { resolve({ error: 'parse' }); } }); });
r.on('error', e => resolve({ error: e.message }));
r.on('timeout', () => { r.destroy(); resolve({ error: 'timeout' }); });
@@ -579,6 +581,33 @@ const appHandler = (req, res) => {
return;
}
+ // Recognize the actual PATTERN/material in a photo (the swatch itself, not the label)
+ // and surface similar items from the WHOLE unified mirror (~169k products). The local
+ // VLM reads the pattern's attributes; the local Postgres mirror — whose `tags` are
+ // AI-enriched with colors/style/motif/material — is ranked by attribute overlap. $0.
+ if (u.pathname === '/api/recognize' && req.method === 'POST') {
+ let body = ''; req.on('data', c => { body += c; if (body.length > 15 * 1024 * 1024) req.destroy(); });
+ req.on('end', async () => {
+ let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
+ if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
+ const b64 = p.dataUrl.replace(/^data:image\/\w+;base64,/, '');
+ const prompt = 'You are looking at a wallcovering or fabric SWATCH — the material itself, NOT a printed label. Describe the PATTERN so it can be matched against a catalog. Reply ONLY as compact JSON: {"description":"<one short sentence>","motif":"<main motif e.g. floral, damask, geometric, grasscloth, stripe, botanical, abstract, ikat, toile>","style":"<e.g. traditional, modern, transitional, scandinavian, art deco, contemporary>","material":"<e.g. grasscloth, non-woven, silk, vinyl, paper, leather>","colors":["<color name>","<color name>"],"background":"<background color name>","scale":"<small | medium | large>","code":"<any SKU or model number printed on it, else empty>"}';
+ const r = await ollamaVision(b64, prompt, 90000); // allow for a cold model load
+ let a = {}; try { a = JSON.parse(r.response || '{}'); } catch (e) { a = {}; }
+ const terms = similarTerms(a);
+ let items = []; try { items = await unifiedSimilar(terms); } catch (e) { items = []; }
+ const code = (a.code || '').toString().toUpperCase().replace(/\s+/g, '');
+ return send(res, 200, { ok: !r.error, model: OLLAMA_VISION_MODEL,
+ recognized: {
+ description: a.description || null, motif: a.motif || null, style: a.style || null,
+ material: a.material || null, colors: Array.isArray(a.colors) ? a.colors.slice(0, 6) : [],
+ background: a.background || null, scale: a.scale || null, code: code || null
+ },
+ terms, total: items.length, items, err: r.error || null });
+ });
+ return;
+ }
+
// /selfcheck — re-runnable health report. Verifies data integrity + AUTO-CLEANS stale
// photo refs (progress entries pointing at a deleted /photos file = the dead-link class).
if (u.pathname === '/selfcheck') {
@@ -783,6 +812,23 @@ const appHandler = (req, res) => {
return;
}
+ // Generic same-origin image proxy — lets the browser read pattern pixels off a
+ // catalog image (Shopify CDN sends no CORS header → a direct <img> taints the
+ // canvas). Host-whitelisted so it can't be turned into an open relay. Used by the
+ // 🎨 Similar visual re-rank to fingerprint each candidate's actual pattern image.
+ if (u.pathname === '/api/imgproxy' && req.method === 'GET') {
+ const src = u.searchParams.get('url') || '';
+ let su; try { su = new URL(src); } catch (e) { return send(res, 400, { err: 'bad url' }); }
+ const OK = /(^|\.)(shopify\.com|shopifycdn\.com|myshopify\.com|cdn\.shopify\.com)$/i;
+ if (su.protocol !== 'https:' || !OK.test(su.hostname)) return send(res, 403, { err: 'host not allowed' });
+ https.get(su.href, ir => {
+ if ((ir.statusCode || 0) >= 400) { ir.resume(); return send(res, 502, { err: 'upstream ' + ir.statusCode }); }
+ res.writeHead(200, { 'Content-Type': ir.headers['content-type'] || 'image/jpeg', 'Cache-Control': 'max-age=86400', 'Access-Control-Allow-Origin': '*' });
+ ir.pipe(res);
+ }).on('error', e => send(res, 502, { err: e.message }));
+ return;
+ }
+
// OCR a photo of a printed mfr#/SKU (macOS Vision, local/free) → ranked SKU candidates.
if (u.pathname === '/api/ocr' && req.method === 'POST') {
let body = '';
@@ -1158,6 +1204,75 @@ async function shopifySearch(q) {
return Array.from(byId.values()).slice(0, 80);
}
+// ── Local dw_unified mirror: pattern-similarity search ───────────────────────
+// This app carries zero runtime deps (see package.json), so — exactly like the
+// Swift OCR binary — we reach Postgres by shelling to `psql` rather than pulling
+// in a driver. The local mirror IS the whole store (~169k products) and, crucially,
+// carries the AI-enriched `tags` (colors / style / motif / material) that turn a
+// VLM's pattern read into real "similar items" instead of a brittle title match.
+const PSQL = [process.env.PSQL, '/opt/homebrew/opt/postgresql@14/bin/psql',
+ '/opt/homebrew/bin/psql', '/usr/local/bin/psql', '/usr/bin/psql']
+ .find(p => p && fs.existsSync(p)) || 'psql';
+const DW_DB = process.env.DW_UNIFIED_DB || 'dw_unified';
+// words too generic to discriminate (every wallcovering tag-set carries them)
+const SIM_STOP = new Set(['wallcovering', 'wallcoverings', 'wallpaper', 'fabric', 'fabrics',
+ 'multi', 'color', 'colour', 'interior', 'designer', 'showroom', 'line', 'non', 'woven',
+ 'nonwoven', 'needs', 'image', 'price', 'width', 'pattern', 'wall', 'background']);
+
+// VLM attribute object → a clean, deduped, injection-safe term list for tag matching.
+// Every term is reduced to [a-z0-9 &-] (quotes/semicolons/backslashes stripped), so the
+// terms can be embedded straight into the ILIKE patterns below with no escape risk.
+function similarTerms(attrs) {
+ const raw = [];
+ const push = v => { if (v) String(v).split(/[,/;&]| and /i).forEach(t => raw.push(t)); };
+ push(attrs.motif); push(attrs.style); push(attrs.material); push(attrs.background);
+ (attrs.colors || []).forEach(push); push(attrs.pattern); push(attrs.type);
+ const seen = new Set(), out = [];
+ for (let t of raw) {
+ t = String(t).toLowerCase().replace(/[^a-z0-9 &-]/g, ' ').replace(/\s+/g, ' ').trim();
+ if (t.length < 3 || SIM_STOP.has(t) || seen.has(t)) continue;
+ seen.add(t); out.push(t);
+ if (out.length >= 8) break;
+ }
+ return out;
+}
+
+// Rank ACTIVE products by how many recognized attribute-terms appear in their enriched
+// tags / title / pattern-name, newest-priced first. Returns standard card-shaped items
+// (keep_images:true so updating a photo on a match ADDS as featured, never wipes).
+function unifiedSimilar(terms) {
+ return new Promise(resolve => {
+ const t = terms.filter(Boolean).slice(0, 8);
+ if (!t.length) return resolve([]);
+ const hay = "lower(coalesce(tags,'')||' '||coalesce(title,'')||' '||coalesce(pattern_name,''))";
+ const score = t.map(x => `(case when ${hay} like '%${x}%' then 1 else 0 end)`).join('+');
+ const anyOf = t.map(x => `${hay} like '%${x}%'`).join(' or ');
+ const SQL = `select regexp_replace(coalesce(shopify_id,''),'\\D','','g') as product_id,
+ coalesce(dw_sku,''), coalesce(mfr_sku,''), coalesce(title,''), coalesce(vendor,''),
+ coalesce(case when price>4.25 then price end, retail_price, min_variant_price, price),
+ coalesce(image_url,''), upper(coalesce(status,'')), (${score}) as score
+ from shopify_products
+ where status ilike 'active' and image_url is not null and image_url <> ''
+ and (${anyOf})
+ order by score desc, (coalesce(price,0)>4.25) desc, updated_at_shopify desc nulls last
+ limit 48`;
+ execFile(PSQL, ['-d', DW_DB, '-tAF', '\t', '-c', SQL],
+ { timeout: 8000, maxBuffer: 8 * 1024 * 1024 }, (err, stdout) => {
+ if (err) return resolve([]);
+ const items = (stdout || '').split('\n').filter(Boolean).map(line => {
+ const [product_id, dw_sku, mfr, title, vendor, price, image, status, sc] = line.split('\t');
+ return {
+ product_id: product_id || null, dw_sku: dw_sku || '', mfr: mfr || '',
+ title: title || '', vendor: vendor || '',
+ price: price ? parseFloat(price) : null, image: image || null,
+ status: status || '', score: parseInt(sc, 10) || 0, keep_images: true, scope: 'unified'
+ };
+ });
+ resolve(items);
+ });
+ });
+}
+
server.listen(PORT, '0.0.0.0', () => {
console.log(`DW Photo Capture on http://0.0.0.0:${PORT} (Shopify push: ${TOKEN ? 'ON' : 'OFF — no token'}, sheet GRS: ${SHEET.length})`);
rebuildIndex(); // sheet-only items searchable immediately
← 39a065c learn: surface logo fingerprint + flag rejected logos (overn
·
back to Dw Photo Capture
·
identify: logo-fingerprint tiebreaker (DTD 3/3 verdict A) 2a4ae14 →