← back to Dw Photo Capture
Ask-the-SKU: voice Q&A about a product via local Ollama ($0)
47a5ecf0383edba901fe6dec5e39f44f381955b2 · 2026-06-25 20:35:47 -0700 · Steve
- server /api/ask: pulls a product's facts (basics + spec-like metafields, prefers
the main non-sample variant) and asks local Ollama (qwen2.5:latest, 127.0.0.1)
- gallery '🎤 Ask' button: speak a question about the open SKU -> answer shown in a
banner + spoken back via speechSynthesis; prompt() fallback where no SpeechRecognition
- OLLAMA_URL/OLLAMA_MODEL env-overridable; 30s timeout, graceful 'AI unreachable'
Files touched
M data/build.jsonM public/index.htmlM server.js
Diff
commit 47a5ecf0383edba901fe6dec5e39f44f381955b2
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jun 25 20:35:47 2026 -0700
Ask-the-SKU: voice Q&A about a product via local Ollama ($0)
- server /api/ask: pulls a product's facts (basics + spec-like metafields, prefers
the main non-sample variant) and asks local Ollama (qwen2.5:latest, 127.0.0.1)
- gallery '🎤 Ask' button: speak a question about the open SKU -> answer shown in a
banner + spoken back via speechSynthesis; prompt() fallback where no SpeechRecognition
- OLLAMA_URL/OLLAMA_MODEL env-overridable; 30s timeout, graceful 'AI unreachable'
---
data/build.json | 5 +-
public/index.html | 33 ++++++++++
server.js | 183 +++++++++++++++++++++++++++++++++++++-----------------
3 files changed, 163 insertions(+), 58 deletions(-)
diff --git a/data/build.json b/data/build.json
index 30f1227..2a81bb3 100644
--- a/data/build.json
+++ b/data/build.json
@@ -1,5 +1,5 @@
{
- "next": 45,
+ "next": 46,
"map": {
"63863152": 27,
"578af86f": 2,
@@ -43,6 +43,7 @@
"7d42b31f": 41,
"260c91c6": 42,
"3b47a090": 43,
- "798e88ce": 44
+ "798e88ce": 44,
+ "77dbf02a": 45
}
}
\ No newline at end of file
diff --git a/public/index.html b/public/index.html
index 69550b8..d69d874 100644
--- a/public/index.html
+++ b/public/index.html
@@ -90,6 +90,8 @@
.gallery{position:fixed;inset:0;z-index:90;background:rgba(10,9,7,.98);display:flex;flex-direction:column}
.gallery[hidden]{display:none}
.gal-head{position:sticky;top:0;display:flex;align-items:center;gap:10px;padding:max(12px,env(safe-area-inset-top)) 14px 12px;border-bottom:1px solid var(--line);background:rgba(10,9,7,.98)}
+ #galAnswer{margin:10px 14px;padding:12px 14px;border-radius:12px;background:rgba(212,175,107,.12);border:1px solid var(--line);color:#f3efe7;font-size:15px;line-height:1.45}
+ #galAnswer .qq{color:var(--muted);font-size:13px;display:block;margin-bottom:5px}
.gal-x{background:#16140f;border:1px solid var(--line);color:var(--ink);border-radius:10px;padding:9px 13px;font-size:14px;cursor:pointer}
.gal-title{flex:1;min-width:0;line-height:1.2} .gal-title b{display:block;font:600 15px/1.2 "SF Pro Display",sans-serif} .gal-title span{font:600 12px/1 ui-monospace,Menlo,monospace;color:var(--gold)}
.gal-add{position:relative;overflow:hidden;background:var(--gold);color:#1b1407;border-radius:10px;padding:9px 14px;font-weight:700;font-size:14px;cursor:pointer}
@@ -206,7 +208,9 @@
<label class="gal-add gal-add-multi">➕ Multiple<input type="file" accept="image/*" multiple id="galAddMulti"></label>
<label class="gal-add">📷 Add<input type="file" accept="image/*" capture="environment" id="galAdd"></label>
<label class="gal-add gal-add-multi" title="Upload a product video">🎥 Video<input type="file" accept="video/*" capture="environment" id="galVid"></label>
+ <button class="gal-add" id="galAsk" title="Ask a question about this SKU (local AI)">🎤 Ask</button>
</div>
+ <div id="galAnswer" hidden></div>
<div class="gal-grid" id="galGrid"></div>
</div>
@@ -463,6 +467,7 @@ async function openGallery(x){
GAL_X=x;
$('#galName').textContent=(x.title||'').replace(/ \| Fentucci$/,'');
$('#galSku').textContent=x.dw_sku||'';
+ $('#galAnswer').hidden=true; $('#galAnswer').innerHTML=''; // clear any prior Q&A
$('#gallery').hidden=false; document.body.style.overflow='hidden';
await renderGallery();
}
@@ -491,6 +496,7 @@ async function imgAction(act,image_id){
const d=await r.json(); if(d.ok){ toast(act==='delete'?'Removed':'Featured ✓'); renderGallery(); } else toast('Failed'); }catch(e){ toast('Failed'); }
}
$('#galClose').addEventListener('click',closeGallery);
+$('#galAsk').addEventListener('click',voiceAsk);
// Batch add: take several photos (in your camera roll), pick them all here, save at once.
$('#galAddMulti').addEventListener('change',async e=>{
const files=[...e.target.files]; e.target.value=''; if(!files.length||!GAL_X)return;
@@ -894,6 +900,33 @@ function runVoiceQuery(t){
// and (via the mfr# index) printed vendor codes. Exactly what "look for a vendor" needs.
chipTo('shop'); doShopSearch();
}
+
+// ── Ask-the-SKU: speak a question about the OPEN product → local Ollama answers (spoken back) ──
+function speak(text){ try{ const u=new SpeechSynthesisUtterance(text); u.rate=1.05; speechSynthesis.cancel(); speechSynthesis.speak(u); }catch(e){} }
+function voiceAsk(){
+ if(!GAL_X||!GAL_X.product_id){ toast('Open a product first'); return; }
+ const SR=window.SpeechRecognition||window.webkitSpeechRecognition;
+ if(!SR){ const q=prompt('Ask about '+(GAL_X.dw_sku||'this SKU')+':'); if(q) askSku(q); return; }
+ if(_listening){ try{ _recog&&_recog.stop(); }catch(e){} return; }
+ unlockAudio();
+ _recog=new SR(); _recog.lang='en-US'; _recog.interimResults=false; _recog.maxAlternatives=1;
+ _listening=true; const b=$('#galAsk'); if(b) b.textContent='🔴 …';
+ toast('🎤 Ask about this SKU…');
+ _recog.onresult=e=>{ const t=((e.results[0][0]||{}).transcript||'').trim(); if(t) askSku(t); };
+ _recog.onerror=e=>{ toast('Mic: '+(e.error||'error')); };
+ _recog.onend=()=>{ _listening=false; const b=$('#galAsk'); if(b) b.textContent='🎤 Ask'; };
+ try{ _recog.start(); }catch(e){ _listening=false; toast('Mic busy — try again'); }
+}
+async function askSku(question){
+ if(!GAL_X||!GAL_X.product_id) return;
+ const a=$('#galAnswer'); a.hidden=false; a.innerHTML='<span class="qq">'+question+'</span>💭 thinking…';
+ try{
+ const r=await fetch('/api/ask',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({product_id:GAL_X.product_id,question})});
+ const d=await r.json(); const ans=d.answer||'No answer.';
+ a.innerHTML='<span class="qq">'+question+'</span>'+ans;
+ speak(ans);
+ }catch(e){ a.innerHTML='<span class="qq">'+question+'</span>⚠️ AI unreachable on the LAN.'; }
+}
async function openLiveScan(){
unlockAudio(); // we're inside the user's tap → unlock audio for the lock-in cue
try{ await startScanStream(); }
diff --git a/server.js b/server.js
index c435755..e2ea11c 100644
--- a/server.js
+++ b/server.js
@@ -261,6 +261,68 @@ function send(res, code, body, headers = {}) {
res.end(typeof body === 'string' ? body : JSON.stringify(body));
}
+// ── OCR → ranked SKU candidates ────────────────────────────────────────────────
+// Allocated ONCE at load (the live scanner hits /api/ocr ~every 600ms, so we never
+// want to rebuild these regexes per request). analyzeOcr() is pure — it takes parsed
+// rows and returns the response shape, so the ranking is unit-testable without HTTP.
+//
+// Brand lexicon: when one of these vendor names is printed on the swatch, the biggest
+// alnum code on the page is almost certainly its model #/SKU — so we boost a code whose
+// prefix matches that brand and let the scanner lock on the FIRST read (topStrong).
+const VENDOR_LEX = [
+ { name: 'Winfield Thybony', re: /WINFIELD\s*THYBONY|THYBONY/, pfx: /^WD/ },
+ { name: 'Fentucci', re: /FENTUCCI/, pfx: /^(GRS|WOS|SG|BA)/ },
+ { name: 'Phillip Jeffries', re: /PHILLIP\s*JEFFRIES/, pfx: /^(WC|PJ)/ },
+ { name: 'Thibaut', re: /THIBAUT|ANNA\s*FRENCH/, pfx: /^(T|AT|AF)\d/ },
+ { name: 'Schumacher', re: /SCHUMACHER/, pfx: /^\d{4,}/ },
+];
+const STRONG_PFX = /^(GRS-|WD[A-Z]*\d|DWNAT|DW[A-Z]{2})/; // self-evidently a SKU, no brand needed
+const CODE_TOKEN = /[A-Z0-9][A-Z0-9-]{2,13}/g; // candidate token shape
+
+// Parse the OCR binary's "<heightThousandths>\t<text>" lines. Height = printed font
+// size on the label (taller box = bigger print), which drives "largest letters first".
+function parseOcrRows(stdout) {
+ return (stdout || '').trim().split('\n').filter(Boolean).map(l => {
+ const i = l.indexOf('\t');
+ return i < 0 ? { h: 0, t: l } : { h: parseInt(l.slice(0, i), 10) || 0, t: l.slice(i + 1) };
+ });
+}
+
+// Pure: OCR rows → { text, vendor, candidates, top, topStrong }. No I/O.
+function analyzeOcr(rows) {
+ const text = rows.map(r => r.t).join('\n');
+ const vlex = VENDOR_LEX.find(v => v.re.test(text.toUpperCase())) || null;
+ const vendor = vlex ? vlex.name : null;
+ // SKU-likeness tie-breaker: brand-matching prefix first (the swatch told us the vendor),
+ // then known prefixes (GRS- = Fentucci, WD = Winfield Thybony), dashed/alnum, length.
+ const skuScore = t =>
+ ((vlex && vlex.pfx.test(t)) ? 140 : 0) + // matches the brand printed on the swatch
+ (t.startsWith('GRS-') ? 100 : 0) +
+ (/^WD[A-Z]*\d/.test(t) ? 60 : 0) + // Winfield Thybony WDW2310, WD-prefixed
+ (t.includes('-') ? 12 : 0) +
+ (/[A-Z]/.test(t) && /\d/.test(t) ? 20 : 0) +
+ Math.min(t.length, 12);
+ // CODE NUMBERS ONLY — alnum tokens holding a digit (WDW2310, GRS-26220, 5255-16). Plain
+ // names and marketing prose are intentionally ignored: scan the code, never the text.
+ const seen = new Set(); const codes = [];
+ for (const r of rows) {
+ for (const t of (r.t.toUpperCase().match(CODE_TOKEN) || [])) {
+ if (!/\d/.test(t) || t.length < 4 || seen.has(t)) continue; // must hold a digit, >=4 chars
+ seen.add(t); codes.push({ t, h: r.h, s: skuScore(t) });
+ }
+ }
+ // LARGEST CODE FIRST: font height, then SKU-likeness (vendor prefix), then length.
+ codes.sort((a, b) => (b.h - a.h) || (b.s - a.s) || (b.t.length - a.t.length));
+ const top = codes[0] || null;
+ const hMax = codes.reduce((m, c) => Math.max(m, c.h), 0);
+ // STRONG = safe to LOCK on the first sighting: a known SKU prefix, a brand-prefix match,
+ // or a brand name on the swatch + this being the font-dominant code (biggest = the SKU).
+ const topStrong = !!top && (
+ STRONG_PFX.test(top.t) || (vlex && vlex.pfx.test(top.t)) || (!!vendor && top.h >= hMax)
+ );
+ return { text, vendor, candidates: codes.map(c => c.t), top: top ? top.t : null, topStrong };
+}
+
const appHandler = (req, res) => {
// public (no-auth) paths: health + the home-screen-install assets iOS fetches without creds
const PUBLIC = ['/healthz', '/icon-180.png', '/icon-512.png', '/apple-touch-icon.png', '/manifest.webmanifest'];
@@ -474,6 +536,25 @@ const appHandler = (req, res) => {
return;
}
+ // Ask a question about a SKU — local Ollama answers from the product's own Shopify facts. $0.
+ if (u.pathname === '/api/ask' && req.method === 'POST') {
+ let body = '';
+ req.on('data', c => { body += c; if (body.length > 1e5) req.destroy(); });
+ req.on('end', async () => {
+ let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
+ const question = (p.question || '').trim();
+ if (!question) return send(res, 400, { err: 'question required' });
+ try {
+ const ctx = await productContext(p.product_id);
+ if (!ctx) return send(res, 200, { answer: "I can't find that product's details." });
+ const prompt = `You are a concise assistant for a wallcovering showroom. Answer the question using ONLY the product facts below. If a fact isn't listed, say you don't have that detail — don't guess. One or two short sentences.\n\nPRODUCT FACTS:\n${ctx}\n\nQUESTION: ${question}\nANSWER:`;
+ const answer = await ollamaAsk(prompt);
+ send(res, 200, { answer: (answer || '').trim() || 'No answer.', model: OLLAMA_MODEL });
+ } catch (e) { send(res, 200, { answer: 'Sorry — the local AI is unreachable right now.', err: e.message }); }
+ });
+ return;
+ }
+
if (u.pathname.startsWith('/photos/')) {
const f = path.join(PHOTOS, path.basename(u.pathname));
if (fs.existsSync(f)) { res.writeHead(200, { 'Content-Type': 'image/jpeg' }); return res.end(fs.readFileSync(f)); }
@@ -552,62 +633,8 @@ const appHandler = (req, res) => {
catch (e) { return send(res, 500, { err: 'write fail' }); }
execFile(path.join(ROOT, 'bin/ocr'), [tmp], { timeout: 8000 }, (err, stdout) => {
try { fs.unlinkSync(tmp); } catch (e) {}
- // Each OCR line is "<heightThousandths>\t<text>" — bigger font on the label = taller box.
- // We rank candidates by font height FIRST ("largest letters first"), so the SKU/model
- // printed biggest on a sample (e.g. Winfield Thybony WDW2310) is tried before fine print.
- const rows = (stdout || '').trim().split('\n').filter(Boolean).map(l => {
- const i = l.indexOf('\t');
- return i < 0 ? { h: 0, t: l } : { h: parseInt(l.slice(0, i), 10) || 0, t: l.slice(i + 1) };
- });
- const text = rows.map(r => r.t).join('\n');
- const U = text.toUpperCase();
- // BRAND ON THE SWATCH → confidence signal. When a known vendor name is printed on
- // the sample (e.g. "Winfield Thybony", "Fentucci"), the biggest alnum code on the
- // page is almost certainly its model #/SKU. We (a) boost a code whose prefix matches
- // that vendor, and (b) tell the scanner it can lock on the FIRST read (topStrong).
- const VENDOR_LEX = [
- { name: 'Winfield Thybony', re: /WINFIELD\s*THYBONY|THYBONY/, pfx: /^WD/ },
- { name: 'Fentucci', re: /FENTUCCI/, pfx: /^(GRS|WOS|SG|BA)/ },
- { name: 'Phillip Jeffries', re: /PHILLIP\s*JEFFRIES/, pfx: /^(WC|PJ)/ },
- { name: 'Thibaut', re: /THIBAUT|ANNA\s*FRENCH/, pfx: /^(T|AT|AF)\d/ },
- { name: 'Schumacher', re: /SCHUMACHER/, pfx: /^\d{4,}/ },
- ];
- const vlex = VENDOR_LEX.find(v => v.re.test(U)) || null;
- const vendor = vlex ? vlex.name : null;
- // SKU-likeness tie-breaker: the brand-matching prefix first (the swatch told us the
- // vendor), then known prefixes (GRS- = Fentucci, WD = Winfield Thybony), then alnum
- // codes (letters+digits), dashed codes, longer tokens.
- const skuScore = t =>
- ((vlex && vlex.pfx.test(t)) ? 140 : 0) + // matches the brand printed on the swatch
- (t.startsWith('GRS-') ? 100 : 0) +
- (/^WD[A-Z]*\d/.test(t) ? 60 : 0) + // Winfield Thybony WDW2310, WD-prefixed
- (t.includes('-') ? 12 : 0) +
- (/[A-Z]/.test(t) && /\d/.test(t) ? 20 : 0) +
- Math.min(t.length, 12);
- // CODE NUMBERS ONLY — alphanumeric tokens that contain a digit (SKU / model / lot
- // codes like WDW2310, GRS-26220, 5255-16). Plain names and any other prose on the
- // label are intentionally ignored: scan the code, never the marketing text.
- const seen = new Set(); const codes = [];
- for (const r of rows) {
- for (const t of (r.t.toUpperCase().match(/[A-Z0-9][A-Z0-9-]{2,13}/g) || [])) {
- if (!/\d/.test(t) || t.length < 4 || seen.has(t)) continue; // must hold a digit, ≥4 chars
- seen.add(t); codes.push({ t, h: r.h, s: skuScore(t) });
- }
- }
- // LARGEST CODE FIRST: sort by font height, then SKU-likeness (vendor prefix), then length.
- codes.sort((a, b) => (b.h - a.h) || (b.s - a.s) || (b.t.length - a.t.length));
- const cand = codes.map(x => x.t);
- // STRONG read = safe to LOCK on the first sighting (no need to wait for a 2nd matching
- // frame). True when the top code carries a known SKU prefix, OR a vendor brand name was
- // on the swatch AND the top code is the font-dominant one (biggest letters = the SKU).
- const top = codes[0] || null;
- const hMax = codes.reduce((m, c) => Math.max(m, c.h), 0);
- const topStrong = !!top && (
- /^(GRS-|WD[A-Z]*\d|DWNAT|DW[A-Z]{2})/.test(top.t) ||
- (vlex && vlex.pfx.test(top.t)) ||
- (!!vendor && top.h >= hMax)
- );
- send(res, 200, { ok: true, text, vendor, candidates: cand, top: top ? top.t : null, topStrong });
+ // OCR output → ranked SKU candidates (parsing + ranking extracted to module scope).
+ send(res, 200, Object.assign({ ok: true }, analyzeOcr(parseOcrRows(stdout))));
});
});
return;
@@ -894,6 +921,50 @@ const _PNODE = `legacyResourceId title status vendor
mf:metafield(namespace:"custom",key:"manufacturer_sku"){value}
mf2:metafield(namespace:"dwc",key:"manufacturer_sku"){value}
v:variants(first:5){edges{node{sku title price}}}`;
+// ── Local Ollama Q&A (free, on-LAN) ──────────────────────────────────────────
+const OLLAMA_URL = process.env.OLLAMA_URL || 'http://127.0.0.1:11434';
+const OLLAMA_MODEL = process.env.OLLAMA_MODEL || 'qwen2.5:latest';
+function ollamaAsk(prompt) {
+ return new Promise((resolve, reject) => {
+ const ou = new URL(OLLAMA_URL + '/api/generate');
+ const data = JSON.stringify({ model: OLLAMA_MODEL, prompt, stream: false, options: { num_predict: 160, temperature: 0.2 } });
+ const lib = ou.protocol === 'https:' ? https : http;
+ const r = lib.request({ hostname: ou.hostname, port: ou.port, path: ou.pathname, method: 'POST',
+ headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) } }, resp => {
+ let d = ''; resp.on('data', c => d += c); resp.on('end', () => { try { resolve((JSON.parse(d).response) || ''); } catch (e) { reject(e); } });
+ });
+ r.setTimeout(30000, () => r.destroy(new Error('ollama timeout')));
+ r.on('error', reject); r.write(data); r.end();
+ });
+}
+// Pull a product's facts (basics + spec-like metafields) → a compact context block for the LLM.
+async function productContext(productId) {
+ if (!productId) return null;
+ const gid = `gid://shopify/Product/${String(productId).replace(/\D/g, '')}`;
+ const Q = `query($id:ID!){ product(id:$id){ title vendor productType tags
+ variants(first:3){edges{node{sku price}}}
+ metafields(first:50){edges{node{key value}}} } }`;
+ const r = await gql(Q, { id: gid });
+ const node = r.data && r.data.product;
+ if (!node) return null;
+ const vs = node.variants.edges.map(e => e.node);
+ const v = vs.find(x => x.sku && !/-sample$/i.test(x.sku)) || vs[0] || {}; // prefer the main (non-sample) variant
+ const facts = [`Title: ${node.title}`];
+ if (node.vendor) facts.push(`Vendor: ${node.vendor}`);
+ if (node.productType) facts.push(`Type: ${node.productType}`);
+ if (v.price) facts.push(`Price: $${v.price}`);
+ if (v.sku) facts.push(`SKU: ${v.sku}`);
+ const WANT = /(width|content|repeat|fire|flam|weight|care|material|origin|backing|finish|colou?r|usage|durab|clean|length|yard|coverage|railroad|match)/i;
+ const seen = new Set();
+ for (const e of (node.metafields.edges || [])) {
+ const k = e.node.key, val = (e.node.value || '').toString();
+ if (!val || val.length > 120 || /^https?:/.test(val) || /^[[{]/.test(val)) continue; // skip urls / json blobs
+ if (WANT.test(k) && !seen.has(k)) { seen.add(k); facts.push(`${k.replace(/_/g, ' ')}: ${val}`); }
+ }
+ if (node.tags && node.tags.length) facts.push(`Tags: ${node.tags.slice(0, 12).join(', ')}`);
+ return facts.join('\n');
+}
+
async function shopifySearch(q) {
// If the query is a single code-like token (has a digit), try the manufacturer-SKU
// index FIRST — a scanned vendor code (WDW2310) resolves to its DW house SKU (CHC-…)
← fc5196a auto-save: 2026-06-25T18:36:37 (1 files) — data/recents.json
·
back to Dw Photo Capture
·
Learn how each vendor labels their samples (180 vendor profi 5ebe4e6 →