feat: max-info extract on scan + voice-only-as-needed. /api/extract: Gemini pulls EVERY label field (vendor, mfr, pattern, colorway, collection, width, length, repeat, content, price) + fuzzy-maps vendor. Add-item photo auto-prefills all fields. Then voice ONLY as needed: if a required field (mfr/vendor) is still blank, the app SPEAKS the prompt (‘Enter vendor name’) via TTS + listens, while the dropdown stays usable + the field is highlighted. Verified extract: WDW2310/Winfield/Compass Clay.

264f5cc38022ab5775277646ce5f25e3f4b85129 · 2026-07-06 19:31:56 -0700 · Steve Abrams
Files touched

M public/index.html
M server.js
Diff

commit 264f5cc38022ab5775277646ce5f25e3f4b85129
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 6 19:31:56 2026 -0700

    feat: max-info extract on scan + voice-only-as-needed. /api/extract: Gemini pulls EVERY label field (vendor, mfr, pattern, colorway, collection, width, length, repeat, content, price) + fuzzy-maps vendor. Add-item photo auto-prefills all fields. Then voice ONLY as needed: if a required field (mfr/vendor) is still blank, the app SPEAKS the prompt (‘Enter vendor name’) via TTS + listens, while the dropdown stays usable + the field is highlighted. Verified extract: WDW2310/Winfield/Compass Clay.
---
 public/index.html | 30 +++++++++++++++++++++++++++++-
 server.js         | 17 +++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/public/index.html b/public/index.html
index 7b9a116..af3eef3 100644
--- a/public/index.html
+++ b/public/index.html
@@ -158,6 +158,7 @@
   .add-field{margin-bottom:10px;flex:1}.add-field label{display:block;font-size:11px;color:#9a917d;margin-bottom:3px}
   .add-field input,.add-field select{width:100%;box-sizing:border-box;background:#0f0e0b;border:1px solid var(--line);border-radius:8px;padding:9px;color:var(--ink);font-size:15px}
   .add-row{display:flex;gap:10px}
+  .add-field input.need,.add-field select.need{border-color:#b8902f;box-shadow:0 0 0 2px rgba(184,144,47,.35)}
   /* onload chooser — three white pills on black */
   #homeScreen{position:fixed;inset:0;z-index:200;background:#000;display:flex;align-items:center;justify-content:center}
   #homeScreen[hidden]{display:none}
@@ -384,6 +385,7 @@
     <div class="samp-pat" style="margin-bottom:10px">Add new item</div>
     <button class="fb-tile" id="addPhotoTile" style="margin-bottom:12px;flex:none;width:100%"><div class="fb-thumb" id="addThumb" style="aspect-ratio:auto;height:64px;width:64px">📷</div><span>Photo (optional)</span></button>
     <input type="file" accept="image/*" capture="environment" id="addPhotoInput" hidden>
+    <button class="home-skip" id="addVoice" style="opacity:.85;color:var(--ink);margin:0 0 8px">🎤 Voice-fill what's missing</button>
     <div class="add-field"><label>Mfr # / SKU</label><input id="addMfr" autocapitalize="characters" placeholder="e.g. TR2581"></div>
     <div class="add-field"><label>Vendor</label><select id="addVendor"><option value="">Loading…</option></select></div>
     <div class="add-row"><div class="add-field"><label>VID</label><input id="addVid" placeholder="THI"></div><div class="add-field"><label>Color</label><input id="addColor"></div></div>
@@ -1435,8 +1437,34 @@ $('#addBtn').addEventListener('click',()=>openAddModal());
 $('#addClose').addEventListener('click',()=>{ $('#addModal').hidden=true; document.body.style.overflow=''; });
 $('#addVendor').addEventListener('change',e=>{ const o=e.target.selectedOptions[0]; if(o&&o.dataset.vid&&!$('#addVid').value) $('#addVid').value=o.dataset.vid; });
 $('#addPhotoTile').addEventListener('click',()=>$('#addPhotoInput').click());
-$('#addPhotoInput').addEventListener('change',async e=>{ const f=e.target.files[0]; if(f){ _addPhoto=await downscale(f,1400); $('#addThumb').style.backgroundImage=`url('${_addPhoto}')`; $('#addThumb').textContent=''; } });
+$('#addPhotoInput').addEventListener('change',async e=>{ const f=e.target.files[0]; if(!f)return; _addPhoto=await downscale(f,1400); $('#addThumb').style.backgroundImage=`url('${_addPhoto}')`; $('#addThumb').textContent=''; addExtract(); });
 $('#addPreview').addEventListener('click',addPreview);
+// import AS MUCH INFO AS POSSIBLE from the label photo → prefill every field, then VOICE ONLY as
+// needed: if a required field is still blank, the app speaks the prompt + listens (dropdown stays).
+async function addExtract(){ if(!_addPhoto)return; $('#addNote').textContent='📷 Reading label…';
+  try{ const r=await(await fetch('/api/extract',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({dataUrl:_addPhoto})})).json();
+    const f=r.fields||{}, setIf=(id,v)=>{ if(v&&!$('#'+id).value) $('#'+id).value=v; };
+    setIf('addMfr',f.mfr_sku); setIf('addName',f.pattern_name); setIf('addColor',f.colorway); setIf('addPrice',String(f.price||'').replace(/[^0-9.]/g,''));
+    if(r.vendor_matched){ await loadVendors(); const opt=[...$('#addVendor').options].find(o=>o.value.toLowerCase()===r.vendor_matched.toLowerCase()); if(opt){ $('#addVendor').value=opt.value; if(opt.dataset.vid&&!$('#addVid').value)$('#addVid').value=opt.dataset.vid; } }
+    $('#addNote').textContent='✓ Imported from label'; addPromptMissing();
+  }catch(e){ $('#addNote').textContent=''; }
+}
+// speak a prompt out loud, then listen for the answer (the on-screen field stays usable too)
+function speakThenListen(prompt, onResult){
+  const go=()=>{ const SR=window.SpeechRecognition||window.webkitSpeechRecognition; if(!SR)return; const r=new SR(); r.lang='en-US'; r.interimResults=false; r.onresult=e=>onResult(((e.results[0][0]||{}).transcript||'').trim()); r.onerror=()=>{}; try{ r.start(); }catch(e){} };
+  try{ const u=new SpeechSynthesisUtterance(prompt); u.rate=1.05; u.onend=go; speechSynthesis.cancel(); speechSynthesis.speak(u); setTimeout(()=>{ if(speechSynthesis.speaking)return; go(); }, 2500); }catch(e){ go(); }
+}
+// voice ONLY as needed — ask for the first missing REQUIRED field (mfr, then vendor)
+function addPromptMissing(){
+  if(!$('#addMfr').value.trim()){ $('#addMfr').classList.add('need'); $('#addNote').innerHTML='🔊 <b>Enter the SKU number</b> — say it or type it';
+    return speakThenListen('Enter the S K U number', t=>{ const c=(t.toUpperCase().match(/[A-Z]{1,4}[- ]?\d{2,6}/)||t.match(/\d{3,6}/)||[])[0]; if(c){ $('#addMfr').value=c.replace(/\s/g,''); $('#addMfr').classList.remove('need'); } $('#addNote').textContent='🎤 “'+t+'”'; addPromptMissing(); }); }
+  if(!$('#addVendor').value){ $('#addVendor').classList.add('need'); $('#addNote').innerHTML='🔊 <b>Enter vendor name</b> — say it or pick from the list';
+    return speakThenListen('Enter vendor name', async t=>{ await loadVendors(); const up=t.toUpperCase(); const opt=[...$('#addVendor').options].find(o=>o.value&&(o.value.toUpperCase().includes(up.split(' ')[0])||up.includes(o.value.toUpperCase()))); if(opt){ $('#addVendor').value=opt.value; if(opt.dataset.vid&&!$('#addVid').value)$('#addVid').value=opt.dataset.vid; $('#addVendor').classList.remove('need'); } $('#addNote').textContent='🎤 “'+t+'”'; }); }
+  $('#addNote').textContent='✓ Ready — Preview';
+}
+$('#addVoice').addEventListener('click',addPromptMissing);   // manual re-trigger of the voice fill
+$('#addMfr').addEventListener('input',()=>$('#addMfr').classList.remove('need'));
+$('#addVendor').addEventListener('change',()=>$('#addVendor').classList.remove('need'));
 
 // ── onload chooser: Add New SKU · Update SKU · Check Sample In ──
 function routeHome(act){ $('#homeScreen').hidden=true;
diff --git a/server.js b/server.js
index 1d3a6d4..8ebb719 100644
--- a/server.js
+++ b/server.js
@@ -1097,6 +1097,23 @@ const appHandler = (req, res) => {
     return;
   }
 
+  // Import AS MUCH INFO AS POSSIBLE from a label photo — a rich Gemini extraction of every field.
+  if (u.pathname === '/api/extract' && req.method === 'POST') {
+    let body = ''; req.on('data', c => { body += c; if (body.length > 15 * 1024 * 1024) req.destroy(); });
+    req.on('end', async () => {
+      let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
+      if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
+      const b64 = p.dataUrl.replace(/^data:image\/\w+;base64,/, '');
+      const prompt = 'Read this wallcovering/fabric SAMPLE label and extract EVERY field you can find. Reply ONLY as compact JSON with these keys (use "" for anything not present, preserve SKU/model codes EXACTLY, do not invent): {"vendor":"","mfr_sku":"","pattern_name":"","colorway":"","collection":"","width":"","length":"","repeat":"","content":"","match":"","price":"","other":""}.';
+      const r = await geminiVision(b64, prompt, 30000);
+      let a = {}; try { a = JSON.parse(r.response || '{}'); } catch (e) { a = {}; }
+      // map the read vendor to a known catalog vendor (fuzzy)
+      const vendorMatch = a.vendor ? (fuzzyVendor(a.vendor) || a.vendor) : null;
+      send(res, 200, { ok: !r.error, fields: a, vendor_matched: vendorMatch, cost_usd: 0.0006, err: r.error || null });
+    });
+    return;
+  }
+
   // Vendor + vid list for the "add new item" dropdown (distinct real vendors from the catalog).
   if (u.pathname === '/api/vendors' && req.method === 'GET') {
     getVendors().then(list => send(res, 200, { ok: true, vendors: list })).catch(e => send(res, 200, { ok: false, err: e.message, vendors: [] }));
← 496dc6c chore: embed-daemon.sh — resumable batch loop (20k chunks + · back to Dw Photo Capture · Capture full wallcovering spec set on scan (repeat/width/rol 3fcbec5 →