← back to Dw Photo Capture
feat: pluggable vision engines (macvision|gemini|gcv), runtime-selectable + engine=all — enables Linux/Kamatera hosting
75c75e3d0b9366dc97a7407ed696ba0c2911c7f7 · 2026-07-06 14:43:18 -0700 · Steve Abrams
- OCR + logo-identify + pattern-recognize now route through resolveEngines(): macvision
(Apple Vision bin/ocr + local Ollama), gemini (2.5-flash, OCR+identify), gcv (Cloud Vision OCR).
- /api/ocr,/api/identify,/api/recognize take {engine:...}; engine=all fans out to every
available engine and returns by_engine + per-engine cost_usd. Default macvision on macOS, gemini elsewhere.
- Backward-compatible: single-engine responses keep the same top-level shape; 74/74 tests green.
- Verified end-to-end: both engines extract WDW2310->Winfield Thybony from a label.
Files touched
Diff
commit 75c75e3d0b9366dc97a7407ed696ba0c2911c7f7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 6 14:43:18 2026 -0700
feat: pluggable vision engines (macvision|gemini|gcv), runtime-selectable + engine=all — enables Linux/Kamatera hosting
- OCR + logo-identify + pattern-recognize now route through resolveEngines(): macvision
(Apple Vision bin/ocr + local Ollama), gemini (2.5-flash, OCR+identify), gcv (Cloud Vision OCR).
- /api/ocr,/api/identify,/api/recognize take {engine:...}; engine=all fans out to every
available engine and returns by_engine + per-engine cost_usd. Default macvision on macOS, gemini elsewhere.
- Backward-compatible: single-engine responses keep the same top-level shape; 74/74 tests green.
- Verified end-to-end: both engines extract WDW2310->Winfield Thybony from a label.
---
server.js | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 151 insertions(+), 16 deletions(-)
diff --git a/server.js b/server.js
index 42cb92a..8c0d8e2 100644
--- a/server.js
+++ b/server.js
@@ -415,6 +415,131 @@ function ollamaVision(b64, prompt, timeoutMs, model) {
});
}
+// ── Pluggable vision engines: macvision (Apple Vision + local Ollama) · gemini · gcv ──
+// Steve 2026-07-06: so the scanner runs on Kamatera (Linux), where the macOS `bin/ocr`
+// Vision binary and a local Ollama don't exist. Each engine is selectable per request
+// ({engine:"gemini"} or ?engine=gcv); engine:"all" fans out to every AVAILABLE engine and
+// returns each result for comparison. Default = macvision on macOS, gemini elsewhere.
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY || '';
+const GEMINI_VISION_MODEL = process.env.GEMINI_VISION_MODEL || 'gemini-2.5-flash';
+const GCV_API_KEY = process.env.GCV_API_KEY || process.env.GOOGLE_VISION_API_KEY || '';
+const IS_DARWIN = process.platform === 'darwin';
+const HAS_MACVISION = IS_DARWIN && fs.existsSync(path.join(ROOT, 'bin/ocr'));
+const DEFAULT_ENGINE = (process.env.OCR_ENGINE || (HAS_MACVISION ? 'macvision' : 'gemini')).toLowerCase();
+const ALL_ENGINES = ['macvision', 'gemini', 'gcv'];
+// Rough per-scan cost so the caller always sees the $ (Steve's "always show costs" rule).
+const ENGINE_COST_USD = { macvision: 0, gemini: 0.0006, gcv: 0.0015 };
+function engineAvailable(e) {
+ if (e === 'macvision') return HAS_MACVISION;
+ if (e === 'gemini') return !!GEMINI_API_KEY;
+ if (e === 'gcv') return !!GCV_API_KEY;
+ return false;
+}
+const availableEngines = () => ALL_ENGINES.filter(engineAvailable);
+// Resolve a requested engine to the concrete engine(s) to run. 'all' => every available one;
+// an unavailable/unknown pick falls back to the default (then any available) so a scan never dead-ends.
+function resolveEngines(req) {
+ const want = String(req || DEFAULT_ENGINE).toLowerCase().trim();
+ if (want === 'all') return availableEngines();
+ if (engineAvailable(want)) return [want];
+ if (engineAvailable(DEFAULT_ENGINE)) return [DEFAULT_ENGINE];
+ return availableEngines().slice(0, 1);
+}
+
+// Gemini multimodal (OCR + reasoning identify). Returns { response:<text>, model, error }
+// — `response` mirrors Ollama's field so JSON.parse(r.response) callers work unchanged.
+function geminiVision(b64, prompt, timeoutMs) {
+ return new Promise(resolve => {
+ if (!GEMINI_API_KEY) return resolve({ error: 'no GEMINI_API_KEY', model: GEMINI_VISION_MODEL });
+ let target; try { target = new URL(`https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_VISION_MODEL}:generateContent?key=${encodeURIComponent(GEMINI_API_KEY)}`); }
+ catch (e) { return resolve({ error: 'bad gemini url', model: GEMINI_VISION_MODEL }); }
+ const payload = JSON.stringify({ contents: [{ parts: [{ text: prompt }, { inline_data: { mime_type: 'image/jpeg', data: b64 } }] }],
+ generationConfig: { temperature: 0, responseMimeType: 'application/json' } });
+ const r = https.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: timeoutMs || 30000 },
+ res => { let d = ''; res.on('data', c => d += c); res.on('end', () => {
+ try {
+ const j = JSON.parse(d);
+ if (j.error) return resolve({ error: j.error.message || 'gemini error', model: GEMINI_VISION_MODEL });
+ const parts = (j.candidates && j.candidates[0] && j.candidates[0].content && j.candidates[0].content.parts) || [];
+ resolve({ response: parts.map(x => x.text || '').join(''), model: GEMINI_VISION_MODEL });
+ } catch (e) { resolve({ error: 'parse', model: GEMINI_VISION_MODEL }); } }); });
+ r.on('error', e => resolve({ error: e.message, model: GEMINI_VISION_MODEL }));
+ r.on('timeout', () => { r.destroy(); resolve({ error: 'timeout', model: GEMINI_VISION_MODEL }); });
+ r.write(payload); r.end();
+ });
+}
+
+// Google Cloud Vision DOCUMENT_TEXT_DETECTION (OCR only — no reasoning). Returns { text, error }.
+function gcvOcr(b64, timeoutMs) {
+ return new Promise(resolve => {
+ if (!GCV_API_KEY) return resolve({ error: 'no GCV_API_KEY' });
+ let target; try { target = new URL(`https://vision.googleapis.com/v1/images:annotate?key=${encodeURIComponent(GCV_API_KEY)}`); }
+ catch (e) { return resolve({ error: 'bad gcv url' }); }
+ const payload = JSON.stringify({ requests: [{ image: { content: b64 }, features: [{ type: 'DOCUMENT_TEXT_DETECTION' }] }] });
+ const r = https.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: timeoutMs || 20000 },
+ res => { let d = ''; res.on('data', c => d += c); res.on('end', () => {
+ try {
+ const j = JSON.parse(d);
+ if (j.error) return resolve({ error: j.error.message || 'gcv error' });
+ const r0 = (j.responses && j.responses[0]) || {};
+ if (r0.error) return resolve({ error: r0.error.message || 'gcv image error' });
+ resolve({ text: (r0.fullTextAnnotation && r0.fullTextAnnotation.text) || '' });
+ } catch (e) { resolve({ error: 'parse' }); } }); });
+ r.on('error', e => resolve({ error: e.message }));
+ r.on('timeout', () => { r.destroy(); resolve({ error: 'timeout' }); });
+ r.write(payload); r.end();
+ });
+}
+
+// Cloud OCR returns a flat text block; synthesize analyzeOcr rows from it. Font height is
+// unknown from a text-only read, so every line gets a uniform height — analyzeOcr then ranks
+// by SKU-likeness (vendor prefix / dashes / length) instead of by font size. Barcodes aren't
+// decoded by text OCR (macvision-only via Apple Vision), so cloud reads carry no `bc` ground-truth line.
+function textToRows(text) {
+ return String(text || '').split(/\r?\n/).map(l => l.trim()).filter(Boolean).map(l => ({ h: 100, t: l }));
+}
+
+// OCR one JPEG buffer with ONE engine → { engine, ok, cost_usd, ...analyzeOcr(), err }.
+function runOcr(engine, buf) {
+ return new Promise(resolve => {
+ const cost = ENGINE_COST_USD[engine] || 0;
+ const b64 = buf.toString('base64');
+ if (engine === 'macvision') {
+ const tmp = path.join(PHOTOS, `_ocr-${engine}-${Date.now()}.jpg`);
+ try { fs.writeFileSync(tmp, buf); } catch (e) { return resolve({ engine, ok: false, cost_usd: cost, err: 'write fail' }); }
+ return execFile(path.join(ROOT, 'bin/ocr'), [tmp], { timeout: 8000 }, (err, stdout) => {
+ try { fs.unlinkSync(tmp); } catch (e) {}
+ if (err) return resolve({ engine, ok: false, cost_usd: cost, err: err.message });
+ resolve(Object.assign({ engine, ok: true, cost_usd: cost }, analyzeOcr(parseOcrRows(stdout))));
+ });
+ }
+ const OCR_PROMPT = 'You are an OCR engine reading a wallcovering/fabric SAMPLE label. Transcribe EVERY piece of text you see, one text element per line. Preserve SKUs / model codes EXACTLY as printed — do NOT normalize O<->0 or I<->1, do not invent or correct anything. Reply ONLY as compact JSON: {"lines":["<line>","<line>"]}.';
+ if (engine === 'gemini') {
+ return geminiVision(b64, OCR_PROMPT, 30000).then(r => {
+ if (r.error) return resolve({ engine, ok: false, cost_usd: cost, err: r.error });
+ let lines = []; try { const j = JSON.parse(r.response || '{}'); lines = Array.isArray(j.lines) ? j.lines : []; } catch (e) { lines = String(r.response || '').split(/\r?\n/); }
+ resolve(Object.assign({ engine, ok: true, cost_usd: cost }, analyzeOcr(textToRows(lines.join('\n')))));
+ });
+ }
+ if (engine === 'gcv') {
+ return gcvOcr(b64, 20000).then(r => {
+ if (r.error) return resolve({ engine, ok: false, cost_usd: cost, err: r.error });
+ resolve(Object.assign({ engine, ok: true, cost_usd: cost }, analyzeOcr(textToRows(r.text))));
+ });
+ }
+ resolve({ engine, ok: false, cost_usd: 0, err: 'unknown engine' });
+ });
+}
+
+// Reasoning vision (brand/pattern identify) for one engine. macvision => local Ollama;
+// gemini/gcv => Gemini (GCV is OCR-only, so identify always routes to Gemini there).
+// Returns { response:<json-text>, model, error } to match the Ollama shape callers parse.
+function visionJSON(engine, b64, prompt, opts) {
+ opts = opts || {};
+ if (engine === 'macvision') return ollamaVision(b64, prompt, opts.timeoutMs, opts.model);
+ return geminiVision(b64, prompt, opts.timeoutMs || 30000);
+}
+
// Fuzzy-match a VLM brand guess to a real known vendor (the VLM mis-spells: "BRUNSWIG"
// -> "Brunschwig & Fils"). Normalized Levenshtein over seed + learned vendor names.
function _lev(a, b) {
@@ -653,9 +778,10 @@ const appHandler = (req, res) => {
if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
const b64 = p.dataUrl.replace(/^data:image\/\w+;base64,/, '');
const prompt = 'This is a wallcovering or fabric SAMPLE label. Identify the BRAND from its logo or wordmark and note the typesetting. Reply ONLY as compact JSON: {"brand":"<manufacturer/brand or empty if unsure>","confidence":<0-1>,"logo":"<short logo/wordmark description>","typeface":"<e.g. serif wordmark / sans caps / script>","code":"<any SKU or model number visible, else empty>"}';
- // model toggle: {model:"fast"|"moondream"|"qwen2.5vl:7b"|...} or {fast:true} → moondream (faster, less accurate)
- const useModel = pickVisionModel(p.fast ? 'fast' : p.model);
- const r = await ollamaVision(b64, prompt, p.fast ? 25000 : undefined, useModel);
+ // engine-pluggable ({engine:...}); macvision keeps the model toggle ({model:.. }/{fast:true} → moondream).
+ const engine = resolveEngines(p.engine)[0] || DEFAULT_ENGINE;
+ const useModel = engine === 'macvision' ? pickVisionModel(p.fast ? 'fast' : p.model) : GEMINI_VISION_MODEL;
+ const r = await visionJSON(engine, b64, prompt, { timeoutMs: p.fast ? 25000 : undefined, model: useModel });
let out = {}; try { out = JSON.parse(r.response || '{}'); } catch (e) { out = {}; }
const brand = (out.brand || '').toString().trim();
// primary: cross-check the VLM's brand guess against the learned lexicon, then fuzzy name.
@@ -667,7 +793,7 @@ const appHandler = (req, res) => {
const fp = fingerprintVendor(brand, out.typeface);
if (fp) { matched = fp.vendor; matchVia = 'fingerprint'; }
}
- return send(res, 200, { ok: !r.error, model: useModel,
+ return send(res, 200, { ok: !r.error, model: useModel, engine, cost_usd: ENGINE_COST_USD[engine] || 0,
brand: brand || null, confidence: (out.confidence ?? null), logo: out.logo || null,
typeface: out.typeface || null, code: (out.code || '').toString().toUpperCase().replace(/\s+/g, '') || null,
vendor: matched, match_via: matchVia, err: r.error || null });
@@ -686,12 +812,13 @@ const appHandler = (req, res) => {
if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
const b64 = p.dataUrl.replace(/^data:image\/\w+;base64,/, '');
const prompt = 'You are looking at a wallcovering or fabric SWATCH — the material itself, NOT a printed label. Describe the PATTERN so it can be matched against a catalog. Reply ONLY as compact JSON: {"description":"<one short sentence>","motif":"<main motif e.g. floral, damask, geometric, grasscloth, stripe, botanical, abstract, ikat, toile>","style":"<e.g. traditional, modern, transitional, scandinavian, art deco, contemporary>","material":"<e.g. grasscloth, non-woven, silk, vinyl, paper, leather>","colors":["<color name>","<color name>"],"background":"<background color name>","scale":"<small | medium | large>","code":"<any SKU or model number printed on it, else empty>"}';
- const r = await ollamaVision(b64, prompt, 90000); // allow for a cold model load
+ const engine = resolveEngines(p.engine)[0] || DEFAULT_ENGINE;
+ const r = await visionJSON(engine, b64, prompt, { timeoutMs: 90000 }); // allow for a cold model load
let a = {}; try { a = JSON.parse(r.response || '{}'); } catch (e) { a = {}; }
const terms = similarTerms(a);
let items = []; try { items = await unifiedSimilar(terms); } catch (e) { items = []; }
const code = (a.code || '').toString().toUpperCase().replace(/\s+/g, '');
- return send(res, 200, { ok: !r.error, model: OLLAMA_VISION_MODEL,
+ return send(res, 200, { ok: !r.error, model: (engine === 'macvision' ? OLLAMA_VISION_MODEL : GEMINI_VISION_MODEL), engine,
recognized: {
description: a.description || null, motif: a.motif || null, style: a.style || null,
material: a.material || null, colors: Array.isArray(a.colors) ? a.colors.slice(0, 6) : [],
@@ -916,21 +1043,29 @@ const appHandler = (req, res) => {
return;
}
- // OCR a photo of a printed mfr#/SKU (macOS Vision, local/free) → ranked SKU candidates.
+ // OCR a photo of a printed mfr#/SKU → ranked SKU candidates. Engine-pluggable:
+ // {engine:"macvision"|"gemini"|"gcv"|"all"} (or ?engine=). 'all' runs every available
+ // engine and returns each under by_engine, picking the strongest read as the top-level result.
if (u.pathname === '/api/ocr' && req.method === 'POST') {
let body = '';
req.on('data', c => { body += c; if (body.length > 15 * 1024 * 1024) req.destroy(); });
- req.on('end', () => {
+ req.on('end', async () => {
let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
- const tmp = path.join(PHOTOS, `_ocr-${Date.now()}.jpg`);
- try { fs.writeFileSync(tmp, Buffer.from(p.dataUrl.replace(/^data:image\/\w+;base64,/, ''), 'base64')); }
- catch (e) { return send(res, 500, { err: 'write fail' }); }
- execFile(path.join(ROOT, 'bin/ocr'), [tmp], { timeout: 8000 }, (err, stdout) => {
- try { fs.unlinkSync(tmp); } catch (e) {}
- // OCR output → ranked SKU candidates (parsing + ranking extracted to module scope).
- send(res, 200, Object.assign({ ok: true }, analyzeOcr(parseOcrRows(stdout))));
- });
+ let buf; try { buf = Buffer.from(p.dataUrl.replace(/^data:image\/\w+;base64,/, ''), 'base64'); }
+ catch (e) { return send(res, 400, { err: 'bad dataUrl' }); }
+ const engines = resolveEngines(p.engine || u.searchParams.get('engine'));
+ if (!engines.length) return send(res, 503, { err: 'no OCR engine available', all: ALL_ENGINES, available: availableEngines() });
+ try {
+ const results = await Promise.all(engines.map(e => runOcr(e, buf)));
+ // Best read = barcode ground-truth > strong lock > most candidates (only among ok reads).
+ const rank = r => (r.ok ? 1 : 0) * ((r.barcode ? 1000 : 0) + (r.topStrong ? 100 : 0) + ((r.candidates || []).length));
+ const primary = results.length === 1 ? results[0] : results.slice().sort((a, b) => rank(b) - rank(a))[0];
+ const out = Object.assign({}, primary, { ok: !!(primary && primary.ok),
+ engine_used: primary && primary.engine, engines_run: engines, available: availableEngines() });
+ if (results.length > 1) { out.by_engine = {}; results.forEach(r => { out.by_engine[r.engine] = r; }); }
+ send(res, 200, out);
+ } catch (e) { send(res, 500, { err: 'ocr fail: ' + e.message }); }
});
return;
}
← 7bb6521 docs: expose photocapture publicly via Cloudflare Tunnel at
·
back to Dw Photo Capture
·
chore: Kamatera deploy artifacts — .deploy.conf (gemini engi d3b12d1 →