← back to Dw Photo Capture
auto-save: 2026-06-25T23:07:50 (4 files) — bin/ocr bin/ocr.swift data/build.json server.js
8fae114e6ee4be9f633f29ff5fa959adccf492f4 · 2026-06-25 23:07:52 -0700 · Steve Abrams
Files touched
M bin/ocrM bin/ocr.swiftM data/build.jsonM server.js
Diff
commit 8fae114e6ee4be9f633f29ff5fa959adccf492f4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 23:07:52 2026 -0700
auto-save: 2026-06-25T23:07:50 (4 files) — bin/ocr bin/ocr.swift data/build.json server.js
---
bin/ocr | Bin 69824 -> 66368 bytes
bin/ocr.swift | 27 +++++++++++++++++------
data/build.json | 5 +++--
server.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++------
4 files changed, 83 insertions(+), 16 deletions(-)
diff --git a/bin/ocr b/bin/ocr
index 368627e..b8e30cf 100755
Binary files a/bin/ocr and b/bin/ocr differ
diff --git a/bin/ocr.swift b/bin/ocr.swift
index 753a8e5..6570158 100644
--- a/bin/ocr.swift
+++ b/bin/ocr.swift
@@ -4,16 +4,29 @@ import AppKit
let args = CommandLine.arguments
guard args.count > 1, let img = NSImage(contentsOfFile: args[1]),
let cg = img.cgImage(forProposedRect: nil, context: nil, hints: nil) else { print(""); exit(1) }
-let req = VNRecognizeTextRequest()
-req.recognitionLevel = .accurate
-req.usesLanguageCorrection = false
+
+// One image pass, two requests: printed-text OCR + barcode/QR detection.
+let textReq = VNRecognizeTextRequest()
+textReq.recognitionLevel = .accurate
+textReq.usesLanguageCorrection = false
+let barReq = VNDetectBarcodesRequest() // many sample labels carry a barcode = exact SKU
+
let h = VNImageRequestHandler(cgImage: cg, options: [:])
-try? h.perform([req])
+try? h.perform([textReq, barReq])
+
var lines: [String] = []
-for obs in (req.results ?? []) {
+// Barcodes FIRST and tagged "BC" — the server treats a payload as ground truth (it locks
+// immediately, ahead of any OCR'd code). symbology is appended for context.
+for obs in (barReq.results ?? []) {
+ if let payload = obs.payloadStringValue, !payload.isEmpty {
+ let sym = obs.symbology.rawValue.replacingOccurrences(of: "VNBarcodeSymbology", with: "")
+ lines.append("BC\t\(payload)\t\(sym)")
+ }
+}
+// Text lines as "<heightThousandths>\t<text>" — boundingBox.height is normalized, so bigger
+// printed text = taller box; the server ranks "largest letters first".
+for obs in (textReq.results ?? []) {
if let t = obs.topCandidates(1).first?.string {
- // boundingBox.height is normalized (fraction of image height) → bigger printed
- // text = taller box. Emit it so the server can rank "largest letters first".
let hgt = Int((obs.boundingBox.height * 1000).rounded())
lines.append("\(hgt)\t\(t)")
}
diff --git a/data/build.json b/data/build.json
index 2a81bb3..1d2a7da 100644
--- a/data/build.json
+++ b/data/build.json
@@ -1,5 +1,5 @@
{
- "next": 46,
+ "next": 47,
"map": {
"63863152": 27,
"578af86f": 2,
@@ -44,6 +44,7 @@
"260c91c6": 42,
"3b47a090": 43,
"798e88ce": 44,
- "77dbf02a": 45
+ "77dbf02a": 45,
+ "5ec6a90a": 46
}
}
\ No newline at end of file
diff --git a/server.js b/server.js
index 42ff2f2..88873c0 100644
--- a/server.js
+++ b/server.js
@@ -284,13 +284,21 @@ const CODE_TOKEN = /[A-Z0-9][A-Z0-9-]{2,13}/g; // candidate token sh
function parseOcrRows(stdout) {
return (stdout || '').trim().split('\n').filter(Boolean).map(l => {
const i = l.indexOf('\t');
- return i < 0 ? { h: 0, t: l } : { h: parseInt(l.slice(0, i), 10) || 0, t: l.slice(i + 1) };
+ if (i < 0) return { h: 0, t: l };
+ const tag = l.slice(0, i), rest = l.slice(i + 1);
+ if (tag === 'BC') { // barcode line: "BC\t<payload>\t<symbology>"
+ const j = rest.indexOf('\t');
+ return j < 0 ? { h: 10000, t: rest, bc: true } : { h: 10000, t: rest.slice(0, j), bc: true, sym: rest.slice(j + 1) };
+ }
+ return { h: parseInt(tag, 10) || 0, t: rest };
});
}
// Pure: OCR rows → { text, vendor, candidates, top, topStrong }. No I/O.
function analyzeOcr(rows) {
- const text = rows.map(r => r.t).join('\n');
+ // Barcode payloads are ground truth (exact SKU) — they outrank every OCR'd code.
+ const barcodes = rows.filter(r => r.bc).map(r => r.t.toUpperCase().replace(/\s+/g, '')).filter(b => b.length >= 4);
+ const text = rows.filter(r => !r.bc).map(r => r.t).join('\n');
const vlex = LEXICON.find(v => v.re.test(text.toUpperCase())) || null;
const vendor = vlex ? vlex.name : null;
// SKU-likeness tie-breaker: brand-matching prefix first (the swatch told us the vendor),
@@ -306,6 +314,7 @@ function analyzeOcr(rows) {
// names and marketing prose are intentionally ignored: scan the code, never the text.
const seen = new Set(); const codes = [];
for (const r of rows) {
+ if (r.bc) continue;
for (const t of (r.t.toUpperCase().match(CODE_TOKEN) || [])) {
if (!/\d/.test(t) || t.length < 4 || seen.has(t)) continue; // must hold a digit, >=4 chars
seen.add(t); codes.push({ t, h: r.h, s: skuScore(t) });
@@ -313,14 +322,18 @@ function analyzeOcr(rows) {
}
// LARGEST CODE FIRST: font height, then SKU-likeness (vendor prefix), then length.
codes.sort((a, b) => (b.h - a.h) || (b.s - a.s) || (b.t.length - a.t.length));
- const top = codes[0] || null;
const hMax = codes.reduce((m, c) => Math.max(m, c.h), 0);
- // STRONG = safe to LOCK on the first sighting: a known SKU prefix, a brand-prefix match,
- // or a brand name on the swatch + this being the font-dominant code (biggest = the SKU).
+ // Candidate order: barcode payloads (ground truth) first, then text codes largest-first.
+ const candidates = [...new Set([...barcodes, ...codes.map(c => c.t)])];
+ const top = candidates[0] || null;
+ const tc = codes[0] || null;
+ // STRONG = safe to LOCK on the first sighting: a barcode (always), a known SKU prefix,
+ // a brand-prefix match, or a brand on the swatch + the font-dominant code.
const topStrong = !!top && (
- STRONG_PFX.test(top.t) || (vlex && vlex.pfx.test(top.t)) || (!!vendor && top.h >= hMax)
+ barcodes.length > 0 ||
+ STRONG_PFX.test(top) || (vlex && vlex.pfx.test(top)) || (!!vendor && tc && tc.h >= hMax)
);
- return { text, vendor, candidates: codes.map(c => c.t), top: top ? top.t : null, topStrong };
+ return { text, vendor, candidates, top, topStrong, barcode: barcodes[0] || null };
}
@@ -366,6 +379,24 @@ function rebuildLexicon() {
}
rebuildLexicon();
+// ── Local VLM brand/logo recognition (Ollama qwen2.5vl, $0) ────────────────────
+// Reads a STYLIZED logo/wordmark + typesetting that the deterministic OCR can't, to name
+// the vendor when no brand text was cleanly recognized. An on-lock assist, not per-frame.
+const OLLAMA_VISION_MODEL = process.env.OLLAMA_VISION_MODEL || 'qwen2.5vl:7b'; // reuses OLLAMA_URL declared with ollamaAsk
+function ollamaVision(b64, prompt) {
+ return new Promise(resolve => {
+ let target; try { target = new URL('/api/generate', OLLAMA_URL); } catch (e) { return resolve({ error: 'bad OLLAMA_URL' }); }
+ const lib = target.protocol === 'https:' ? https : http;
+ const payload = JSON.stringify({ model: OLLAMA_VISION_MODEL, prompt, images: [b64],
+ stream: false, format: 'json', options: { temperature: 0 } });
+ const r = lib.request(target, { method: 'POST', headers: { 'Content-Type': 'application/json' }, timeout: 35000 },
+ res => { let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { resolve({ error: 'parse' }); } }); });
+ r.on('error', e => resolve({ error: e.message }));
+ r.on('timeout', () => { r.destroy(); resolve({ error: 'timeout' }); });
+ r.write(payload); r.end();
+ });
+}
+
const appHandler = (req, res) => {
// public (no-auth) paths: health + the home-screen-install assets iOS fetches without creds
const PUBLIC = ['/healthz', '/icon-180.png', '/icon-512.png', '/apple-touch-icon.png', '/manifest.webmanifest'];
@@ -500,6 +531,28 @@ const appHandler = (req, res) => {
return res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8', 'Cache-Control': 'no-store' }), res.end(html);
}
+ // Recognize the BRAND by its logo/wordmark + typesetting via the local VLM (qwen2.5vl).
+ // The assist for when OCR couldn't read the brand text but the logo is visible. $0 (local).
+ if (u.pathname === '/api/identify' && req.method === 'POST') {
+ let body = ''; req.on('data', c => { body += c; if (body.length > 15 * 1024 * 1024) req.destroy(); });
+ req.on('end', async () => {
+ let p; try { p = JSON.parse(body); } catch (e) { return send(res, 400, { err: 'bad json' }); }
+ if (!p.dataUrl) return send(res, 400, { err: 'dataUrl required' });
+ const b64 = p.dataUrl.replace(/^data:image\/\w+;base64,/, '');
+ const prompt = 'This is a wallcovering or fabric SAMPLE label. Identify the BRAND from its logo or wordmark and note the typesetting. Reply ONLY as compact JSON: {"brand":"<manufacturer/brand or empty if unsure>","confidence":<0-1>,"logo":"<short logo/wordmark description>","typeface":"<e.g. serif wordmark / sans caps / script>","code":"<any SKU or model number visible, else empty>"}';
+ const r = await ollamaVision(b64, prompt);
+ let out = {}; try { out = JSON.parse(r.response || '{}'); } catch (e) { out = {}; }
+ const brand = (out.brand || '').toString().trim();
+ // cross-check the VLM's brand guess against the learned lexicon for a canonical name
+ const matched = brand ? ((LEXICON.find(v => v.re.test(brand.toUpperCase())) || {}).name || null) : null;
+ return send(res, 200, { ok: !r.error, model: OLLAMA_VISION_MODEL,
+ brand: brand || null, confidence: (out.confidence ?? null), logo: out.logo || null,
+ typeface: out.typeface || null, code: (out.code || '').toString().toUpperCase().replace(/\s+/g, '') || null,
+ vendor: matched, err: r.error || null });
+ });
+ return;
+ }
+
// /selfcheck — re-runnable health report. Verifies data integrity + AUTO-CLEANS stale
// photo refs (progress entries pointing at a deleted /photos file = the dead-link class).
if (u.pathname === '/selfcheck') {
← 5ebe4e6 Learn how each vendor labels their samples (180 vendor profi
·
back to Dw Photo Capture
·
Scan: recognize barcodes (exact SKU) + logos/typeface (local d30783b →