← back to AbramsOS
tick 1: receipt extractor tier-2 LLM fallback (Mac1 qwen3:14b)
323f8a8f54c4dc1e474a02c7716881f71e102e13 · 2026-05-10 00:26:18 -0700 · Steve
- lib/ollama.js: thin Ollama client with strict-JSON mode + timeout
- lib/receipt-extractor.js: enrichWithLlm() merges into low-conf heuristic results;
heuristic-trust rule (LLM never overrides a heuristic-filled field)
- routes/connectors.js: gmail sync now calls extractWithFallback when conf < 0.7
- tests: 3 new in tests/extractor-llm.test.js with cache-injected mock ollama
- Total: 22/22 green; live smoke against Mac1 confirms end-to-end works
Files touched
A lib/ollama.jsM lib/receipt-extractor.jsM routes/connectors.jsA tests/extractor-llm.test.js
Diff
commit 323f8a8f54c4dc1e474a02c7716881f71e102e13
Author: Steve <steve@designerwallcoverings.com>
Date: Sun May 10 00:26:18 2026 -0700
tick 1: receipt extractor tier-2 LLM fallback (Mac1 qwen3:14b)
- lib/ollama.js: thin Ollama client with strict-JSON mode + timeout
- lib/receipt-extractor.js: enrichWithLlm() merges into low-conf heuristic results;
heuristic-trust rule (LLM never overrides a heuristic-filled field)
- routes/connectors.js: gmail sync now calls extractWithFallback when conf < 0.7
- tests: 3 new in tests/extractor-llm.test.js with cache-injected mock ollama
- Total: 22/22 green; live smoke against Mac1 confirms end-to-end works
---
lib/ollama.js | 59 ++++++++++++++++++++++++++++++
lib/receipt-extractor.js | 88 +++++++++++++++++++++++++++++++++++++++++++--
routes/connectors.js | 8 ++++-
tests/extractor-llm.test.js | 82 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 234 insertions(+), 3 deletions(-)
diff --git a/lib/ollama.js b/lib/ollama.js
new file mode 100644
index 0000000..91eaeba
--- /dev/null
+++ b/lib/ollama.js
@@ -0,0 +1,59 @@
+// Local Ollama client. Mac1 is the bulk default; Mac2 is overflow.
+// Per Steve's rules: NEVER call the Anthropic API. Local Ollama only.
+
+const DEFAULT_BASE = process.env.OLLAMA_BASE_URL || 'http://192.168.1.133:11434';
+const DEFAULT_MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+async function generate(prompt, { model = DEFAULT_MODEL, base = DEFAULT_BASE, timeoutMs = DEFAULT_TIMEOUT_MS, format = null, system = null } = {}) {
+ const ctrl = new AbortController();
+ const t = setTimeout(() => ctrl.abort(), timeoutMs);
+ try {
+ const body = { model, prompt, stream: false };
+ if (format) body.format = format;
+ if (system) body.system = system;
+ const r = await fetch(`${base}/api/generate`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ signal: ctrl.signal,
+ });
+ if (!r.ok) throw new Error(`ollama ${r.status}: ${await r.text().catch(() => '')}`);
+ const j = await r.json();
+ return (j.response || '').trim();
+ } finally {
+ clearTimeout(t);
+ }
+}
+
+async function generateJson(prompt, opts = {}) {
+ // Use Ollama's structured-output mode by passing format: "json".
+ const raw = await generate(prompt, { ...opts, format: 'json' });
+ // Some models still wrap in code fences; strip if present.
+ const cleaned = raw.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/, '').trim();
+ try {
+ return JSON.parse(cleaned);
+ } catch (err) {
+ // Last-resort: pull first {...} block
+ const m = cleaned.match(/\{[\s\S]*\}/);
+ if (m) {
+ try { return JSON.parse(m[0]); } catch (_) {}
+ }
+ throw new Error(`ollama returned non-JSON: ${cleaned.slice(0, 200)}`);
+ }
+}
+
+async function reachable(base = DEFAULT_BASE, timeoutMs = 3000) {
+ const ctrl = new AbortController();
+ const t = setTimeout(() => ctrl.abort(), timeoutMs);
+ try {
+ const r = await fetch(`${base}/api/tags`, { signal: ctrl.signal });
+ return r.ok;
+ } catch (_) {
+ return false;
+ } finally {
+ clearTimeout(t);
+ }
+}
+
+module.exports = { generate, generateJson, reachable, DEFAULT_BASE, DEFAULT_MODEL };
diff --git a/lib/receipt-extractor.js b/lib/receipt-extractor.js
index 36d8c30..3081456 100644
--- a/lib/receipt-extractor.js
+++ b/lib/receipt-extractor.js
@@ -1,7 +1,12 @@
// Tier 1: heuristic extractor over Gmail message summaries.
-// Tier 2: local Ollama qwen3:14b fallback on Mac1 (TODO — invoked when confidence < 0.7).
+// Tier 2: local Ollama qwen3:14b fallback on Mac1 (invoked when tier-1 confidence < 0.7).
// Tier 3: PDF parsing — TODO (will need pdfjs-dist or similar).
+const ollama = require('./ollama');
+
+const LLM_THRESHOLD = 0.7; // call LLM when heuristic conf < this
+const LLM_BODY_LIMIT = 4000; // chars sent to the model (≈ 1k tokens)
+
const MERCHANT_DOMAIN_OVERRIDES = {
'amazon.com': 'Amazon',
'amazon.co.uk': 'Amazon',
@@ -160,4 +165,83 @@ function extract(summary) {
};
}
-module.exports = { extract, inferMerchant, extractTotal, extractOrderNumber, stripHtml };
+/**
+ * Tier-2 LLM enrichment. Sends the message text to local Ollama qwen3:14b
+ * with a strict-JSON prompt; merges the LLM's answers into the heuristic
+ * result for any fields the heuristic was unsure about. NEVER replaces a
+ * field the heuristic already filled with high confidence — LLM output is
+ * untrusted by default.
+ *
+ * @param {object} summary output of gmail-fetcher.summarize()
+ * @param {object} heur output of extract(summary); MUST be non-null
+ * @param {object} opts { force?: bool, model?: string, base?: string, timeoutMs?: number }
+ * @returns enriched purchase or the original heur (unchanged) on LLM failure
+ */
+async function enrichWithLlm(summary, heur, opts = {}) {
+ if (!heur) return null;
+ if (!opts.force && heur.confidence >= LLM_THRESHOLD) return heur;
+
+ const subject = summary.headers?.subject || '';
+ const from = summary.headers?.from || '';
+ const text = (summary.body?.text || stripHtml(summary.body?.html || '')).slice(0, LLM_BODY_LIMIT);
+
+ const prompt = `You are a receipt extraction tool. Extract the structured purchase from this email and return ONLY a JSON object — no prose, no code fences. Use null for fields you cannot determine. Schema:
+{
+ "merchant": string, // brand the user bought from (e.g. "Amazon", not "noreply@amazon.com")
+ "order_number": string|null,
+ "total": number|null, // total amount paid (USD or local currency, just the number)
+ "currency": string|null, // ISO 4217 (USD, EUR, GBP, etc.)
+ "items": [{ "description": string, "quantity": number, "unit_price": number|null }]
+}
+
+Email follows.
+
+From: ${from}
+Subject: ${subject}
+
+${text}`;
+
+ let llm;
+ try {
+ llm = await ollama.generateJson(prompt, {
+ model: opts.model,
+ base: opts.base,
+ timeoutMs: opts.timeoutMs || 30_000,
+ });
+ } catch (err) {
+ // LLM unavailable / timeout / non-JSON: return heuristic unchanged.
+ return { ...heur, source: 'heuristic', llmError: err.message };
+ }
+
+ // Merge: only fill what heuristic missed; never override existing values.
+ const merged = { ...heur };
+ if (!merged.merchant || merged.merchant === 'Unknown') merged.merchant = llm.merchant || merged.merchant;
+ if (!merged.orderNumber && llm.order_number) merged.orderNumber = String(llm.order_number);
+ if (merged.total == null && typeof llm.total === 'number') merged.total = llm.total;
+ if (!merged.currency && llm.currency) merged.currency = String(llm.currency).toUpperCase();
+ if ((!merged.items || !merged.items.length) && Array.isArray(llm.items)) merged.items = llm.items;
+
+ // Recompute confidence with LLM-supplied fields counted in
+ let conf = 0;
+ if (merged.total != null) conf += 0.4;
+ if (merged.orderNumber) conf += 0.25;
+ if (merged.merchant && merged.merchant !== 'Unknown') conf += 0.1;
+ if (merged.items?.length) conf += 0.15;
+ conf += 0.1; // small bonus for LLM having seen the full body
+ merged.confidence = Math.min(1, Math.round(conf * 100) / 100);
+ merged.source = 'heuristic+llm';
+ return merged;
+}
+
+/**
+ * Convenience: tier-1 then optional tier-2.
+ * extractWithFallback(summary, { llm: true })
+ */
+async function extractWithFallback(summary, opts = {}) {
+ const t1 = extract(summary);
+ if (!t1) return null;
+ if (opts.llm === false) return t1;
+ return enrichWithLlm(summary, t1, opts);
+}
+
+module.exports = { extract, enrichWithLlm, extractWithFallback, inferMerchant, extractTotal, extractOrderNumber, stripHtml, LLM_THRESHOLD };
diff --git a/routes/connectors.js b/routes/connectors.js
index 2829b56..b2af977 100644
--- a/routes/connectors.js
+++ b/routes/connectors.js
@@ -102,7 +102,13 @@ router.post('/api/connectors/:id/sync', async (req, res) => {
metadata: { gmail_id: mid, subject: summary.headers.subject?.slice(0, 120) },
});
- const extracted = extractor.extract(summary);
+ // Tier 1 heuristic; if conf < threshold and Mac1 Ollama is reachable, enrich with LLM.
+ let extracted = extractor.extract(summary);
+ if (extracted && extracted.confidence < extractor.LLM_THRESHOLD) {
+ try {
+ extracted = await extractor.extractWithFallback(summary, { llm: true, timeoutMs: 25_000 });
+ } catch (_) { /* fall through with heuristic-only result */ }
+ }
if (extracted) {
const purchaseId = id('purchase');
await db.query(
diff --git a/tests/extractor-llm.test.js b/tests/extractor-llm.test.js
new file mode 100644
index 0000000..eaa3755
--- /dev/null
+++ b/tests/extractor-llm.test.js
@@ -0,0 +1,82 @@
+// Tier-2 LLM enrichment unit tests. Mocks lib/ollama with require-cache override.
+
+const test = require('node:test');
+const assert = require('node:assert');
+
+// Mock ollama with fresh module-cache injection. Wipe any pre-loaded extractor
+// so it picks up our mock via require().
+const fakeOllama = {
+ generate: async () => '',
+ generateJson: async (_prompt) => ({
+ merchant: 'Best Buy',
+ order_number: 'BBY01-1234567',
+ total: 199.99,
+ currency: 'USD',
+ items: [{ description: 'Wireless headphones', quantity: 1, unit_price: 199.99 }],
+ }),
+ reachable: async () => true,
+ DEFAULT_BASE: 'http://stub',
+ DEFAULT_MODEL: 'stub:tiny',
+};
+
+const ollamaPath = require.resolve('../lib/ollama');
+const extractorPath = require.resolve('../lib/receipt-extractor');
+
+function installMock(mock) {
+ delete require.cache[extractorPath];
+ require.cache[ollamaPath] = { id: ollamaPath, filename: ollamaPath, loaded: true, exports: mock };
+}
+
+installMock(fakeOllama);
+const extractor = require('../lib/receipt-extractor');
+
+test('enrichWithLlm fills missing fields', async () => {
+ const summary = {
+ headers: { subject: 'Thank you for your order', from: '"Some Store" <orders@example.com>', date: new Date().toISOString() },
+ body: { text: 'Thanks for your order. Your shipment will arrive Tuesday.', html: '' },
+ };
+ const t1 = extractor.extract(summary);
+ // Heuristic should find SOMETHING low-confidence (subject hints + body lacks total/order number)
+ assert.ok(t1, 'tier-1 should produce a draft');
+ assert.ok(t1.confidence < extractor.LLM_THRESHOLD, `tier-1 confidence (${t1?.confidence}) should be below threshold (${extractor.LLM_THRESHOLD})`);
+
+ const t2 = await extractor.enrichWithLlm(summary, t1);
+ assert.strictEqual(t2.source, 'heuristic+llm');
+ // Heuristic-trust rule: merchant from From: header is kept even though LLM offers 'Best Buy'
+ assert.strictEqual(t2.merchant, t1.merchant, 'heuristic merchant should be preserved');
+ // LLM fills the fields heuristic missed
+ assert.strictEqual(t2.orderNumber, 'BBY01-1234567');
+ assert.strictEqual(t2.total, 199.99);
+ assert.strictEqual(t2.currency, 'USD');
+ assert.strictEqual(t2.items.length, 1);
+ assert.ok(t2.confidence > t1.confidence, `confidence should rise (was ${t1.confidence}, now ${t2.confidence})`);
+});
+
+test('enrichWithLlm respects high-confidence heuristic (skips LLM)', async () => {
+ const summary = {
+ headers: { subject: 'Your Amazon.com order #112-7654321-7654321', from: '"Amazon.com" <auto-confirm@amazon.com>', date: new Date().toISOString() },
+ body: { text: 'Order Total: $99.99. Thanks for your order.', html: '' },
+ };
+ const t1 = extractor.extract(summary);
+ assert.ok(t1.confidence >= extractor.LLM_THRESHOLD, 'tier-1 should already be high-conf');
+ const t2 = await extractor.enrichWithLlm(summary, t1);
+ assert.strictEqual(t2.source, 'heuristic', 'LLM should be skipped');
+ assert.strictEqual(t2, t1, 'result is the same object');
+});
+
+test('enrichWithLlm tolerates Ollama failure gracefully', async () => {
+ const failOllama = { ...fakeOllama, generateJson: async () => { throw new Error('mac1 unreachable'); } };
+ installMock(failOllama);
+ const extractor2 = require('../lib/receipt-extractor');
+ const summary = {
+ headers: { subject: 'Thank you for your order', from: '"Store" <ship@example.com>', date: new Date().toISOString() },
+ body: { text: 'Thanks for your order. Your shipment will arrive tomorrow.', html: '' },
+ };
+ const t1 = extractor2.extract(summary);
+ assert.ok(t1, 'tier-1 should produce a draft for this fixture');
+ const t2 = await extractor2.enrichWithLlm(summary, t1);
+ assert.ok(t2.llmError, 'should surface the LLM error');
+ assert.strictEqual(t2.confidence, t1.confidence, 'confidence unchanged on failure');
+ // Restore for any later tests
+ installMock(fakeOllama);
+});
← 2113e8d feat: 'Import all receipts' button + 2FA gate + Plaid sandbo
·
back to AbramsOS
·
tick 2: Google Drive sync (receipt-shaped PDFs/images) 9e812a4 →