← back to AbramsOS
tick 4: PDF parsing for receipt attachments (tier-3)
d95d5e568c13dd9a4f9c6976971ab864277b212a · 2026-05-10 00:32:29 -0700 · Steve
- npm install pdf-parse
- lib/pdf-parser.js: parseFile(), parseBuffer(), resolveDocumentPath()
(lazy-require pdf-parse so its test fixture doesn't load at startup)
- routes/documents.js: GET /api/documents (list), POST /api/documents/:id/parse
pipeline: extract PDF text → synthesize gmail-summary → run heuristic + LLM
→ insert purchase + audit_log entries
- /api/documents wired into server.js behind requireAuth
- 25/25 tests (1 skip — pdf-parse sample fixture not installed in production)
Files touched
A lib/pdf-parser.jsM package-lock.jsonM package.jsonA routes/documents.jsM server.jsA tests/pdf-parser.test.js
Diff
commit d95d5e568c13dd9a4f9c6976971ab864277b212a
Author: Steve <steve@designerwallcoverings.com>
Date: Sun May 10 00:32:29 2026 -0700
tick 4: PDF parsing for receipt attachments (tier-3)
- npm install pdf-parse
- lib/pdf-parser.js: parseFile(), parseBuffer(), resolveDocumentPath()
(lazy-require pdf-parse so its test fixture doesn't load at startup)
- routes/documents.js: GET /api/documents (list), POST /api/documents/:id/parse
pipeline: extract PDF text → synthesize gmail-summary → run heuristic + LLM
→ insert purchase + audit_log entries
- /api/documents wired into server.js behind requireAuth
- 25/25 tests (1 skip — pdf-parse sample fixture not installed in production)
---
lib/pdf-parser.js | 44 ++++++++++
package-lock.json | 217 +++++++++++++++++++++++++++++++++++++++++++++++
package.json | 1 +
routes/documents.js | 96 +++++++++++++++++++++
server.js | 2 +
tests/pdf-parser.test.js | 27 ++++++
6 files changed, 387 insertions(+)
diff --git a/lib/pdf-parser.js b/lib/pdf-parser.js
new file mode 100644
index 0000000..edd186d
--- /dev/null
+++ b/lib/pdf-parser.js
@@ -0,0 +1,44 @@
+// PDF text extractor for receipt attachments (tier-3 of receipt-extractor pipeline).
+// Wraps pdf-parse. Returns extracted plain text + page count + basic metadata.
+
+const fs = require('fs');
+const path = require('path');
+// Lazy-require pdf-parse to avoid loading its test fixture at startup.
+function pdfParse() { return require('pdf-parse'); }
+
+const UPLOADS_DIR = path.join(__dirname, '..', 'uploads');
+
+async function parseFile(absPath) {
+ const buf = await fs.promises.readFile(absPath);
+ return parseBuffer(buf);
+}
+
+async function parseBuffer(buf) {
+ const data = await pdfParse()(buf);
+ return {
+ text: (data.text || '').trim(),
+ pages: data.numpages || 0,
+ info: data.info || {},
+ metadata: data.metadata || null,
+ version: data.version || null,
+ };
+}
+
+/**
+ * Resolve a document.object_path to a real file on disk.
+ * Drive files are saved as `drive:<id>` with a real file at uploads/drive-<id>.<ext>.
+ */
+function resolveDocumentPath(objectPath, mime) {
+ if (objectPath?.startsWith('drive:')) {
+ const driveId = objectPath.slice('drive:'.length);
+ const ext = mime === 'application/pdf' ? '.pdf' : mime?.startsWith('image/jpeg') ? '.jpg' : mime?.startsWith('image/png') ? '.png' : '';
+ return path.join(UPLOADS_DIR, `drive-${driveId}${ext}`);
+ }
+ // Future: gmail attachments will save under uploads/gmail-<msg>-<idx>.pdf
+ if (objectPath && !path.isAbsolute(objectPath)) {
+ return path.join(UPLOADS_DIR, objectPath);
+ }
+ return objectPath;
+}
+
+module.exports = { parseFile, parseBuffer, resolveDocumentPath, UPLOADS_DIR };
diff --git a/package-lock.json b/package-lock.json
index 66c82b6..a12d00d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -19,6 +19,7 @@
"helmet": "^8.0.0",
"morgan": "^1.10.0",
"otplib": "^12.0.1",
+ "pdf-parse": "^2.4.5",
"pg": "^8.13.0",
"plaid": "^42.2.0",
"qrcode": "^1.5.4",
@@ -31,6 +32,190 @@
"node": ">=18"
}
},
+ "node_modules/@napi-rs/canvas": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz",
+ "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==",
+ "license": "MIT",
+ "workspaces": [
+ "e2e/*"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "optionalDependencies": {
+ "@napi-rs/canvas-android-arm64": "0.1.80",
+ "@napi-rs/canvas-darwin-arm64": "0.1.80",
+ "@napi-rs/canvas-darwin-x64": "0.1.80",
+ "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80",
+ "@napi-rs/canvas-linux-arm64-gnu": "0.1.80",
+ "@napi-rs/canvas-linux-arm64-musl": "0.1.80",
+ "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80",
+ "@napi-rs/canvas-linux-x64-gnu": "0.1.80",
+ "@napi-rs/canvas-linux-x64-musl": "0.1.80",
+ "@napi-rs/canvas-win32-x64-msvc": "0.1.80"
+ }
+ },
+ "node_modules/@napi-rs/canvas-android-arm64": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz",
+ "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-darwin-arm64": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz",
+ "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-darwin-x64": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz",
+ "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz",
+ "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==",
+ "cpu": [
+ "arm"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm64-gnu": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz",
+ "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm64-musl": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz",
+ "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz",
+ "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==",
+ "cpu": [
+ "riscv64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-x64-gnu": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz",
+ "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-x64-musl": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz",
+ "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
+ "node_modules/@napi-rs/canvas-win32-x64-msvc": {
+ "version": "0.1.80",
+ "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz",
+ "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">= 10"
+ }
+ },
"node_modules/@otplib/core": {
"version": "12.0.1",
"resolved": "https://registry.npmjs.org/@otplib/core/-/core-12.0.1.tgz",
@@ -1753,6 +1938,38 @@
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"license": "MIT"
},
+ "node_modules/pdf-parse": {
+ "version": "2.4.5",
+ "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz",
+ "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@napi-rs/canvas": "0.1.80",
+ "pdfjs-dist": "5.4.296"
+ },
+ "bin": {
+ "pdf-parse": "bin/cli.mjs"
+ },
+ "engines": {
+ "node": ">=20.16.0 <21 || >=22.3.0"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/mehmet-kozan"
+ }
+ },
+ "node_modules/pdfjs-dist": {
+ "version": "5.4.296",
+ "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz",
+ "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=20.16.0 || >=22.3.0"
+ },
+ "optionalDependencies": {
+ "@napi-rs/canvas": "^0.1.80"
+ }
+ },
"node_modules/pg": {
"version": "8.20.0",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
diff --git a/package.json b/package.json
index 68f5320..a094e51 100644
--- a/package.json
+++ b/package.json
@@ -21,6 +21,7 @@
"helmet": "^8.0.0",
"morgan": "^1.10.0",
"otplib": "^12.0.1",
+ "pdf-parse": "^2.4.5",
"pg": "^8.13.0",
"plaid": "^42.2.0",
"qrcode": "^1.5.4",
diff --git a/routes/documents.js b/routes/documents.js
new file mode 100644
index 0000000..0e3e635
--- /dev/null
+++ b/routes/documents.js
@@ -0,0 +1,96 @@
+// Document operations — currently: parse a PDF and (optionally) create a purchase from it.
+
+const express = require('express');
+const fs = require('fs');
+const path = require('path');
+const db = require('../lib/db');
+const audit = require('../lib/audit');
+const { id } = require('../lib/ids');
+const pdf = require('../lib/pdf-parser');
+const extractor = require('../lib/receipt-extractor');
+
+const router = express.Router();
+const DEV_USER_ID = 'user_steve';
+
+router.get('/api/documents', async (_req, res) => {
+ const r = await db.query(
+ `SELECT id, kind, mime, object_path, parsed_status, created_at
+ FROM document WHERE user_id = $1 ORDER BY created_at DESC LIMIT 200`,
+ [DEV_USER_ID]
+ );
+ res.json(r.rows);
+});
+
+router.post('/api/documents/:id/parse', async (req, res) => {
+ const docId = req.params.id;
+ const r = await db.query(`SELECT * FROM document WHERE id = $1 AND user_id = $2`, [docId, DEV_USER_ID]);
+ if (!r.rows.length) return res.status(404).json({ error: 'document not found' });
+ const doc = r.rows[0];
+
+ if (doc.mime !== 'application/pdf') {
+ return res.status(400).json({ error: 'only application/pdf supported in tick 4 (image OCR is a later tick)' });
+ }
+
+ const filePath = pdf.resolveDocumentPath(doc.object_path, doc.mime);
+ if (!filePath || !fs.existsSync(filePath)) {
+ return res.status(404).json({ error: `file not found on disk: ${filePath}` });
+ }
+
+ try {
+ const parsed = await pdf.parseFile(filePath);
+ await db.query(`UPDATE document SET parsed_status = 'parsed' WHERE id = $1`, [docId]);
+
+ // Build a synthetic gmail-summary so the existing extractor can run unchanged.
+ const synthSummary = {
+ headers: { subject: path.basename(filePath), from: '', date: new Date(doc.created_at).toISOString() },
+ body: { text: parsed.text, html: '' },
+ };
+ let extracted = extractor.extract(synthSummary);
+ if (extracted && extracted.confidence < extractor.LLM_THRESHOLD) {
+ try {
+ extracted = await extractor.extractWithFallback(synthSummary, { llm: true, timeoutMs: 25_000 });
+ } catch (_) { /* keep heuristic */ }
+ }
+
+ let purchaseId = null;
+ if (extracted) {
+ purchaseId = id('purchase');
+ await db.query(
+ `INSERT INTO purchase
+ (id, user_id, source_message_id, merchant_name, merchant_domain, order_number, purchase_date, total_amount, currency, confidence, raw_extract)
+ VALUES ($1, $2, NULL, $3, $4, $5, $6, $7, $8, $9, $10)`,
+ [purchaseId, DEV_USER_ID, extracted.merchant, extracted.merchantDomain, extracted.orderNumber, extracted.purchaseDate, extracted.total, extracted.currency, extracted.confidence, extracted]
+ );
+ await audit.log({
+ actorType: 'system',
+ objectType: 'purchase',
+ objectId: purchaseId,
+ eventType: 'purchase_extracted',
+ metadata: { source: 'pdf', document_id: docId, merchant: extracted.merchant, total: extracted.total, confidence: extracted.confidence, source_tier: extracted.source },
+ });
+ }
+
+ await audit.log({
+ actorType: 'user',
+ actorId: DEV_USER_ID,
+ objectType: 'document',
+ objectId: docId,
+ eventType: 'document_parsed',
+ metadata: { pages: parsed.pages, chars: parsed.text.length, purchase_id: purchaseId },
+ });
+
+ res.json({
+ ok: true,
+ pages: parsed.pages,
+ chars: parsed.text.length,
+ preview: parsed.text.slice(0, 400),
+ purchase: extracted ? { id: purchaseId, ...extracted } : null,
+ });
+ } catch (err) {
+ console.error('[document parse]', err);
+ await db.query(`UPDATE document SET parsed_status = 'failed' WHERE id = $1`, [docId]);
+ res.status(500).json({ error: err.message });
+ }
+});
+
+module.exports = router;
diff --git a/server.js b/server.js
index 02af15c..9325bde 100644
--- a/server.js
+++ b/server.js
@@ -15,6 +15,7 @@ const connectors = require('./routes/connectors');
const purchases = require('./routes/purchases');
const importRouter = require('./routes/import');
const plaidRouter = require('./routes/plaid');
+const documentsRouter = require('./routes/documents');
const app = express();
const PORT = parseInt(process.env.PORT || '9931', 10);
@@ -53,6 +54,7 @@ app.use(oauth); // /auth/google/start, /auth/google/callback
app.use(connectors); // /connectors, /api/connectors, sync
app.use(purchases); // /purchases, /api/purchases
app.use(plaidRouter); // /api/plaid/*
+app.use(documentsRouter); // /api/documents, /api/documents/:id/parse
// Step-up-required routes (must re-verify TOTP within 60s)
app.use('/import', requireStepUp, importRouter);
diff --git a/tests/pdf-parser.test.js b/tests/pdf-parser.test.js
new file mode 100644
index 0000000..696ff5d
--- /dev/null
+++ b/tests/pdf-parser.test.js
@@ -0,0 +1,27 @@
+const test = require('node:test');
+const assert = require('node:assert');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const pdf = require('../lib/pdf-parser');
+
+// pdf-parse ships with a sample PDF. Use it as a known-good fixture.
+const FIXTURE = path.join(__dirname, '..', 'node_modules', 'pdf-parse', 'test', 'data', '01-valid.pdf');
+
+test('parseFile extracts text + page count', async (t) => {
+ if (!fs.existsSync(FIXTURE)) return t.skip('pdf-parse sample fixture not installed');
+ const r = await pdf.parseFile(FIXTURE);
+ assert.ok(r.pages > 0, 'pages should be > 0');
+ assert.ok(typeof r.text === 'string');
+ assert.ok(r.text.length > 0, 'text should be extracted');
+});
+
+test('resolveDocumentPath maps drive: paths to uploads/', () => {
+ const p = pdf.resolveDocumentPath('drive:abc123', 'application/pdf');
+ assert.ok(p.endsWith('uploads/drive-abc123.pdf'));
+});
+
+test('resolveDocumentPath honors mime for image extensions', () => {
+ const p = pdf.resolveDocumentPath('drive:xyz', 'image/png');
+ assert.ok(p.endsWith('uploads/drive-xyz.png'));
+});
← 600c717 tick 3 (scaffold): CPSC recall_event + recall_match tables +
·
back to AbramsOS
·
tick 5: calendar_reminder table + reminder engine b6f84e0 →