← back to AbramsOS
tick 15: auto-parse Drive PDFs on sync (backlog item 17)
a5da75e7ed727a390bbd5661b5419761520c25de · 2026-05-10 10:01:01 -0700 · Steve
- routes/connectors.js syncDrive(): after a Drive PDF lands in document table,
immediately calls pdf.parseFile() + receipt-extractor (heuristic + LLM).
Inserts a purchase row when extractor returns a draft.
- Images skipped (OCR is a later tick); failed parses flip parsed_status='failed'
- audit_log gets document_parsed + purchase_extracted (source: 'drive_pdf')
- 34/35 tests still green
Files touched
Diff
commit a5da75e7ed727a390bbd5661b5419761520c25de
Author: Steve <steve@designerwallcoverings.com>
Date: Sun May 10 10:01:01 2026 -0700
tick 15: auto-parse Drive PDFs on sync (backlog item 17)
- routes/connectors.js syncDrive(): after a Drive PDF lands in document table,
immediately calls pdf.parseFile() + receipt-extractor (heuristic + LLM).
Inserts a purchase row when extractor returns a draft.
- Images skipped (OCR is a later tick); failed parses flip parsed_status='failed'
- audit_log gets document_parsed + purchase_extracted (source: 'drive_pdf')
- 34/35 tests still green
---
routes/connectors.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 47 insertions(+)
diff --git a/routes/connectors.js b/routes/connectors.js
index d532353..0919d4c 100644
--- a/routes/connectors.js
+++ b/routes/connectors.js
@@ -10,6 +10,7 @@ const fetcher = require('../lib/gmail-fetcher');
const drive = require('../lib/drive-fetcher');
const extractor = require('../lib/receipt-extractor');
const commitmentExtractor = require('../lib/commitment-extractor');
+const pdf = require('../lib/pdf-parser');
const UPLOADS_DIR = path.join(__dirname, '..', 'uploads');
fs.mkdirSync(UPLOADS_DIR, { recursive: true });
@@ -60,6 +61,52 @@ async function syncDrive(refreshToken, connectorId) {
eventType: 'document_persisted',
metadata: { source: 'drive', drive_id: f.id, name: f.name, mime: f.mimeType },
});
+
+ // Auto-parse PDFs (images deferred until OCR ships)
+ if (f.mimeType === 'application/pdf') {
+ try {
+ const parsed = await pdf.parseFile(localPath);
+ await db.query(`UPDATE document SET parsed_status = 'parsed' WHERE id = $1`, [docId]);
+
+ const synthSummary = {
+ headers: { subject: f.name || `drive:${f.id}`, from: '', date: new Date(f.modifiedTime || Date.now()).toISOString() },
+ body: { text: parsed.text, html: '' },
+ };
+ let extracted = extractor.extract(synthSummary);
+ if (extracted && extracted.confidence < extractor.LLM_THRESHOLD) {
+ try {
+ extracted = await extractor.extractWithFallback(synthSummary, { llm: true, timeoutMs: 25_000 });
+ } catch (_) { /* keep heuristic */ }
+ }
+
+ if (extracted) {
+ const purchaseId = id('purchase');
+ await db.query(
+ `INSERT INTO purchase
+ (id, user_id, source_message_id, merchant_name, merchant_domain, order_number, purchase_date, total_amount, currency, confidence, raw_extract)
+ VALUES ($1, $2, NULL, $3, $4, $5, $6, $7, $8, $9, $10)`,
+ [purchaseId, DEV_USER_ID, extracted.merchant, extracted.merchantDomain, extracted.orderNumber, extracted.purchaseDate, extracted.total, extracted.currency, extracted.confidence, extracted]
+ );
+ await audit.log({
+ actorType: 'system',
+ objectType: 'purchase',
+ objectId: purchaseId,
+ eventType: 'purchase_extracted',
+ metadata: { source: 'drive_pdf', document_id: docId, merchant: extracted.merchant, total: extracted.total, confidence: extracted.confidence, source_tier: extracted.source },
+ });
+ }
+ await audit.log({
+ actorType: 'system',
+ objectType: 'document',
+ objectId: docId,
+ eventType: 'document_parsed',
+ metadata: { pages: parsed.pages, chars: parsed.text.length },
+ });
+ } catch (err) {
+ console.warn('[drive-pdf-parse]', f.id, err.message);
+ await db.query(`UPDATE document SET parsed_status = 'failed' WHERE id = $1`, [docId]);
+ }
+ }
}
return { fetched: files.length, added };
}
← 8adb083 tick 14: unit tests for lib/crypto.js + lib/ids.js (backlog
·
back to AbramsOS
·
tick 16: Compliance Guardian (URL allowlist + injection scan 1666967 →