[object Object]

← back to AbramsOS

tick 18: receipt extractor regression fixtures + heuristic fixes (item 19)

3458fe150cc81f66ea914cd8bb55ca7710b21921 · 2026-05-10 12:05:26 -0700 · Steve

- 7 new golden-file fixtures in tests/fixtures/receipts/ (Amazon, Best Buy,
  DoorDash, Apple, Etsy, Uber, Walmart, plus a non-receipt newsletter)
- tests/extractor-regression.test.js auto-discovers fixtures and asserts
  merchant + total + orderNumber + minConfidence per file
- lib/receipt-extractor.js: 2 surgical heuristic fixes uncovered by fixtures
  · extractTotal now picks the highest-rank label ("order total" >
    "grand total" > "total charged" > plain "total"); allows up to 40 chars
    between label and $amount (handles "Total charged to Visa ••1234: $18.42");
    rejects "subtotal" via 3-char lookbehind
  · extractOrderNumber adds Best-Buy-style "Order #: BBY01-806589123456"
    (alpha prefix + digits) and a tighter Amazon-only pattern
- 15/15 tests across the regression + base extractor suites

Files touched

Diff

commit 3458fe150cc81f66ea914cd8bb55ca7710b21921
Author: Steve <steve@designerwallcoverings.com>
Date:   Sun May 10 12:05:26 2026 -0700

    tick 18: receipt extractor regression fixtures + heuristic fixes (item 19)
    
    - 7 new golden-file fixtures in tests/fixtures/receipts/ (Amazon, Best Buy,
      DoorDash, Apple, Etsy, Uber, Walmart, plus a non-receipt newsletter)
    - tests/extractor-regression.test.js auto-discovers fixtures and asserts
      merchant + total + orderNumber + minConfidence per file
    - lib/receipt-extractor.js: 2 surgical heuristic fixes uncovered by fixtures
      · extractTotal now picks the highest-rank label ("order total" >
        "grand total" > "total charged" > plain "total"); allows up to 40 chars
        between label and $amount (handles "Total charged to Visa ••1234: $18.42");
        rejects "subtotal" via 3-char lookbehind
      · extractOrderNumber adds Best-Buy-style "Order #: BBY01-806589123456"
        (alpha prefix + digits) and a tighter Amazon-only pattern
    - 15/15 tests across the regression + base extractor suites
---
 lib/receipt-extractor.js                           | 42 +++++++++++++++---
 tests/extractor-regression.test.js                 | 51 ++++++++++++++++++++++
 tests/fixtures/receipts/amazon-shipped.json        | 20 +++++++++
 tests/fixtures/receipts/apple.json                 | 19 ++++++++
 tests/fixtures/receipts/best-buy.json              | 20 +++++++++
 tests/fixtures/receipts/doordash.json              | 19 ++++++++
 tests/fixtures/receipts/etsy.json                  | 19 ++++++++
 .../fixtures/receipts/non-receipt-newsletter.json  | 17 ++++++++
 tests/fixtures/receipts/uber.json                  | 19 ++++++++
 tests/fixtures/receipts/walmart.json               | 20 +++++++++
 10 files changed, 239 insertions(+), 7 deletions(-)

diff --git a/lib/receipt-extractor.js b/lib/receipt-extractor.js
index bd5eb9a..84ecade 100644
--- a/lib/receipt-extractor.js
+++ b/lib/receipt-extractor.js
@@ -73,13 +73,35 @@ const CURRENCY_SYMBOLS = { '$': 'USD', '€': 'EUR', '£': 'GBP', '¥': 'JPY' };
 
 function extractTotal(text) {
   if (!text) return { amount: null, currency: null };
-  // Look for "total" / "grand total" / "order total" / "amount charged" near a money value
-  const re = /(?:order\s+total|grand\s+total|total\s+charged|amount\s+charged|total)[\s:$£€¥]*([£€¥$])\s*([0-9]{1,3}(?:[,0-9]{0,3})*(?:\.[0-9]{2})?)/i;
-  const m = text.match(re);
-  if (m) {
-    const symbol = m[1];
-    const value = parseFloat(m[2].replace(/,/g, ''));
-    if (!isNaN(value)) return { amount: value, currency: CURRENCY_SYMBOLS[symbol] || 'USD' };
+  // Find ALL money-prefixed-by-total matches; rank by how "final" the label is.
+  // "order total" / "grand total" / "total charged" / "amount charged" beat plain "total"
+  // (which beats "subtotal", "trip fare", "subtotal amount", etc. that we explicitly REJECT).
+  // Allow ≤40 non-newline non-money chars between label and money (e.g. "Total charged to Visa ••1234: $18.42")
+  const re = /(order\s+total|grand\s+total|total\s+charged|total\s+amount|amount\s+charged|total)[^\n$£€¥]{0,40}([£€¥$])\s*([0-9]{1,3}(?:[,0-9]{0,3})*(?:\.[0-9]{2})?)/gi;
+  const labelRank = (lbl) => {
+    const l = lbl.toLowerCase();
+    if (l.includes('order total') || l.includes('grand total')) return 5;
+    if (l.includes('total charged') || l.includes('amount charged')) return 4;
+    if (l.includes('total amount')) return 3;
+    return 2; // plain "total"
+  };
+  const matches = [];
+  for (const m of text.matchAll(re)) {
+    // Reject when this match is part of "subtotal" — overlap check
+    const beforeStart = Math.max(0, m.index - 3);
+    if (text.slice(beforeStart, m.index).toLowerCase() === 'sub') continue;
+    matches.push({
+      label: m[1], symbol: m[2],
+      amount: parseFloat(m[3].replace(/,/g, '')),
+      rank: labelRank(m[1]),
+      idx: m.index,
+    });
+  }
+  if (matches.length) {
+    // Highest rank wins; tie → latest in text (totals are usually at the bottom)
+    matches.sort((a, b) => b.rank - a.rank || b.idx - a.idx);
+    const best = matches[0];
+    if (!isNaN(best.amount)) return { amount: best.amount, currency: CURRENCY_SYMBOLS[best.symbol] || 'USD' };
   }
   // Fallback: any money value
   const fallback = text.match(/([£€¥$])\s*([0-9]{1,3}(?:[,0-9]{0,3})*\.[0-9]{2})/);
@@ -94,7 +116,13 @@ function extractTotal(text) {
 function extractOrderNumber(text) {
   if (!text) return null;
   const patterns = [
+    // Amazon-style hyphenated
+    /order\s*#?\s*([0-9]{3}-[0-9]{6,8}-[0-9]{6,8})/i,
+    // Best Buy / merchant-prefix style: "Order #: BBY01-806589123456"
+    /order\s*(?:number|#|id)?\s*[:#]\s*([A-Z]{2,5}\d{0,3}[-_]?\d{6,})/i,
+    // Generic alphanumeric with hyphen
     /order\s*(?:number|#|id)?\s*[:#]?\s*([A-Z0-9]{3,4}-?\d{3,}-?[A-Z0-9]+)/i,
+    // Long digit-only
     /order\s*(?:number|#|id)?\s*[:#]?\s*(#?\d{6,})/i,
     /confirmation\s*(?:number|#|code)?\s*[:#]?\s*([A-Z0-9-]{6,})/i,
     /invoice\s*(?:number|#)?\s*[:#]?\s*([A-Z0-9-]{4,})/i,
diff --git a/tests/extractor-regression.test.js b/tests/extractor-regression.test.js
new file mode 100644
index 0000000..e207d21
--- /dev/null
+++ b/tests/extractor-regression.test.js
@@ -0,0 +1,51 @@
+// Golden-file regression tests for tier-1 receipt extractor.
+// Adds new fixture? Drop a JSON file in tests/fixtures/receipts/ with shape:
+//   { fixture, expected: { merchant, total, orderNumber, minConfidence, shouldExtract? }, summary }
+// The test discovers all *.json files in that dir.
+
+const test = require('node:test');
+const assert = require('node:assert');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { extract } = require('../lib/receipt-extractor');
+
+const FIXTURES_DIR = path.join(__dirname, 'fixtures', 'receipts');
+const fixtures = fs.readdirSync(FIXTURES_DIR)
+  .filter(f => f.endsWith('.json'))
+  .map(f => ({ file: f, data: JSON.parse(fs.readFileSync(path.join(FIXTURES_DIR, f), 'utf8')) }));
+
+test(`fixtures discovered (${fixtures.length})`, () => {
+  assert.ok(fixtures.length >= 5, `expected at least 5 fixtures, found ${fixtures.length}`);
+});
+
+for (const { file, data } of fixtures) {
+  const { fixture, expected, summary } = data;
+  test(`fixture · ${fixture}`, () => {
+    const r = extract(summary);
+
+    if (expected.shouldExtract === false) {
+      assert.strictEqual(r, null, `${fixture}: expected null (non-receipt), got ${JSON.stringify(r)}`);
+      return;
+    }
+
+    assert.ok(r, `${fixture}: expected extraction, got null`);
+
+    if (expected.merchant) {
+      assert.match(r.merchant, new RegExp(expected.merchant, 'i'),
+        `${fixture}: merchant '${r.merchant}' doesn't match '${expected.merchant}'`);
+    }
+    if (expected.total != null) {
+      assert.strictEqual(r.total, expected.total,
+        `${fixture}: total ${r.total} ≠ expected ${expected.total}`);
+    }
+    if (expected.orderNumber) {
+      assert.strictEqual(r.orderNumber, expected.orderNumber,
+        `${fixture}: orderNumber '${r.orderNumber}' ≠ '${expected.orderNumber}'`);
+    }
+    if (expected.minConfidence != null) {
+      assert.ok(r.confidence >= expected.minConfidence,
+        `${fixture}: confidence ${r.confidence} < ${expected.minConfidence}`);
+    }
+  });
+}
diff --git a/tests/fixtures/receipts/amazon-shipped.json b/tests/fixtures/receipts/amazon-shipped.json
new file mode 100644
index 0000000..d36c05a
--- /dev/null
+++ b/tests/fixtures/receipts/amazon-shipped.json
@@ -0,0 +1,20 @@
+{
+  "fixture": "amazon-shipped",
+  "expected": {
+    "merchant": "Amazon",
+    "total": 24.99,
+    "orderNumber": "112-7654321-8765432",
+    "minConfidence": 0.7
+  },
+  "summary": {
+    "headers": {
+      "subject": "Your Amazon.com order #112-7654321-8765432 has shipped",
+      "from": "\"Amazon.com\" <ship-confirm@amazon.com>",
+      "date": "2026-04-15T18:32:00Z"
+    },
+    "body": {
+      "text": "Hello,\n\nYour Amazon.com order #112-7654321-8765432 has shipped.\n\nOrder Details:\n- 1× Anker USB-C Charger 65W ............... $24.99\n- Subtotal: $24.99\n- Shipping: $0.00\n- Tax: $0.00\n- Order Total: $24.99\n\nShipping to: [REDACTED]\nEstimated delivery: April 17, 2026\n\nThanks for shopping at Amazon.com.",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/apple.json b/tests/fixtures/receipts/apple.json
new file mode 100644
index 0000000..78640f8
--- /dev/null
+++ b/tests/fixtures/receipts/apple.json
@@ -0,0 +1,19 @@
+{
+  "fixture": "apple",
+  "expected": {
+    "merchant": "Apple",
+    "total": 9.99,
+    "minConfidence": 0.6
+  },
+  "summary": {
+    "headers": {
+      "subject": "Your receipt from Apple.",
+      "from": "\"Apple\" <no_reply@email.apple.com>",
+      "date": "2026-02-10T03:14:00Z"
+    },
+    "body": {
+      "text": "Your receipt from Apple\n\nApple ID: [REDACTED]\nDate: Feb 10, 2026\n\nApp / Service                                                  Price\niCloud+ 200GB Monthly Subscription                              $2.99\nApple TV+ Monthly                                              $9.99\n\nSubtotal: $12.98\nTotal: $9.99\n\nThanks for your purchase!",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/best-buy.json b/tests/fixtures/receipts/best-buy.json
new file mode 100644
index 0000000..a21abdb
--- /dev/null
+++ b/tests/fixtures/receipts/best-buy.json
@@ -0,0 +1,20 @@
+{
+  "fixture": "best-buy",
+  "expected": {
+    "merchant": "Best Buy",
+    "total": 1499.99,
+    "orderNumber": "BBY01-806589123456",
+    "minConfidence": 0.7
+  },
+  "summary": {
+    "headers": {
+      "subject": "Thanks for your order — order #BBY01-806589123456",
+      "from": "\"Best Buy Orders\" <BestBuyInfo@emailinfo.bestbuy.com>",
+      "date": "2026-03-22T14:08:00Z"
+    },
+    "body": {
+      "text": "Thanks for your order!\n\nOrder #: BBY01-806589123456\nOrder Date: March 22, 2026\n\nItems:\n- LG 65\" OLED C3 Series TV — $1,499.99\n\nOrder Total: $1,499.99\n\nYour TV will be delivered between March 26-28.",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/doordash.json b/tests/fixtures/receipts/doordash.json
new file mode 100644
index 0000000..b0576eb
--- /dev/null
+++ b/tests/fixtures/receipts/doordash.json
@@ -0,0 +1,19 @@
+{
+  "fixture": "doordash",
+  "expected": {
+    "merchant": "DoorDash",
+    "total": 32.45,
+    "minConfidence": 0.6
+  },
+  "summary": {
+    "headers": {
+      "subject": "Order Confirmation — DoorDash",
+      "from": "\"DoorDash\" <no-reply@doordash.com>",
+      "date": "2026-04-01T19:42:00Z"
+    },
+    "body": {
+      "text": "Your order is on the way!\n\nFrom: Pizza Palace\n2× Margherita Pizza ............. $24.00\n1× Garlic Knots ................. $4.50\nDelivery fee: $2.99\nTip: $0.96\nTotal: $32.45\n\nThanks for ordering with DoorDash!",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/etsy.json b/tests/fixtures/receipts/etsy.json
new file mode 100644
index 0000000..732b762
--- /dev/null
+++ b/tests/fixtures/receipts/etsy.json
@@ -0,0 +1,19 @@
+{
+  "fixture": "etsy",
+  "expected": {
+    "merchant": "Etsy",
+    "total": 47.50,
+    "minConfidence": 0.6
+  },
+  "summary": {
+    "headers": {
+      "subject": "Thanks for your purchase! Order confirmation from Etsy",
+      "from": "\"Etsy\" <transaction@mail.etsy.com>",
+      "date": "2026-03-30T11:23:00Z"
+    },
+    "body": {
+      "text": "Order confirmation\n\nThank you for your order from HandmadeWorkshop!\n\nItems:\n- Custom Wooden Cutting Board: $42.00\n- Shipping: $5.50\n\nOrder Total: $47.50\n\nEstimated delivery: April 4-7, 2026",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/non-receipt-newsletter.json b/tests/fixtures/receipts/non-receipt-newsletter.json
new file mode 100644
index 0000000..f77e794
--- /dev/null
+++ b/tests/fixtures/receipts/non-receipt-newsletter.json
@@ -0,0 +1,17 @@
+{
+  "fixture": "non-receipt-newsletter",
+  "expected": {
+    "shouldExtract": false
+  },
+  "summary": {
+    "headers": {
+      "subject": "Weekly newsletter from Tech Daily",
+      "from": "\"Tech Daily\" <hello@techdaily.example>",
+      "date": "2026-04-14T08:00:00Z"
+    },
+    "body": {
+      "text": "Hello reader!\n\nThis week in tech:\n- Apple announces new MacBook Pro\n- Google updates Gemini\n- Meta unveils new AR glasses\n\nDon't miss our top stories. Click to read more.\n\nUnsubscribe.",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/uber.json b/tests/fixtures/receipts/uber.json
new file mode 100644
index 0000000..61e635f
--- /dev/null
+++ b/tests/fixtures/receipts/uber.json
@@ -0,0 +1,19 @@
+{
+  "fixture": "uber",
+  "expected": {
+    "merchant": "Uber",
+    "total": 18.42,
+    "minConfidence": 0.5
+  },
+  "summary": {
+    "headers": {
+      "subject": "Your Tuesday afternoon trip with Uber",
+      "from": "\"Uber Receipts\" <noreply@uber.com>",
+      "date": "2026-04-08T16:55:00Z"
+    },
+    "body": {
+      "text": "Thanks for riding, traveler.\n\nYou rode with Marco.\n2.4 mi · 9 min\n\nTrip fare: $14.50\nBooking fee: $2.50\nTip: $1.42\nTotal charged to Visa ••1234: $18.42\n\nWe hope you enjoyed your ride this afternoon.",
+      "html": ""
+    }
+  }
+}
diff --git a/tests/fixtures/receipts/walmart.json b/tests/fixtures/receipts/walmart.json
new file mode 100644
index 0000000..bafc69e
--- /dev/null
+++ b/tests/fixtures/receipts/walmart.json
@@ -0,0 +1,20 @@
+{
+  "fixture": "walmart",
+  "expected": {
+    "merchant": "Walmart",
+    "total": 87.31,
+    "orderNumber": "200012345678901",
+    "minConfidence": 0.7
+  },
+  "summary": {
+    "headers": {
+      "subject": "Your Walmart.com order has been confirmed (#200012345678901)",
+      "from": "\"Walmart\" <help@walmart.com>",
+      "date": "2026-04-12T10:01:00Z"
+    },
+    "body": {
+      "text": "Your order has been confirmed.\n\nOrder #: 200012345678901\nOrder Date: April 12, 2026\n\nItems:\n- Tide Pods 81ct: $19.97\n- Bounty Paper Towels 6pk: $14.97\n- Various groceries: $52.37\n\nSubtotal: $87.31\nOrder Total: $87.31\n\nReady for pickup tomorrow.",
+      "html": ""
+    }
+  }
+}

← fe7ffca tick 17: manual CSV upload route (backlog item 15)  ·  back to AbramsOS  ·  add noreferrer to target=_blank external links 20f8f36 →