`lib/receipt-extractor.js`

293 lines
// Tier 1: heuristic extractor over Gmail message summaries.
// Tier 2: local Ollama qwen3:14b fallback on Mac1 (invoked when tier-1 confidence < 0.7).
// Tier 3: PDF parsing — TODO (will need pdfjs-dist or similar).

const ollama = require('./ollama');
const guardian = require('./compliance-guardian');

const LLM_THRESHOLD = 0.7;          // call LLM when heuristic conf < this
const LLM_BODY_LIMIT = 4000;        // chars sent to the model (≈ 1k tokens)

const MERCHANT_DOMAIN_OVERRIDES = {
  'amazon.com': 'Amazon',
  'amazon.co.uk': 'Amazon',
  'amzn.com': 'Amazon',
  'shopify.com': null, // shopify is a platform; merchant is in subdomain or display name
  'doordash.com': 'DoorDash',
  'uber.com': 'Uber',
  'ubereats.com': 'Uber Eats',
  'lyft.com': 'Lyft',
  'apple.com': 'Apple',
  'paypal.com': 'PayPal',
  'stripe.com': null,
  'squareup.com': null,
  'ebay.com': 'eBay',
  'etsy.com': 'Etsy',
  'walmart.com': 'Walmart',
  'target.com': 'Target',
  'bestbuy.com': 'Best Buy',
  'homedepot.com': 'Home Depot',
  'lowes.com': "Lowe's",
};

function parseSenderEmail(fromHeader) {
  if (!fromHeader) return { displayName: null, email: null, domain: null };
  const m = fromHeader.match(/(?:"?([^"<]*)"?\s*)?<?([^\s<>]+@[^\s<>]+)>?/);
  if (!m) return { displayName: fromHeader.trim(), email: null, domain: null };
  const email = m[2].toLowerCase();
  const domain = email.split('@')[1] || null;
  const displayName = (m[1] || '').trim() || null;
  return { displayName, email, domain };
}

function rootDomain(domain) {
  if (!domain) return null;
  const parts = domain.split('.');
  if (parts.length <= 2) return domain;
  return parts.slice(-2).join('.');
}

function inferMerchant({ from, subject }) {
  const { displayName, domain } = parseSenderEmail(from);
  const root = rootDomain(domain);
  if (root && Object.prototype.hasOwnProperty.call(MERCHANT_DOMAIN_OVERRIDES, root)) {
    const override = MERCHANT_DOMAIN_OVERRIDES[root];
    if (override) return { name: override, domain: root, source: 'override' };
  }
  if (displayName && displayName.length > 1 && !displayName.includes('@')) {
    const cleaned = displayName.replace(/\b(team|orders?|receipts?|sales|noreply|no-reply|info|support|hello)\b/gi, '').replace(/[<>"']/g, '').trim();
    if (cleaned) return { name: cleaned, domain: root, source: 'display_name' };
  }
  if (root) {
    const stem = root.split('.')[0];
    return { name: stem.charAt(0).toUpperCase() + stem.slice(1), domain: root, source: 'domain_stem' };
  }
  if (subject) {
    const m = subject.match(/from\s+([A-Z][\w& ]{2,40})/);
    if (m) return { name: m[1].trim(), domain: null, source: 'subject' };
  }
  return { name: 'Unknown', domain: null, source: 'fallback' };
}

const CURRENCY_SYMBOLS = { '$': 'USD', '€': 'EUR', '£': 'GBP', '¥': 'JPY' };

function extractTotal(text) {
  if (!text) return { amount: null, currency: null };
  // Find ALL money-prefixed-by-total matches; rank by how "final" the label is.
  // "order total" / "grand total" / "total charged" / "amount charged" beat plain "total"
  // (which beats "subtotal", "trip fare", "subtotal amount", etc. that we explicitly REJECT).
  // Allow ≤40 non-newline non-money chars between label and money (e.g. "Total charged to Visa ••1234: $18.42")
  const re = /(order\s+total|grand\s+total|total\s+charged|total\s+amount|amount\s+charged|total)[^\n$£€¥]{0,40}([£€¥$])\s*([0-9]{1,3}(?:[,0-9]{0,3})*(?:\.[0-9]{2})?)/gi;
  const labelRank = (lbl) => {
    const l = lbl.toLowerCase();
    if (l.includes('order total') || l.includes('grand total')) return 5;
    if (l.includes('total charged') || l.includes('amount charged')) return 4;
    if (l.includes('total amount')) return 3;
    return 2; // plain "total"
  };
  const matches = [];
  for (const m of text.matchAll(re)) {
    // Reject when this match is part of "subtotal" — overlap check
    const beforeStart = Math.max(0, m.index - 3);
    if (text.slice(beforeStart, m.index).toLowerCase() === 'sub') continue;
    matches.push({
      label: m[1], symbol: m[2],
      amount: parseFloat(m[3].replace(/,/g, '')),
      rank: labelRank(m[1]),
      idx: m.index,
    });
  }
  if (matches.length) {
    // Highest rank wins; tie → latest in text (totals are usually at the bottom)
    matches.sort((a, b) => b.rank - a.rank || b.idx - a.idx);
    const best = matches[0];
    if (!isNaN(best.amount)) return { amount: best.amount, currency: CURRENCY_SYMBOLS[best.symbol] || 'USD' };
  }
  // Fallback: any money value
  const fallback = text.match(/([£€¥$])\s*([0-9]{1,3}(?:[,0-9]{0,3})*\.[0-9]{2})/);
  if (fallback) {
    const symbol = fallback[1];
    const value = parseFloat(fallback[2].replace(/,/g, ''));
    if (!isNaN(value)) return { amount: value, currency: CURRENCY_SYMBOLS[symbol] || 'USD' };
  }
  return { amount: null, currency: null };
}

function extractOrderNumber(text) {
  if (!text) return null;
  const patterns = [
    // Amazon-style hyphenated
    /order\s*#?\s*([0-9]{3}-[0-9]{6,8}-[0-9]{6,8})/i,
    // Best Buy / merchant-prefix style: "Order #: BBY01-806589123456"
    /order\s*(?:number|#|id)?\s*[:#]\s*([A-Z]{2,5}\d{0,3}[-_]?\d{6,})/i,
    // Generic alphanumeric with hyphen
    /order\s*(?:number|#|id)?\s*[:#]?\s*([A-Z0-9]{3,4}-?\d{3,}-?[A-Z0-9]+)/i,
    // Long digit-only
    /order\s*(?:number|#|id)?\s*[:#]?\s*(#?\d{6,})/i,
    /confirmation\s*(?:number|#|code)?\s*[:#]?\s*([A-Z0-9-]{6,})/i,
    /invoice\s*(?:number|#)?\s*[:#]?\s*([A-Z0-9-]{4,})/i,
  ];
  for (const re of patterns) {
    const m = text.match(re);
    if (m) return m[1].replace(/^#/, '').trim();
  }
  return null;
}

function stripHtml(html) {
  if (!html) return '';
  return html
    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/\s+/g, ' ')
    .trim();
}

/**
 * Extract a purchase from a Gmail message summary (the output of gmail-fetcher.summarize()).
 * Returns { merchant, merchantDomain, orderNumber, total, currency, purchaseDate, confidence, items, source: 'heuristic' }
 * — or null if the message clearly isn't a receipt.
 */
function extract(summary) {
  const subject = summary.headers?.subject || '';
  const from = summary.headers?.from || '';
  const dateHeader = summary.headers?.date || null;
  const purchaseDate = dateHeader ? new Date(dateHeader) : new Date();

  const haystack = [subject, summary.body?.text || '', stripHtml(summary.body?.html || '')].filter(Boolean).join('\n');

  // Quick reject: clearly not a receipt
  const lowerSubj = subject.toLowerCase();
  if (/unsubscribe|newsletter|account\s+statement\s+is\s+ready|password\s+reset|verify\s+your\s+email/.test(lowerSubj)) {
    return null;
  }

  const { name: merchantName, domain: merchantDomain } = inferMerchant({ from, subject });
  const { amount: total, currency } = extractTotal(haystack);
  const orderNumber = extractOrderNumber(haystack);

  // Confidence
  let confidence = 0.0;
  if (total != null) confidence += 0.4;
  if (orderNumber) confidence += 0.25;
  if (/receipt|order\s+(?:confirmation|placed|shipped|delivered)|invoice|thank\s+you\s+for\s+your\s+order|thanks\s+for\s+your\s+order|purchase\s+confirmation/i.test(haystack)) confidence += 0.25;
  if (merchantName && merchantName !== 'Unknown') confidence += 0.1;
  confidence = Math.min(1, Math.round(confidence * 100) / 100);

  if (confidence < 0.3) return null; // probably not a receipt

  return {
    merchant: merchantName,
    merchantDomain,
    orderNumber,
    total,
    currency: currency || 'USD',
    purchaseDate,
    confidence,
    items: [],
    source: 'heuristic',
  };
}

/**
 * Tier-2 LLM enrichment. Sends the message text to local Ollama qwen3:14b
 * with a strict-JSON prompt; merges the LLM's answers into the heuristic
 * result for any fields the heuristic was unsure about. NEVER replaces a
 * field the heuristic already filled with high confidence — LLM output is
 * untrusted by default.
 *
 * @param {object} summary  output of gmail-fetcher.summarize()
 * @param {object} heur     output of extract(summary); MUST be non-null
 * @param {object} opts     { force?: bool, model?: string, base?: string, timeoutMs?: number }
 * @returns enriched purchase or the original heur (unchanged) on LLM failure
 */
async function enrichWithLlm(summary, heur, opts = {}) {
  if (!heur) return null;
  if (!opts.force && heur.confidence >= LLM_THRESHOLD) return heur;

  const subject = summary.headers?.subject || '';
  const from = summary.headers?.from || '';
  const rawText = (summary.body?.text || stripHtml(summary.body?.html || '')).slice(0, LLM_BODY_LIMIT);

  // Compliance Guardian: redact PII + scan for prompt-injection BEFORE feeding to LLM.
  // If injection patterns detected in the source, refuse to enrich and return heuristic
  // unchanged (with a guardian flag so caller can route to manual review later).
  const guarded = guardian.guardForLlm(rawText);
  if (!guarded.safe) {
    return {
      ...heur,
      source: 'heuristic',
      llmError: 'compliance_guardian_blocked',
      injectionPatterns: guarded.injection.patterns,
    };
  }
  const text = guarded.text;
  const fromGuarded = guardian.redactPii(from).redacted;
  const subjectGuarded = guardian.redactPii(subject).redacted;

  const prompt = `You are a receipt extraction tool. Extract the structured purchase from this email and return ONLY a JSON object — no prose, no code fences. Use null for fields you cannot determine. Schema:
{
  "merchant": string,        // brand the user bought from (e.g. "Amazon", not "noreply@amazon.com")
  "order_number": string|null,
  "total": number|null,      // total amount paid (USD or local currency, just the number)
  "currency": string|null,   // ISO 4217 (USD, EUR, GBP, etc.)
  "items": [{ "description": string, "quantity": number, "unit_price": number|null }]
}

Email follows.

From: ${fromGuarded}
Subject: ${subjectGuarded}

${text}`;

  let llm;
  try {
    llm = await ollama.generateJson(prompt, {
      model: opts.model,
      base: opts.base,
      timeoutMs: opts.timeoutMs || 30_000,
    });
  } catch (err) {
    // LLM unavailable / timeout / non-JSON: return heuristic unchanged.
    return { ...heur, source: 'heuristic', llmError: err.message };
  }

  // Merge: only fill what heuristic missed; never override existing values.
  const merged = { ...heur };
  if (!merged.merchant || merged.merchant === 'Unknown') merged.merchant = llm.merchant || merged.merchant;
  if (!merged.orderNumber && llm.order_number) merged.orderNumber = String(llm.order_number);
  if (merged.total == null && typeof llm.total === 'number') merged.total = llm.total;
  if (!merged.currency && llm.currency) merged.currency = String(llm.currency).toUpperCase();
  if ((!merged.items || !merged.items.length) && Array.isArray(llm.items)) merged.items = llm.items;

  // Recompute confidence with LLM-supplied fields counted in
  let conf = 0;
  if (merged.total != null) conf += 0.4;
  if (merged.orderNumber) conf += 0.25;
  if (merged.merchant && merged.merchant !== 'Unknown') conf += 0.1;
  if (merged.items?.length) conf += 0.15;
  conf += 0.1; // small bonus for LLM having seen the full body
  merged.confidence = Math.min(1, Math.round(conf * 100) / 100);
  merged.source = 'heuristic+llm';
  return merged;
}

/**
 * Convenience: tier-1 then optional tier-2.
 *   extractWithFallback(summary, { llm: true })
 */
async function extractWithFallback(summary, opts = {}) {
  const t1 = extract(summary);
  if (!t1) return null;
  if (opts.llm === false) return t1;
  return enrichWithLlm(summary, t1, opts);
}

module.exports = { extract, enrichWithLlm, extractWithFallback, inferMerchant, extractTotal, extractOrderNumber, stripHtml, LLM_THRESHOLD };