← back to Rentv Adintel
src/connectors/email-upload.js
380 lines
'use strict';
/**
* Email/EML upload parser for sponsor evidence — spec §16.
*
* Deterministic parsing using only Node.js built-ins.
*
* Supported:
* .eml / raw email string → parseEml(text) → EmlResult
* HTML string → parseHtml(text) → HtmlResult
*
* Unsupported (stubs):
* .msg → { unsupported: true, note: 'PDF/.msg parsing needs a parser...' }
* .pdf → { unsupported: true, note: 'PDF/.msg parsing needs a parser...' }
*
* Optional LLM extraction hook:
* When OLLAMA_ENABLED=true, the extractWithOllama() function is the
* intended extension point. It is NOT implemented here — add it behind
* the feature flag and subject results to human review before use.
*
* Design principles:
* - Deterministic first: regex/header parsing with no external deps
* - Never auto-open email tracking links (store the URL, flag it)
* - EXIF stripping is noted but not implemented here (needs sips/sharp)
* - No personal mobile numbers, home addresses, or sensitive attributes
*
* @module src/connectors/email-upload
*/
// ---------------------------------------------------------------------------
// URL extraction (avoid opening tracking links)
// ---------------------------------------------------------------------------
/**
* Extract all http/https URLs from text.
* Tracking links (click.*, trk., ccsend.com, etc.) are flagged but NOT opened.
*
* @param {string} text
* @returns {Array<{ url: string, likelyTracking: boolean }>}
*/
function extractUrls(text) {
const URL_RE = /https?:\/\/[^\s"'<>)]+/g;
const TRACKING_PATTERNS = [
/click\./i, /trk\./i, /track\./i, /ccsend\.com/i,
/mailchimp\.com\/track/i, /list-manage\.com/i,
/sendgrid\.net/i, /exacttarget\.com/i, /r\.exactdn\.com/i,
/click\.email/i, /em\.rentv/i,
];
const found = [];
const seen = new Set();
let match;
while ((match = URL_RE.exec(text)) !== null) {
const url = match[0].replace(/[,;.]+$/, ''); // strip trailing punctuation
if (seen.has(url)) continue;
seen.add(url);
const likelyTracking = TRACKING_PATTERNS.some((p) => p.test(url));
found.push({ url, likelyTracking });
}
return found;
}
// ---------------------------------------------------------------------------
// Organization name extraction (naive NER — deterministic)
// ---------------------------------------------------------------------------
/**
* Extract likely organization names from text.
* Uses a heuristic: sequences of capitalized words followed by common
* CRE organization suffixes.
*
* @param {string} text
* @returns {string[]}
*/
function extractOrganizationNames(text) {
// Match sequences like "Hanley Investment Group" or "Chase Partners LLC"
const ORG_RE = /\b([A-Z][a-zA-Z&''-]+(?:\s+[A-Z][a-zA-Z&''-]+){0,5}(?:\s+(?:Group|Partners|Capital|Realty|Properties|Investment|Development|Investments|Lenders|Bank|Savings|Mortgage|Title|Escrow|Fund|Advisors|Advisory|Associates|Corp|LLC|LP|LLP|Inc|Co|Company|Real Estate|Brokerage|Commercial))\b)/g;
const names = new Set();
let m;
while ((m = ORG_RE.exec(text)) !== null) {
names.add(m[1].trim());
}
return Array.from(names);
}
// ---------------------------------------------------------------------------
// Sponsor label extraction
// ---------------------------------------------------------------------------
/**
* Extract visible sponsor labels: lines/phrases matching typical sponsor
* disclosure patterns in CRE newsletters/emails.
*
* @param {string} text
* @returns {string[]}
*/
function extractSponsorLabels(text) {
const patterns = [
/(?:sponsored\s+by|sponsor(?:ed)?\s*[:–—]\s*)([^\n\r<]{1,80})/gi,
/(?:presented\s+by|presented\s+to\s+you\s+by)\s+([^\n\r<]{1,80})/gi,
/(?:property\s+spotlight\s*(?:[:–—]\s*)?)([^\n\r<]{1,80})/gi,
/(?:cre\s+talk\s+(?:sponsor|brought\s+to\s+you\s+by)\s*[:–—]?\s*)([^\n\r<]{1,80})/gi,
/(?:thank\s+you\s+to\s+our\s+(?:sponsor|partner)s?\s*[:–—]?\s*)([^\n\r<]{1,80})/gi,
];
const labels = new Set();
for (const re of patterns) {
let m;
while ((m = re.exec(text)) !== null) {
const label = m[0].trim();
if (label.length > 3) labels.add(label);
}
re.lastIndex = 0;
}
return Array.from(labels);
}
// ---------------------------------------------------------------------------
// EML header parser
// ---------------------------------------------------------------------------
/**
* Parse RFC 2822 headers from an EML string.
* Returns a plain object of { header-name-lowercase: value }.
* Handles folded headers (continuation lines starting with whitespace).
*
* @param {string} text
* @returns {{ headers: Object.<string,string>, bodyStart: number }}
*/
function parseRfc2822Headers(text) {
const headerBodySep = text.indexOf('\r\n\r\n');
const sepLen = 4;
const altSep = text.indexOf('\n\n');
const sepIdx = headerBodySep !== -1 ? headerBodySep : altSep;
const headerSection = sepIdx !== -1 ? text.slice(0, sepIdx) : text;
const bodyStart = sepIdx !== -1 ? sepIdx + (headerBodySep !== -1 ? sepLen : 2) : text.length;
// Unfold: join continuation lines
const unfolded = headerSection.replace(/\r\n([ \t])/g, ' ').replace(/\n([ \t])/g, ' ');
const lines = unfolded.split(/\r?\n/);
const headers = {};
for (const line of lines) {
const colon = line.indexOf(':');
if (colon < 1) continue;
const name = line.slice(0, colon).trim().toLowerCase();
const value = line.slice(colon + 1).trim();
if (name && !headers[name]) headers[name] = value;
}
return { headers, bodyStart };
}
/**
* Decode a base64 or quoted-printable encoded string to UTF-8.
* Falls back to returning the raw text.
*
* @param {string} text
* @param {string} encoding - 'base64' | 'quoted-printable' | ''
* @returns {string}
*/
function decodeBody(text, encoding) {
const enc = (encoding || '').toLowerCase().trim();
if (enc === 'base64') {
try { return Buffer.from(text.replace(/\s+/g, ''), 'base64').toString('utf8'); } catch { return text; }
}
if (enc === 'quoted-printable') {
return text
.replace(/=\r?\n/g, '')
.replace(/=([0-9A-Fa-f]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
}
return text;
}
/**
* Strip HTML tags from a string, preserving whitespace structure.
* @param {string} html
* @returns {string}
*/
function stripHtml(html) {
return html
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, ' ')
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, ' ')
.replace(/<br\s*\/?>/gi, '\n')
.replace(/<\/p>/gi, '\n')
.replace(/<[^>]+>/g, ' ')
.replace(/ /gi, ' ')
.replace(/&/gi, '&')
.replace(/</gi, '<')
.replace(/>/gi, '>')
.replace(/"/gi, '"')
.replace(/'/gi, "'")
.replace(/[ \t]+/g, ' ')
.trim();
}
// ---------------------------------------------------------------------------
// Public: parseEml
// ---------------------------------------------------------------------------
/**
* Parse an EML string into a structured evidence object.
*
* @param {string} emlText - raw .eml file contents
* @returns {{
* from: string,
* to: string,
* subject: string,
* date: string,
* messageId: string,
* urls: Array<{ url: string, likelyTracking: boolean }>,
* organizationNames: string[],
* visibleSponsorLabels: string[],
* plainText: string,
* parserVersion: string,
* parseWarnings: string[]
* }}
*/
function parseEml(emlText) {
if (!emlText || typeof emlText !== 'string') {
throw new TypeError('parseEml: input must be a non-empty string');
}
const warnings = [];
const { headers, bodyStart } = parseRfc2822Headers(emlText);
const rawBody = emlText.slice(bodyStart);
// Detect content-type and transfer-encoding for the top-level body
const contentType = headers['content-type'] || 'text/plain';
const transferEncoding = headers['content-transfer-encoding'] || '';
// Find a usable text body — handles simple single-part and basic multipart
let textBody = '';
if (/multipart/i.test(contentType)) {
const boundaryMatch = contentType.match(/boundary\s*=\s*"?([^";\s]+)"?/i);
if (boundaryMatch) {
const boundary = boundaryMatch[1];
const parts = rawBody.split(new RegExp('--' + boundary.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
for (const part of parts) {
if (!part.trim() || part.trim() === '--') continue;
const partSep = part.indexOf('\r\n\r\n') !== -1 ? part.indexOf('\r\n\r\n') + 4 : (part.indexOf('\n\n') + 2);
const partHeaders = part.slice(0, partSep).toLowerCase();
const partBody = part.slice(partSep);
const partCte = (partHeaders.match(/content-transfer-encoding:\s*(\S+)/) || [])[1] || '';
if (/text\/plain/.test(partHeaders)) {
textBody = decodeBody(partBody, partCte);
break;
} else if (/text\/html/.test(partHeaders) && !textBody) {
textBody = stripHtml(decodeBody(partBody, partCte));
}
}
} else {
warnings.push('multipart boundary not found — falling back to raw body');
textBody = rawBody;
}
} else if (/text\/html/i.test(contentType)) {
textBody = stripHtml(decodeBody(rawBody, transferEncoding));
} else {
textBody = decodeBody(rawBody, transferEncoding);
}
if (!textBody) {
warnings.push('empty body after decoding');
textBody = '';
}
const urls = extractUrls(emlText); // search raw EML so we catch headers too
const organizationNames = extractOrganizationNames(textBody);
const visibleSponsorLabels = extractSponsorLabels(textBody);
return {
from: headers['from'] || '',
to: headers['to'] || '',
subject: headers['subject'] || '',
date: headers['date'] || '',
messageId: headers['message-id'] || '',
urls,
organizationNames,
visibleSponsorLabels,
plainText: textBody.slice(0, 4000), // truncate for evidence excerpt
parserVersion: 'email-upload/1.0',
parseWarnings: warnings,
};
}
// ---------------------------------------------------------------------------
// Public: parseHtml
// ---------------------------------------------------------------------------
/**
* Parse an HTML email/newsletter fragment for sponsor evidence.
*
* @param {string} htmlText
* @returns {{
* urls: Array<{ url: string, likelyTracking: boolean }>,
* organizationNames: string[],
* visibleSponsorLabels: string[],
* plainText: string,
* parserVersion: string
* }}
*/
function parseHtml(htmlText) {
if (!htmlText || typeof htmlText !== 'string') {
throw new TypeError('parseHtml: input must be a non-empty string');
}
const plainText = stripHtml(htmlText);
return {
urls: extractUrls(htmlText),
organizationNames: extractOrganizationNames(plainText),
visibleSponsorLabels: extractSponsorLabels(plainText),
plainText: plainText.slice(0, 4000),
parserVersion: 'email-upload/1.0',
};
}
// ---------------------------------------------------------------------------
// Public: unsupported format stubs
// ---------------------------------------------------------------------------
/**
* Stub for PDF and .msg files.
* Returns a standardised unsupported marker so callers can surface a
* human-review request rather than failing silently.
*
* @returns {{ unsupported: true, note: string }}
*/
function parseMsg() {
return {
unsupported: true,
note: 'PDF/.msg parsing needs a parser — manual upload of extracted text for now. ' +
'For PDF, use pdftotext (poppler) or pdf-parse (npm). ' +
'For .msg (Outlook), use msg-reader or @kenjiuno/msgreader. ' +
'Feed the extracted plain text back through parseHtml() or parseEml().',
};
}
const parsePdf = parseMsg; // same note
// ---------------------------------------------------------------------------
// Optional LLM extraction hook (not implemented)
// ---------------------------------------------------------------------------
/**
* Placeholder for optional Ollama-backed LLM extraction.
*
* When OLLAMA_ENABLED=true and OLLAMA_BASE_URL is set, this function should:
* 1. POST the plainText excerpt to the Ollama /api/generate endpoint.
* 2. Use a strict JSON schema prompt requesting { sponsors, advertisers, urls, events }.
* 3. Validate the returned JSON against the schema.
* 4. Return the structured extraction with a confidence score.
* 5. Flag results as REQUIRES_HUMAN_REVIEW before persisting.
*
* Model: process.env.OLLAMA_MODEL (default: qwen2.5:7b) — $0 cost (local).
*
* NOT IMPLEMENTED: add it here when Steve enables OLLAMA_ENABLED=true.
*
* @param {string} _plainText
* @throws {Error} always — not yet implemented
*/
async function extractWithOllama(_plainText) {
if (process.env.OLLAMA_ENABLED !== 'true') {
throw new Error('OLLAMA_ENABLED is not set to true — LLM extraction is disabled');
}
throw new Error('Ollama extraction not yet implemented — add it in email-upload.js extractWithOllama()');
}
module.exports = {
parseEml,
parseHtml,
parseMsg,
parsePdf,
extractUrls,
extractOrganizationNames,
extractSponsorLabels,
extractWithOllama,
};