← back to Commercialrealestate
scripts/email-alerts.js
328 lines
#!/usr/bin/env node
/*
* CRCP email-alerts — DTD verdict A (launchd snapshot-diff + George).
*
* Diffs the current snapshots (listings.json, brokers-snapshot.json) against the
* last-seen state, finds what's NEW per metro, and composes a per-subscriber HTML
* digest. DB-less by design — pure snapshot files, exactly like the rest of CRCP.
*
* Cadence is adjustable at the config level (data/alerts-config.json `cadence`) and
* at the schedule level (the launchd plist interval). Sends are GATED: dry-run by
* default (writes HTML to data/alerts-out/); --send routes through George and is the
* only path that advances the seen-state.
*
* Usage:
* node scripts/email-alerts.js # dry-run: build digests, DO NOT send, DO NOT advance state
* node scripts/email-alerts.js --send # GATED: send via George + advance state
* node scripts/email-alerts.js --commit-state # advance state without sending (baseline seed)
*/
const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');
const ROOT = path.join(__dirname, '..');
const DATA = path.join(ROOT, 'data');
const OUT = path.join(DATA, 'alerts-out');
const CONFIG_PATH = path.join(DATA, 'alerts-config.json');
const STATE_PATH = path.join(DATA, 'alerts-state.json');
const SEND = process.argv.includes('--send');
const COMMIT_STATE = process.argv.includes('--commit-state') || SEND;
// ---- metro tagging -------------------------------------------------------
// A listing/broker belongs to a metro if its city (slugified) matches the metro
// slug or one of its alias cities. Subscribers with metros:["*"] get everything.
const METRO_CITIES = {
'los-angeles': ['los angeles', 'la', 'hollywood', 'west hollywood', 'downtown', 'dtla', 'silver lake', 'echo park', 'koreatown', 'mid-city', 'venice', 'santa monica', 'culver city', 'pasadena', 'long beach', 'burbank', 'glendale', 'sherman oaks', 'studio city', 'north hollywood', 'van nuys', 'encino', 'woodland hills', 'tarzana', 'inglewood', 'san pedro', 'torrance', 'gardena', 'huntington park', 'bellflower', 'lakewood', 'downey', 'norwalk', 'compton', 'carson', 'hawthorne', 'lynwood', 'south gate', 'whittier', 'montebello', 'alhambra', 'monterey park', 'el monte', 'pomona', 'lancaster', 'palmdale', 'santa clarita', 'redondo beach', 'manhattan beach', 'hermosa beach', 'westchester', 'reseda', 'canoga park', 'northridge', 'chatsworth', 'sun valley', 'panorama city', 'sylmar', 'wilmington', 'harbor city', 'san fernando', 'arcadia', 'temple city', 'rosemead', 'baldwin park', 'covina', 'west covina', 'la puente', 'hacienda heights', 'rowland heights', 'diamond bar', 'walnut', 'cerritos', 'artesia', 'paramount', 'bell', 'bell gardens', 'maywood', 'cudahy', 'commerce', 'vernon', 'san gabriel', 'altadena', 'sierra madre', 'la crescenta', 'tujunga', 'eagle rock', 'highland park', 'boyle heights', 'lincoln heights', 'south pasadena', 'glassell park'],
'beverly-hills': ['beverly hills'],
'malibu': ['malibu'],
'newport-beach': ['newport beach', 'newport coast', 'corona del mar'],
'montecito': ['montecito', 'santa barbara'],
'san-francisco': ['san francisco', 'sf'],
'silicon-valley': ['palo alto', 'menlo park', 'mountain view', 'los altos', 'los altos hills', 'sunnyvale', 'cupertino', 'san jose'],
'atherton': ['atherton'],
'hillsborough': ['hillsborough'],
'woodside': ['woodside'],
'miami': ['miami', 'coral gables', 'coconut grove', 'brickell'],
'miami-beach': ['miami beach', 'south beach'],
'palm-beach': ['palm beach', 'west palm beach'],
'naples': ['naples'],
'nyc': ['new york', 'nyc', 'manhattan', 'brooklyn'],
'greenwich': ['greenwich'],
'hamptons': ['east hampton', 'southampton', 'bridgehampton', 'sag harbor', 'montauk', 'water mill'],
'las-vegas': ['las vegas', 'henderson', 'summerlin'],
'texas': ['austin', 'dallas', 'houston', 'san antonio', 'fort worth'],
'aspen': ['aspen', 'snowmass'],
'vail': ['vail', 'beaver creek'],
'telluride': ['telluride'],
};
const slug = s => (s || '').toString().toLowerCase().trim();
function metroOf(city) {
const c = slug(city);
if (!c) return null;
for (const [metro, cities] of Object.entries(METRO_CITIES)) {
if (c === metro.replace(/-/g, ' ')) return metro;
if (cities.some(x => c === x || c.includes(x))) return metro;
}
return null;
}
// ---- data loads ----------------------------------------------------------
function loadJSON(p, fallback) { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return fallback; } }
function listValues(d) { if (Array.isArray(d)) return d; if (d && typeof d === 'object') { return d.listings || d.data || d.brokers || Object.values(d); } return []; }
function loadListings() {
const raw = loadJSON(path.join(DATA, 'listings.json'), {});
return listValues(raw).filter(x => x && (x.id != null)).map(x => ({
id: String(x.id), address: x.address, city: x.city, price: x.price, type: x.type,
cap_rate: x.cap_rate, units: x.units, status: x.status, upside_note: x.upside_note,
metro: metroOf(x.city),
}));
}
function loadBrokers() {
const raw = loadJSON(path.join(DATA, 'brokers-snapshot.json'), {});
const arr = raw.brokers || listValues(raw);
return arr.filter(x => x && x.id != null).map(x => ({
id: String(x.id), name: x.name, firm: x.firm, agent_type: x.agent_type,
city: x.office_addr, metro: metroOf(x.office_addr) || metroOf(x.firm) || null,
}));
}
// ---- config + state ------------------------------------------------------
function loadConfig() {
if (fs.existsSync(CONFIG_PATH)) return loadJSON(CONFIG_PATH, null);
// seed a sensible default (Steve gets everything, daily)
const seed = {
from: 'steve@designerwallcoverings.com',
subscribers: [
{ email: 'steve@designerwallcoverings.com', name: 'Steve', metros: ['*'], segments: ['listings', 'brokers'], cadence: 'daily' },
],
note: 'cadence is adjustable per-subscriber (daily|weekly|instant); the launchd plist sets how often this script runs.',
};
fs.writeFileSync(CONFIG_PATH, JSON.stringify(seed, null, 2));
console.log(`seeded default config → ${CONFIG_PATH}`);
return seed;
}
// Per-subscriber state (fixes the cadence contract: each subscriber has their own
// seen-watermark + last_sent, so a weekly subscriber isn't spammed daily AND sees
// items added since THEIR last send — not since some other subscriber's send).
const GOV_STATE_PATH = path.join(DATA, 'gov-seen-state.json'); // sidecar: gov license IDs kept OUT of the main state file so it can't balloon
function loadState() {
const s = loadJSON(STATE_PATH, { last_run: null, subs: {} });
s.subs = s.subs || {};
// one-time migration: legacy global seen.{listings,brokers} → seed nothing extra; subs seed lazily from it
s._legacySeen = s.seen && (s.seen.listings || s.seen.brokers) ? { listings: new Set(s.seen.listings || []), brokers: new Set(s.seen.brokers || []) } : null;
delete s.seen;
return s;
}
function subState(state, email) {
if (!state.subs[email]) {
// seed a new subscriber's watermark from the legacy global baseline (so we don't blast the whole catalog on first run)
state.subs[email] = {
last_sent: null,
seen_listings: state._legacySeen ? [...state._legacySeen.listings] : [],
seen_brokers: state._legacySeen ? [...state._legacySeen.brokers] : [],
};
}
return state.subs[email];
}
// cadence → min interval before the next send is "due"
const CADENCE_MS = { instant: 0, daily: 20 * 3600e3, weekly: 6.5 * 24 * 3600e3, monthly: 27 * 24 * 3600e3 };
function isDue(sub, ss, nowMs) {
if (!ss.last_sent) return true;
const gap = CADENCE_MS[sub.cadence] ?? CADENCE_MS.daily;
return (nowMs - Date.parse(ss.last_sent)) >= gap;
}
function loadGovSeen() { return loadJSON(GOV_STATE_PATH, {}); } // { email: { metro: [license_numbers] } }
// Gov-agent segment (DTD verdict C, opt-in): read data/gov/<metro>.json for the
// metros any opt-in subscriber wants, return the NEW licensees vs seen.gov[metro].
function loadGovForMetros(metros) {
const out = {}; // metro -> [{name, firm, license_number, license_type, city}]
for (const m of metros) {
const p = path.join(DATA, 'gov', `${m}.json`);
if (!fs.existsSync(p)) continue;
const arr = loadJSON(p, []);
if (Array.isArray(arr)) out[m] = arr.filter(a => a && a.license_number).map(a => ({
name: a.name, firm: a.firm, license_number: String(a.license_number), license_type: a.license_type, city: a.city,
}));
}
return out;
}
// ---- digest --------------------------------------------------------------
function money(n) { const v = Number(n); return isFinite(v) && v > 0 ? '$' + v.toLocaleString() : '—'; }
function metroLabel(m) { return (m || 'other').replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase()); }
function buildDigestHTML(sub, newListings, newBrokers, govNewByMetro) {
const wants = new Set(sub.metros);
const all = wants.has('*');
const fL = newListings.filter(l => all || wants.has(l.metro));
const fB = newBrokers.filter(b => all || wants.has(b.metro));
const showListings = sub.segments.includes('listings') ? fL : [];
const showBrokers = sub.segments.includes('brokers') ? fB : [];
// opt-in gov-agent segment (DTD verdict C)
const showGov = sub.segments.includes('gov-agents')
? Object.entries(govNewByMetro || {}).filter(([m]) => all || wants.has(m)).flatMap(([m, arr]) => arr.map(a => ({ ...a, metro: m })))
: [];
const byMetro = arr => arr.reduce((m, x) => { (m[x.metro || 'other'] ||= []).push(x); return m; }, {});
const lMetros = byMetro(showListings);
let body = `<div style="font-family:-apple-system,Segoe UI,Arial,sans-serif;max-width:680px;margin:0 auto;color:#1a1f2b">
<div style="background:#0a0d13;padding:22px 26px;border-radius:12px 12px 0 0">
<div style="color:#C8A24B;font-weight:800;letter-spacing:3px;font-size:13px;text-transform:uppercase">CRCP · Commercial Real Estate Command Post</div>
<div style="color:#EAECEF;font-size:24px;font-weight:800;margin-top:6px">New activity in your markets</div>
<div style="color:#8A93A2;font-size:13px;margin-top:4px">${new Date().toLocaleDateString(undefined,{weekday:'long',month:'long',day:'numeric',year:'numeric'})}</div>
</div>
<div style="border:1px solid #e6e8ec;border-top:none;border-radius:0 0 12px 12px;padding:24px 26px">`;
if (!showListings.length && !showBrokers.length && !showGov.length) {
body += `<p style="color:#6b7280">No new activity in your markets since the last update.</p>`;
}
if (showListings.length) {
body += `<h2 style="font-size:16px;margin:0 0 12px">🏢 ${showListings.length} new listing${showListings.length>1?'s':''}</h2>`;
for (const [metro, items] of Object.entries(lMetros)) {
body += `<div style="font-size:12px;font-weight:700;color:#C8A24B;text-transform:uppercase;letter-spacing:1px;margin:14px 0 6px">${metroLabel(metro)}</div>`;
for (const l of items.slice(0, 25)) {
body += `<div style="padding:10px 12px;border:1px solid #eef0f3;border-radius:8px;margin-bottom:8px">
<div style="font-weight:700">${l.address || 'Address on file'} <span style="color:#8A93A2;font-weight:400">· ${l.city || ''}</span></div>
<div style="font-size:13px;color:#4b5563;margin-top:3px">${money(l.price)}${l.type?` · ${l.type}`:''}${l.units?` · ${l.units} units`:''}${l.cap_rate?` · ${l.cap_rate}% cap`:''}</div>
${l.upside_note?`<div style="font-size:12px;color:#6b7280;margin-top:3px">${l.upside_note}</div>`:''}
</div>`;
}
}
}
if (showBrokers.length) {
body += `<h2 style="font-size:16px;margin:20px 0 12px">👤 ${showBrokers.length} new broker${showBrokers.length>1?'s':''} indexed</h2>`;
for (const b of showBrokers.slice(0, 40)) {
body += `<div style="font-size:13px;padding:6px 0;border-bottom:1px solid #f2f4f6">
<b>${b.name || 'Unknown'}</b>${b.firm?` · ${b.firm}`:''}${b.agent_type?` <span style="color:#8A93A2">(${b.agent_type})</span>`:''}</div>`;
}
}
if (showGov.length) {
const govByMetro = showGov.reduce((m, x) => { (m[x.metro || 'other'] ||= []).push(x); return m; }, {});
body += `<h2 style="font-size:16px;margin:20px 0 12px">🪪 ${showGov.length} newly-licensed agent${showGov.length>1?'s':''}</h2>`;
for (const [metro, items] of Object.entries(govByMetro)) {
body += `<div style="font-size:12px;font-weight:700;color:#C8A24B;text-transform:uppercase;letter-spacing:1px;margin:14px 0 6px">${metroLabel(metro)}</div>`;
for (const g of items.slice(0, 40)) {
body += `<div style="font-size:13px;padding:5px 0;border-bottom:1px solid #f2f4f6"><b>${g.name || 'Unknown'}</b>${g.firm?` · ${g.firm}`:''}${g.license_type?` <span style="color:#8A93A2">(${g.license_type})</span>`:''}${g.city?` <span style="color:#8A93A2">— ${g.city}</span>`:''}</div>`;
}
}
}
body += `<div style="margin-top:22px;padding-top:16px;border-top:1px solid #e6e8ec;font-size:12px;color:#8A93A2">
You're getting the <b>${sub.cadence}</b> digest for ${all?'all markets':sub.metros.map(metroLabel).join(', ')}.
Open <a href="https://crcp.agentabrams.com" style="color:#C8A24B">CRCP</a> to work these leads.
</div></div></div>`;
return { html: body, listingCount: showListings.length, brokerCount: showBrokers.length, govCount: showGov.length };
}
// ---- George send (GATED) -------------------------------------------------
function georgeSend({ to, subject, html, from }) {
return new Promise((resolve, reject) => {
const base = process.env.GEORGE_URL || 'http://127.0.0.1:9850';
const token = process.env.GEORGE_EXTERNAL_SEND_TOKEN || '';
if (!token) return reject(new Error('GEORGE_EXTERNAL_SEND_TOKEN not set — external send blocked (fail-closed)'));
const payload = JSON.stringify({ to, subject, body: html, from, account: 'steve-office' });
const u = new URL(base + '/api/send');
const lib = u.protocol === 'https:' ? https : http;
const basic = Buffer.from(process.env.GEORGE_BASIC_AUTH || 'admin:').toString('base64');
const req = lib.request(u, { method: 'POST', headers: {
'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload),
'X-Send-Approval': token, 'Authorization': 'Basic ' + basic,
} }, r => { let s = ''; r.on('data', d => s += d); r.on('end', () => (r.statusCode < 300 ? resolve(s) : reject(new Error(`George ${r.statusCode}: ${s.slice(0,200)}`)))); });
req.on('error', reject); req.write(payload); req.end();
});
}
// ---- main ----------------------------------------------------------------
(async () => {
fs.mkdirSync(OUT, { recursive: true });
const cfg = loadConfig();
const state = loadState();
const govSeen = loadGovSeen();
const nowMs = Date.now();
const listings = loadListings();
const brokers = loadBrokers();
// Fix (contrarian #3): untagged listings were silently dropped from every digest.
// Now counted + logged so the gap is visible and fixable (add cities to METRO_CITIES).
const untagged = listings.filter(l => !l.metro).length;
if (untagged) console.warn(`⚠ ${untagged}/${listings.length} listings UNTAGGED (city not in METRO_CITIES) — not deliverable to any digest. Add their cities to METRO_CITIES to fix.`);
console.log(`snapshot: ${listings.length} listings, ${brokers.length} brokers`);
console.log(`mode: ${SEND ? 'SEND (gated)' : 'DRY-RUN'}`);
const date = new Date().toISOString().slice(0, 10);
const results = [];
let sendFailures = 0, dirtyGov = false;
for (const sub of cfg.subscribers) {
const ss = subState(state, sub.email);
// Fix (contrarian #1): enforce cadence. A non-due subscriber is skipped on send.
if (SEND && !isDue(sub, ss, nowMs)) { console.log(` ${sub.email}: not due (${sub.cadence}, last ${ss.last_sent}) — skip`); continue; }
// per-subscriber diff (their own watermark → correct deltas regardless of others' cadence)
const seenL = new Set(ss.seen_listings), seenB = new Set(ss.seen_brokers);
const newListings = listings.filter(l => !seenL.has(l.id));
const newBrokers = brokers.filter(b => !seenB.has(b.id));
// opt-in gov-agents, per-subscriber, from the sidecar file
let govNewByMetro = {}, govCurrentByMetro = {};
if ((sub.segments || []).includes('gov-agents')) {
const metros = sub.metros.includes('*') ? ['los-angeles','miami','texas','nyc','las-vegas','miami-beach','palm-beach','silicon-valley'] : sub.metros;
govCurrentByMetro = loadGovForMetros(metros);
const mineSeen = govSeen[sub.email] || {};
for (const [m, arr] of Object.entries(govCurrentByMetro)) {
const sg = new Set(mineSeen[m] || []);
const fresh = arr.filter(a => !sg.has(a.license_number));
if (fresh.length) govNewByMetro[m] = fresh;
}
}
const { html, listingCount, brokerCount, govCount } = buildDigestHTML(sub, newListings, newBrokers, govNewByMetro);
if (!listingCount && !brokerCount && !govCount) { console.log(` ${sub.email}: nothing new — skip`); continue; }
const subject = `CRCP: ${listingCount} new listing${listingCount!==1?'s':''}${brokerCount?` · ${brokerCount} new broker${brokerCount!==1?'s':''}`:''}${govCount?` · ${govCount} new agent${govCount!==1?'s':''}`:''}`;
const file = path.join(OUT, `${date}-${sub.email.replace(/[^a-z0-9]/gi,'_')}.html`);
fs.writeFileSync(file, html);
results.push({ email: sub.email, subject, listingCount, brokerCount, govCount, file });
console.log(` ${sub.email}: ${listingCount} listings + ${brokerCount} brokers${govCount?` + ${govCount} agents`:''} → ${path.relative(ROOT, file)}`);
let sent = false;
if (SEND) {
try { await georgeSend({ to: sub.email, subject, html, from: cfg.from }); console.log(` ✉ sent via George`); sent = true; }
catch (e) { sendFailures++; console.error(` ⚠ send failed: ${e.message}`); }
}
// advance THIS subscriber's watermark only when actually sent (or baseline seed) — news-loss guard, per-subscriber
if (COMMIT_STATE && (!SEND || sent)) {
ss.seen_listings = listings.map(l => l.id);
ss.seen_brokers = brokers.map(b => b.id);
if (SEND) ss.last_sent = new Date().toISOString();
if (Object.keys(govCurrentByMetro).length) {
govSeen[sub.email] = govSeen[sub.email] || {};
for (const [m, arr] of Object.entries(govCurrentByMetro)) govSeen[sub.email][m] = arr.map(a => a.license_number);
dirtyGov = true;
}
}
}
if (SEND && sendFailures) console.error(`⚠ ${sendFailures} send failure(s) — those subscribers' watermarks NOT advanced; news preserved for retry.`);
if (COMMIT_STATE) {
state.last_run = new Date().toISOString();
delete state._legacySeen;
fs.writeFileSync(STATE_PATH, JSON.stringify(state, null, 2));
if (dirtyGov) fs.writeFileSync(GOV_STATE_PATH, JSON.stringify(govSeen, null, 2)); // gov IDs kept OUT of the main state file
console.log(`state advanced → ${path.relative(ROOT, STATE_PATH)}${dirtyGov?' + gov sidecar':''}`);
} else {
console.log('state NOT advanced (dry-run). Re-run with --send to send + advance, or --commit-state to seed baseline.');
}
console.log(`\ndone: ${results.length} digest(s) built.`);
})();