← back to Re Flyer Aggregator
scripts/cre-news-monitor.mjs
286 lines
#!/usr/bin/env node
// TK-10708 FREE CRE-news closed-deal monitor. Aggregates working CRE trade-press RSS
// feeds (TRD-LA, Commercial Observer, Connect CRE, The Registry NorCal), filters to
// SALE ("closed deal") items with a $ amount in the West footprint, extracts price +
// property + location, dedupes by link -> the same-day "be first" alert feed. $0, no key.
//
// Not "before the news exists" -- it's first to have EVERY outlet's closed deals compiled
// same-day, before a competitor assembles them. Usage: node scripts/cre-news-monitor.mjs [--dry]
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const args = process.argv.slice(2);
const UA = 'Mozilla/5.0 (DW-research; steve@designerwallcoverings.com)';
// social-scraper UA — allowlisted by many Cloudflare/paywall setups for share previews,
// so it pulls MediaNews CA papers (Mercury News, SD Union-Tribune) directly (no proxy).
const FB_UA = 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)';
const SEEN = join(ROOT, 'data', 'crenews-seen.json');
// mode 'rss' = direct feed; mode 'gnews' = Google News RSS proxy for Cloudflare-blocked
// outlets (Bisnow/GlobeSt/Traded/MHN 403 their own feeds). gnews query is geo-scoped so
// those items are in-footprint by construction.
// NATIONAL one-stop CRE news. gnews query is CRE-deal-scoped (no geo) so it catches every
// US market from the Cloudflare-blocked outlets without scraping them.
const GQ = '(commercial OR office OR industrial OR retail OR multifamily OR apartment OR "square feet" OR portfolio) (sells OR acquires OR buys OR "million" OR "billion")';
// Google News window — default 14d (fresh daily run); override GNEWS_WHEN=90d for a 3-month backfill.
const GNEWS_WHEN = process.env.GNEWS_WHEN || '90d'; // keep a rolling 90 days consistently (Steve)
const gnews = site => `https://news.google.com/rss/search?q=${encodeURIComponent(`site:${site} ${GQ} when:${GNEWS_WHEN}`)}&hl=en-US&gl=US&ceid=US:en`;
// client-side recency gate for a "be first" DAILY digest (env-overridable).
const MAX_AGE_DAYS = parseInt(process.env.MAX_AGE_DAYS || '95', 10); // rolling ~90-day window (Steve)
const MS_PER_DAY = 864e5;
const ageDays = s => { const t = Date.parse(s); return isNaN(t) ? null : (Date.now() - t) / MS_PER_DAY; };
const FEEDS = [
// The Real Deal — per-market direct feeds
{ name: 'TRD-National', region: 'National', mode: 'rss', url: 'https://therealdeal.com/national/feed/' },
{ name: 'TRD-NY', region: 'New York', mode: 'rss', url: 'https://therealdeal.com/new-york/feed/' },
{ name: 'TRD-Miami', region: 'Miami', mode: 'rss', url: 'https://therealdeal.com/miami/feed/' },
{ name: 'TRD-Chicago', region: 'Chicago', mode: 'rss', url: 'https://therealdeal.com/chicago/feed/' },
{ name: 'TRD-TriState', region: 'Tri-State', mode: 'rss', url: 'https://therealdeal.com/tristate/feed/' },
{ name: 'TRD-Texas', region: 'Texas', mode: 'rss', url: 'https://therealdeal.com/texas/feed/' },
{ name: 'TRD-LA', region: 'LA', mode: 'rss', url: 'https://therealdeal.com/la/feed/' },
{ name: 'LATimes-RE', region: 'LA', mode: 'rss', url: 'https://www.latimes.com/business/real-estate/rss2.0.xml' },
// National CRE trade press — direct feeds
{ name: 'CommercialObserver', region: 'National', mode: 'rss', url: 'https://commercialobserver.com/feed/' },
{ name: 'ConnectCRE', region: 'National', mode: 'rss', url: 'https://www.connectcre.com/feed/' },
{ name: 'Registry-NorCal', region: 'NorCal', mode: 'rss', url: 'https://news.theregistrysf.com/feed/' },
{ name: 'REBusinessOnline', region: 'National', mode: 'rss', url: 'https://rebusinessonline.com/feed/' },
{ name: 'REJournals', region: 'National', mode: 'rss', url: 'https://rejournals.com/feed/' },
{ name: 'MultifamilyDive', region: 'National', mode: 'rss', url: 'https://www.multifamilydive.com/feeds/news/' },
// California papers — DIRECT via the social-scraper UA (cracks MediaNews Cloudflare)
{ name: 'MercuryNews-RE', region: 'Bay Area', mode: 'rss', ua: FB_UA, url: 'https://www.mercurynews.com/business/real-estate/feed/' },
{ name: 'SDUnionTribune-RE', region: 'San Diego', mode: 'rss', ua: FB_UA, url: 'https://www.sandiegouniontribune.com/business/real-estate/feed/' },
{ name: 'SFYimby', region: 'San Francisco', mode: 'rss', url: 'https://sfyimby.com/feed' },
// CA business journals — stricter Cloudflare, via Google News proxy
{ name: 'LABizJournal', region: 'LA', mode: 'gnews', url: gnews('bizjournals.com/losangeles') },
{ name: 'SFBizTimes', region: 'San Francisco', mode: 'gnews', url: gnews('bizjournals.com/sanfrancisco') },
// Cloudflare-blocked outlets via Google News RSS proxy (national)
{ name: 'Traded', region: 'National', mode: 'gnews', url: gnews('traded.co') },
// Bisnow has a WORKING direct feed (full desc w/ price) — better than the Google News snippet.
{ name: 'Bisnow', region: 'National', mode: 'rss', url: 'https://www.bisnow.com/rss' },
{ name: 'GlobeSt', region: 'National', mode: 'gnews', url: gnews('globest.com') },
{ name: 'MultiHousingNews', region: 'National', mode: 'gnews', url: gnews('multihousingnews.com') },
{ name: 'CommercialSearch', region: 'National', mode: 'gnews', url: gnews('commercialsearch.com') },
{ name: 'RENTV', region: 'CA', mode: 'gnews', url: gnews('rentv.com') },
];
// --- MAJOR BROKER deal-news "scrapes" via the same Google News proxy (their OWN press releases) ---
// Brokers announce with different verbs than journalists ("arranges / brokers / negotiates the sale of").
const GQB = '(arranges OR brokers OR negotiates OR represents OR facilitates OR closes OR "sale of" OR sells OR acquires OR secures OR "million" OR "billion") (property OR building OR portfolio OR "square feet" OR industrial OR office OR retail OR multifamily OR apartments OR lease OR loan OR financing OR sale)';
const gnewsB = site => `https://news.google.com/rss/search?q=${encodeURIComponent(`site:${site} ${GQB} when:${GNEWS_WHEN}`)}&hl=en-US&gl=US&ceid=US:en`;
const BROKERS = [
['CBRE', 'cbre.com'], ['JLL', 'jll.com'], ['Cushman-Wakefield', 'cushmanwakefield.com'],
['Colliers', 'colliers.com'], ['Newmark', 'nmrk.com'], ['Marcus-Millichap', 'marcusmillichap.com'],
['Avison-Young', 'avisonyoung.com'], ['Savills', 'savills.us'], ['Kidder-Mathews', 'kidder.com'],
['Lee-Associates', 'lee-associates.com'], ['Berkadia', 'berkadia.com'], ['Walker-Dunlop', 'walkerdunlop.com'],
['Northmarq', 'northmarq.com'], ['Matthews', 'matthews.com'], ['Transwestern', 'transwestern.com'],
['Stream-Realty', 'streamrealty.com'], ['Institutional-Property-Advisors', 'ipausa.com'], ['Eastdil', 'eastdilsecured.com'],
];
for (const [name, dom] of BROKERS) FEEDS.push({ name: `Broker-${name}`, region: 'National', mode: 'gnews', broker: true, url: gnewsB(dom) });
// --- MORE NEWS SERVICES: direct RSS where a feed exists, else the Google News proxy ---
FEEDS.push(
{ name: 'CommercialPropertyExec', region: 'National', mode: 'rss', url: 'https://www.commercialsearch.com/news/feed/' },
{ name: 'Propmodo', region: 'National', mode: 'rss', url: 'https://www.propmodo.com/feed/' },
{ name: 'YieldPRO', region: 'National', mode: 'rss', url: 'https://yieldpro.com/feed/' },
{ name: 'MultifamilyExec', region: 'National', mode: 'rss', url: 'https://www.multifamilyexecutive.com/rss.xml' },
{ name: 'CoStar', region: 'National', mode: 'gnews', url: gnews('costar.com') }, // 403 direct -> proxy
{ name: 'Crexi', region: 'National', mode: 'gnews', url: gnews('crexi.com') }, // no RSS -> proxy
{ name: 'BizJournals-NY', region: 'New York', mode: 'gnews', url: gnews('bizjournals.com/newyork') },
{ name: 'BizJournals-Chicago', region: 'Chicago', mode: 'gnews', url: gnews('bizjournals.com/chicago') },
{ name: 'BizJournals-Dallas', region: 'Dallas', mode: 'gnews', url: gnews('bizjournals.com/dallas') },
{ name: 'BizJournals-Atlanta', region: 'Atlanta', mode: 'gnews', url: gnews('bizjournals.com/atlanta') },
{ name: 'BizJournals-SouthFlorida', region: 'South Florida', mode: 'gnews', url: gnews('bizjournals.com/southflorida') },
{ name: 'BizJournals-Boston', region: 'Boston', mode: 'gnews', url: gnews('bizjournals.com/boston') },
{ name: 'BizJournals-Phoenix', region: 'Phoenix', mode: 'gnews', url: gnews('bizjournals.com/phoenix') },
{ name: 'BizJournals-Seattle', region: 'Seattle', mode: 'gnews', url: gnews('bizjournals.com/seattle') },
{ name: 'BizJournals-Denver', region: 'Denver', mode: 'gnews', url: gnews('bizjournals.com/denver') },
{ name: 'BizJournals-SanDiego', region: 'San Diego', mode: 'gnews', url: gnews('bizjournals.com/sandiego') },
);
// National market tagger — labels each deal's metro/market for the "Where" column.
const MARKET = /\b(Manhattan|Brooklyn|Queens|New York|NYC|Long Island|Jersey City|Newark|Tri-?State|Los Angeles|SoCal|Southern California|Hollywood|Beverly Hills|Culver City|Santa Monica|Pasadena|San Fernando|\bL\.?A\.?\b|San Diego|Orange County|Inland Empire|Ventura|San Francisco|Bay Area|Silicon Valley|San Jose|Oakland|Sacramento|NorCal|Miami|South Florida|Fort Lauderdale|Palm Beach|Orlando|Tampa|Chicago|Dallas|Fort Worth|Houston|Austin|San Antonio|Texas|Boston|Cambridge|Washington|\bD\.?C\.?\b|Atlanta|Seattle|Portland|Phoenix|Scottsdale|Las Vegas|Reno|Denver|Nashville|Charlotte|Raleigh|Philadelphia|Minneapolis|Detroit|California|Florida|Nevada|Arizona|Oregon|Colorado)\b/i;
// "all of it" (Steve): classify every CRE transaction type, don't exclude any.
// Priority order matters — a "$75M Loan for the Buy of X" leads with the loan, so type=loan.
const TYPE_RES = [
['Lease', /\b(leases?|lease renewal|renews?[^.]*lease|signs?[^.]*lease|lease extension|inks?[^.]*lease|subleases?|renews? lease|lease renewed)\b/i],
['Loan', /\b(loan|refinanc|mortgage|CMBS|financing|recapitaliz|construction loan|bond|debt package|mezz)\b/i],
['Development', /\b(breaks? ground|tops? out|proposes?|to build|greenlights?|breaks ground|groundbreaking)\b/i],
['Sale', /\b(sells?|sold|buys?|acquires?|acquired|purchase[sd]?|pays?|trades?|snaps up|picks up|closes? on|nabs?|grabs?|lands?[^.]*deal|changes? hands|offloads?|unloads?|dispose[sd]?)\b/i],
// NEW-TO-MARKET listings (Steve: "any new listing for CRE in the news") — checked AFTER Sale so closed deals win.
['Listing', /\b(lists?\s+for\s+(sale|lease)|listed\s+for|hits?\s+the\s+market|comes?\s+to\s+market|now\s+available|available\s+for\s+(sale|lease)|seeks?\s+(a\s+)?(buyer|tenant)|asking\s+(rent|\$)|on\s+the\s+market|up\s+for\s+sale|puts?[^.]{0,24}(on|up)\s+(the\s+)?(market|for\s+sale)|marketing[^.]{0,24}for\s+(sale|lease)|for\s+(sale|lease)\b)/i],
];
const decode = s => (s || '').replace(/<!\[CDATA\[|\]\]>/g, '').replace(/&/g, '&').replace(/’|'|’/g, "'").replace(/–|—/g, '-').replace(/"/g, '"').replace(/&#\d+;/g, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
const tag = (block, t) => { const m = block.match(new RegExp(`<${t}[^>]*>([\\s\\S]*?)</${t}>`, 'i')); return m ? decode(m[1]) : ''; };
function priceOf(text) {
const m = text.match(/\$\s?([\d]+(?:\.\d+)?)\s*(billion|million|B|M|K|thousand)\b/i);
if (!m) return { price: null, label: null };
const unit = m[2].toLowerCase();
const mult = unit.startsWith('b') ? 1e9 : unit.startsWith('m') ? 1e6 : (unit === 'k' || unit === 'thousand') ? 1e3 : 1;
return { price: Math.round(parseFloat(m[1]) * mult), label: `$${m[1]}${unit[0].toUpperCase()}` };
}
function locOf(text) { const m = text.match(MARKET); return m ? m[0] : null; }
function typeOf(title) { for (const [t, re] of TYPE_RES) if (re.test(title)) return t; return 'other'; }
function parseItems(xml, feed) {
const items = xml.split(/<item[\s>]/).slice(1).map(b => '<item ' + b.split('</item>')[0] + '</item>');
return items.map(b => {
let title = tag(b, 'title');
if (feed.mode === 'gnews' || feed._deep) title = title.replace(/\s+-\s+[^-]+$/, '').trim();
let link = tag(b, 'link'); if (!link) link = (b.match(/<link[^>]*>([^<]+)</i) || [])[1] || tag(b, 'guid');
const desc = tag(b, 'description'), date = tag(b, 'pubDate'), text = `${title} ${desc}`;
return { feed: feed.name, feed_region: feed.region, title, link: (link || '').trim(), date, ...priceOf(text), location: locOf(text), type: typeOf(title) };
});
}
const fetchXml = async (url, ua) => (await fetch(url, { headers: { 'User-Agent': ua || UA }, signal: AbortSignal.timeout(14000) })).text();
// --- DEEP backfill (Codex-validated): gnews-mirror every source across 90d date-windows, serial + jitter + adaptive backoff ---
const DEEP = args.includes('--deep');
const nap = ms => new Promise(r => setTimeout(r, ms));
const jitter = (a, b) => a + Math.random() * (b - a);
function siteOf(feed) {
if (feed.mode === 'gnews') { const m = decodeURIComponent(feed.url).match(/site:([^ )]+)/); return m ? m[1] : null; }
try { return new URL(feed.url).hostname.replace(/^www\.|^news\./, ''); } catch { return null; }
}
function dateWindows(nDays = 90, win = 15) {
const out = [], now = Date.now();
for (let end = now; end > now - nDays * MS_PER_DAY; end -= win * MS_PER_DAY)
out.push([new Date(end - win * MS_PER_DAY).toISOString().slice(0, 10), new Date(end).toISOString().slice(0, 10)]);
return out;
}
const gnewsWin = (site, q, a, b) => `https://news.google.com/rss/search?q=${encodeURIComponent(`site:${site} ${q} after:${a} before:${b}`)}&hl=en-US&gl=US&ceid=US:en`;
async function pull(feed) {
try {
if (DEEP) {
const site = siteOf(feed);
if (site) {
const q = feed.broker ? GQB : GQ;
const seen = new Set(), out = []; let backoff = 0; // adaptive: thin response = soft-throttle -> back off
for (const [a, b] of dateWindows()) {
try {
const its = parseItems(await fetchXml(gnewsWin(site, q, a, b), feed.ua), { ...feed, _deep: true });
let fresh = 0; for (const it of its) { const k = (it.title || '').toLowerCase().slice(0, 50); if (k && !seen.has(k)) { seen.add(k); out.push(it); fresh++; } }
backoff = fresh <= 1 ? Math.min(20000, (backoff || 6000) + 6000) : Math.max(0, backoff - 3000);
} catch { backoff = Math.min(20000, (backoff || 8000) + 8000); }
await nap(jitter(3000, 5000) + backoff);
}
return out;
}
}
return parseItems(await fetchXml(feed.url, feed.ua), feed);
} catch (e) { console.error(` ${feed.name} feed error: ${e.message}`); return []; }
}
// bounded concurrency — low in DEEP mode (serial-ish) to avoid Google News 429s; normal 8 for the daily run.
const POOL_CONC = DEEP ? 3 : 8;
async function pullPooled(feeds, conc = POOL_CONC) {
const c = conc, out = []; let i = 0;
const worker = async () => { while (i < feeds.length) { const f = feeds[i++]; out.push(...await pull(f)); } };
await Promise.all(Array.from({ length: Math.min(c, feeds.length) }, worker));
return out;
}
if (DEEP) console.log(`DEEP backfill: ${FEEDS.length} sources × ${dateWindows().length} date-windows, serial+jitter+backoff (slow, ~10-15 min)…`);
const all = (await pullPooled(FEEDS)).filter(x => x.title && x.link);
// closed-deal filter: SALE type + a $ amount + in-footprint (regional feeds count as in-footprint)
// One-stop NATIONAL: any priced closed CRE deal qualifies (geo tag is for display, not a gate).
// exclude non-deal headlines w/ a $ figure (personnel, rankings, op-eds) AND tech/VC noise
// (valuation/funding rounds) that the Google News proxy can pull from mixed-topic outlets.
const NOT_DEAL = /\b(to head|to lead|hires?|names?|appoints?|joins?|promotes?|taps|steps down|resigns?|obituary|op-ed|opinion|ranking|list of|top \d+|named to|to oversee|dies at|valuation|raises|raised|series [a-e]\b|funding round|venture|IPO|goes public|hits \$)\b/i;
// require a real-estate signal so we don't surface tech M&A that merely says "acquires $Xb".
// RENTV-breadth: any commercial-real-estate CONTENT or SERVICE, not just a priced closed deal (Steve).
const RE_SIGNAL = /\b(office|industrial|logistics|warehouse|retail|multifamily|apartment|residential|housing|hotel|resort|motel|portfolio|building|tower|complex|campus|mall|shopping center|plaza|land|development|site|self.?storage|storage|senior|life science|lab|medical|mixed.?use|square feet|sq\.?\s?ft|\bSF\b|property|properties|acres?|estate|condo|data center|parcel|REIT|apartments?|commercial real estate|\bCRE\b|real estate|brokerage|broker|leasing|\blease\b|tenant|landlord|sublease|net lease|ground lease|financing|refinanc|mortgage|\bloan\b|capital markets|investment sales|property management|apprais|escrow|title (company|insurance)|construction|architect|zoning|entitlement|cap rate|occupancy|vacancy|absorption|rent(al|s)?|square-foot|rentable|coworking|flex space|cold storage|last.?mile)\b/i;
const roundup = t => (t.match(/·/g) || []).length >= 2; // Traded daily-digest posts, not one deal
// recency fail-CLOSED (Cody Hole 4): a null/unparseable date must NOT count as fresh.
// RENTV = Steve's own outlet: keep ALL its recent RE posts (price optional), 14d window.
const isRentv = x => x.feed === 'RENTV';
const isBroker = x => (x.feed || '').startsWith('Broker-'); // major-broker press releases
const recent = x => { const a = ageDays(x.date); return a !== null && a <= Math.max(MAX_AGE_DAYS, isRentv(x) ? 14 : 0); };
// ALL CRE transaction types (Steve: "all of it" — sale/loan/lease/development), with a $ + RE signal, recent.
// PLUS every recent RENTV post AND every recent MAJOR-BROKER deal announcement (price optional — brokers
// headline the property, not always the $), typed by verb, so broker deal-flow shows even without a price.
// price-optional classes: brokers headline the property not the $, and lease-renewals/subleases/
// new listings usually have no price at all (Steve: "lease renewals, subleases etc" + "any new listing").
const priceOpt = x => isBroker(x) || x.type === 'Lease' || x.type === 'Listing';
// LOOSENED (Steve: "loosen temperature for Deals kept, expand to any CRE service RENTV would use"):
// keep ANY recent CRE-relevant article — price + specific deal-verb no longer required.
const dealItems = all.filter(x => !NOT_DEAL.test(x.title) && !roundup(x.title) && RE_SIGNAL.test(x.title) && recent(x));
for (const x of dealItems) if (x.type === 'other') x.type = isRentv(x) ? 'Post' : 'News'; // generic CRE coverage
// Content SIGNATURE (Cody Hole 2B/3): Google News article URLs rotate, so dedup on the
// deal's content (price + normalized title), not the unstable link. Bounded so it never
// grows without limit; only DEAL sigs are stored (not every fetched item).
const sigOf = x => `${x.price}|${x.title.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().split(' ').slice(0, 7).join(' ')}`;
const seen = existsSync(SEEN) ? new Set(JSON.parse(readFileSync(SEEN, 'utf8'))) : new Set();
let fresh = dealItems.filter(x => !seen.has(sigOf(x)));
fresh.sort((a, b) => (b.price || 0) - (a.price || 0));
// cross-outlet dedup within this run (price + first name token).
const dseen = new Set(); fresh = fresh.filter(x => { const s = `${x.price}|${(x.title.match(/[A-Za-z]+/) || [''])[0].toLowerCase()}`; if (dseen.has(s)) return false; dseen.add(s); return true; });
// ---- ROLLING accumulating deal store (grows across runs; deals persist past the recency window) ----
const STORE = join(ROOT, 'data', 'deal-store.json');
let store = existsSync(STORE) ? JSON.parse(readFileSync(STORE, 'utf8')) : [];
const storeSig = d => `${d.price}|${(d.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 34)}`;
const have = new Set(store.map(storeSig));
let added = 0;
for (const d of dealItems) { const s = storeSig(d); if (!have.has(s)) { have.add(s); store.push({ type: d.type, price: d.price, label: d.label, market: d.location || d.feed_region, title: d.title, source: d.feed, link: d.link, date: d.date, buyer: null, seller: null, added: new Date().toISOString() }); added++; } }
store.sort((a, b) => (b.date || '').localeCompare(a.date || '') || (b.price || 0) - (a.price || 0));
const STORE_CAP = parseInt(process.env.STORE_CAP || '12000', 10); // aligned with the deeds ingest so a backfill doesn't truncate
const SEEN_CAP = parseInt(process.env.SEEN_CAP || '8000', 10); // rolling sig-set cap; env-overridable
if (store.length > STORE_CAP) store = store.slice(0, STORE_CAP);
if (!args.includes('--dry')) writeFileSync(STORE, JSON.stringify(store, null, 0));
// ---- ARTICLE ARCHIVE: record EVERY article going forward (Steve: "build our own") — historical backfill is
// blocked by Google's ~100/query cap + rate-limits, so instead we accumulate everything on each run, deduped,
// NO CAP. Over daily runs this compounds into our own deep CRE-news history that no third party can throttle.
const ARCH = join(ROOT, 'data', 'article-archive.json');
let arch = existsSync(ARCH) ? JSON.parse(readFileSync(ARCH, 'utf8')) : [];
const archSig = a => (a.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 60);
const archHave = new Set(arch.map(archSig));
// extract buyer/seller from the headline (same patterns as build-flyers-site) so search can filter parties
const NM = "[A-Z][\\w.&'’ -]{2,45}?";
const partiesOf = t => {
let b = t.match(new RegExp(`^(${NM})\\s+(?:to\\s+)?(?:buys?|acquires?|purchases?|to pay|pays?|snaps up|picks up|nabs?|grabs?)\\b`, 'i')) || t.match(new RegExp(`\\b(?:sells?|sold|offloads?|unloads?)\\b.*?\\bto\\s+(${NM})(?:\\s+for\\b|,|\\.|$)`, 'i'));
let s = t.match(new RegExp(`^(${NM})\\s+(?:to\\s+)?(?:sells?|sold|offloads?|unloads?|dispose)`, 'i')) || t.match(new RegExp(`\\bfrom\\s+(${NM})(?:\\s+for\\b|,|\\.|$)`, 'i'));
return { buyer: b ? b[1].trim() : null, seller: s ? s[1].trim() : null };
};
let archAdded = 0;
// record every CRE-relevant article (RE signal, not obvious non-deal noise or roundups) — excludes leaks like
// "…restaurant chain; Lockheed lands $35B missile deal". Still very broad (any office/retail/lease/broker/etc.).
for (const x of all) { if (!RE_SIGNAL.test(x.title) || NOT_DEAL.test(x.title) || roundup(x.title)) continue; const s = archSig(x); if (s && !archHave.has(s)) { archHave.add(s); const p = partiesOf(x.title || ''); arch.push({ title: x.title, link: x.link, date: x.date, price: x.label || null, type: x.type === 'other' ? 'News' : x.type, market: x.location || x.feed_region, source: x.feed, buyer: p.buyer, seller: p.seller, first_seen: new Date().toISOString() }); archAdded++; } }
arch.sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0)); // newest first
if (!args.includes('--dry')) writeFileSync(ARCH, JSON.stringify(arch, null, 0));
console.log(`CRE-news scan: ${all.length} items across ${FEEDS.length} feeds -> ${dealItems.length} closed-deal (sale+price+RE, national), ${fresh.length} NEW · store now ${store.length} (+${added}).`);
console.log(`Article archive: +${archAdded} new -> ${arch.length} total articles recorded (our own growing history — no cap).`);
for (const d of fresh.slice(0, 15)) console.log(` ${(d.label||'—').padStart(7)} ${d.type.padEnd(5)} ${(d.location||d.feed_region).padEnd(14)} ${d.title.slice(0,56)} [${d.feed}]`);
if (!args.includes('--dry')) {
dealItems.forEach(x => seen.add(sigOf(x))); // store DEAL sigs only (bounded, not every item)
writeFileSync(SEEN, JSON.stringify([...seen].slice(-SEEN_CAP), null, 0)); // cap so it never grows unbounded
if (fresh.length) {
const stamp = new Date().toISOString().slice(0, 10);
writeFileSync(join(ROOT, 'out', `crenews-new-deals-${stamp}.json`), JSON.stringify(fresh, null, 2));
console.log(`-> out/crenews-new-deals-${stamp}.json (${fresh.length} deals) for the alert feed.`);
}
}
// --- SOURCE PROVENANCE: where the data comes from (per-feed pull + deal counts + samples) ---
const bySrc = {};
for (const f of FEEDS) bySrc[f.name] = {
name: f.name, method: f.broker ? 'Broker press (Google News)' : f.mode === 'gnews' ? 'Google News proxy' : (f.ua ? 'RSS (social-UA)' : 'RSS direct'),
region: f.region, url: f.url, pulled: 0, deals: 0, samples: [], pulled_list: []
};
// store the actual pulled items (cap 60) AND the deals (cap 40) so the dashboard can VIEW them (Steve).
const PULL_CAP = Number(process.env.PULL_CAP) || Infinity, SAMP_CAP = Number(process.env.SAMP_CAP) || Infinity; // NO CAP (Steve) — store every pulled item + every deal per feed
for (const x of all) if (bySrc[x.feed]) { const b = bySrc[x.feed]; b.pulled++; if (b.pulled_list.length < PULL_CAP) b.pulled_list.push({ title: (x.title || '').slice(0, 120), price: x.label || null, type: x.type, market: x.location || x.feed_region, link: x.link, date: x.date }); }
for (const d of dealItems) { const b = bySrc[d.feed]; if (b) { b.deals++; if (b.samples.length < SAMP_CAP) b.samples.push({ title: (d.title || '').slice(0, 120), price: d.label, type: d.type, market: d.location || d.feed_region, link: d.link, date: d.date }); } }
writeFileSync(join(ROOT, 'public', 'sources.json'), JSON.stringify({
generated: new Date().toISOString(), total_items: all.length, total_deals: dealItems.length,
feeds: Object.values(bySrc).sort((a, b) => b.deals - a.deals || b.pulled - a.pulled)
}, null, 1));
console.log(`-> public/sources.json (${FEEDS.length} feeds, provenance)`);