news: record EVERY article to a growing article-archive.json each run (Steve: historical backfill is Google-capped, so build our own history forward instead) — deduped, no cap, compounds daily into our own CRE-news database (TK-10708)

6c18e62897b4a711c2b209a8570068539c2919d7 · 2026-08-20 00:07:31 -0700 · Steve Abrams

Files touched

M scripts/cre-news-monitor.mjs

Diff

commit 6c18e62897b4a711c2b209a8570068539c2919d7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Aug 20 00:07:31 2026 -0700

    news: record EVERY article to a growing article-archive.json each run (Steve: historical backfill is Google-capped, so build our own history forward instead) — deduped, no cap, compounds daily into our own CRE-news database (TK-10708)
---
 scripts/cre-news-monitor.mjs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scripts/cre-news-monitor.mjs b/scripts/cre-news-monitor.mjs
index 93594f4..b5d9701 100644
--- a/scripts/cre-news-monitor.mjs
+++ b/scripts/cre-news-monitor.mjs
@@ -222,7 +222,20 @@ const STORE_CAP = parseInt(process.env.STORE_CAP || '12000', 10);   // aligned w
 if (store.length > STORE_CAP) store = store.slice(0, STORE_CAP);
 if (!args.includes('--dry')) writeFileSync(STORE, JSON.stringify(store, null, 0));
 
+// ---- ARTICLE ARCHIVE: record EVERY article going forward (Steve: "build our own") — historical backfill is
+// blocked by Google's ~100/query cap + rate-limits, so instead we accumulate everything on each run, deduped,
+// NO CAP. Over daily runs this compounds into our own deep CRE-news history that no third party can throttle.
+const ARCH = join(ROOT, 'data', 'article-archive.json');
+let arch = existsSync(ARCH) ? JSON.parse(readFileSync(ARCH, 'utf8')) : [];
+const archSig = a => (a.title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 60);
+const archHave = new Set(arch.map(archSig));
+let archAdded = 0;
+for (const x of all) { const s = archSig(x); if (s && !archHave.has(s)) { archHave.add(s); arch.push({ title: x.title, link: x.link, date: x.date, price: x.label || null, type: x.type === 'other' ? 'News' : x.type, market: x.location || x.feed_region, source: x.feed, first_seen: new Date().toISOString() }); archAdded++; } }
+arch.sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0));   // newest first
+if (!args.includes('--dry')) writeFileSync(ARCH, JSON.stringify(arch, null, 0));
+
 console.log(`CRE-news scan: ${all.length} items across ${FEEDS.length} feeds -> ${dealItems.length} closed-deal (sale+price+RE, national), ${fresh.length} NEW · store now ${store.length} (+${added}).`);
+console.log(`Article archive: +${archAdded} new -> ${arch.length} total articles recorded (our own growing history — no cap).`);
 for (const d of fresh.slice(0, 15)) console.log(`  ${(d.label||'—').padStart(7)}  ${d.type.padEnd(5)} ${(d.location||d.feed_region).padEnd(14)}  ${d.title.slice(0,56)}  [${d.feed}]`);
 
 if (!args.includes('--dry')) {

← e28d009 news: remove pulled_list/samples storage cap (was 250) -> In · back to Re Flyer Aggregator · add refine-loop.sh — overnight loop that records every artic 3721276 →