news: add --deep 90-day backfill mode (Codex-validated) — gnews-mirror every source across 6 date-windows (after:/before:), serial + jitter(3-5s) + adaptive backoff (thin response=soft-throttle) + dedup, low concurrency to dodge Google 429s; opt-in flag, daily cron unchanged. Cracks the ~100-item/query cap (TK-10708)

f819c872667ba4a7c7e8a2525d64802851b381dd · 2026-08-19 23:46:45 -0700 · Steve Abrams
Files touched

M scripts/cre-news-monitor.mjs
Diff

commit f819c872667ba4a7c7e8a2525d64802851b381dd
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 23:46:45 2026 -0700

    news: add --deep 90-day backfill mode (Codex-validated) — gnews-mirror every source across 6 date-windows (after:/before:), serial + jitter(3-5s) + adaptive backoff (thin response=soft-throttle) + dedup, low concurrency to dodge Google 429s; opt-in flag, daily cron unchanged. Cracks the ~100-item/query cap (TK-10708)
---
 scripts/cre-news-monitor.mjs | 58 ++++++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/scripts/cre-news-monitor.mjs b/scripts/cre-news-monitor.mjs
index 4f5ed79..a9d6e57 100644
--- a/scripts/cre-news-monitor.mjs
+++ b/scripts/cre-news-monitor.mjs
@@ -124,30 +124,56 @@ function priceOf(text) {
 function locOf(text) { const m = text.match(MARKET); return m ? m[0] : null; }
 function typeOf(title) { for (const [t, re] of TYPE_RES) if (re.test(title)) return t; return 'other'; }
 
+function parseItems(xml, feed) {
+  const items = xml.split(/<item[\s>]/).slice(1).map(b => '<item ' + b.split('</item>')[0] + '</item>');
+  return items.map(b => {
+    let title = tag(b, 'title');
+    if (feed.mode === 'gnews' || feed._deep) title = title.replace(/\s+-\s+[^-]+$/, '').trim();
+    let link = tag(b, 'link'); if (!link) link = (b.match(/<link[^>]*>([^<]+)</i) || [])[1] || tag(b, 'guid');
+    const desc = tag(b, 'description'), date = tag(b, 'pubDate'), text = `${title} ${desc}`;
+    return { feed: feed.name, feed_region: feed.region, title, link: (link || '').trim(), date, ...priceOf(text), location: locOf(text), type: typeOf(title) };
+  });
+}
+const fetchXml = async (url, ua) => (await fetch(url, { headers: { 'User-Agent': ua || UA }, signal: AbortSignal.timeout(14000) })).text();
+
+// --- DEEP backfill (Codex-validated): gnews-mirror every source across 90d date-windows, serial + jitter + adaptive backoff ---
+const DEEP = args.includes('--deep');
+const nap = ms => new Promise(r => setTimeout(r, ms));
+const jitter = (a, b) => a + Math.random() * (b - a);
+const siteOf = feed => { if (feed.mode === 'gnews') { const m = decodeURIComponent(feed.url).match(/site:([^ )]+)/); return m ? m[1] : null; } try { return new URL(feed.url).hostname.replace(/^www\.|^news\./, ''); } catch { return null; } };
+function dateWindows(nDays = 90, win = 15) { const out = [], day = 864e5, now = Date.now(); for (let end = now; end > now - nDays * day; end -= win * day) out.push([new Date(end - win * day).toISOString().slice(0, 10), new Date(end).toISOString().slice(0, 10)]); return out; }
+const gnewsWin = (site, q, a, b) => `https://news.google.com/rss/search?q=${encodeURIComponent(`site:${site} ${q} after:${a} before:${b}`)}&hl=en-US&gl=US&ceid=US:en`;
+
 async function pull(feed) {
   try {
-    const xml = await (await fetch(feed.url, { headers: { 'User-Agent': feed.ua || UA }, signal: AbortSignal.timeout(12000) })).text();
-    const items = xml.split(/<item[\s>]/).slice(1).map(b => '<item ' + b.split('</item>')[0] + '</item>');
-    return items.map(b => {
-      let title = tag(b, 'title');
-      // Google News titles are "Headline - Publication" -> strip the trailing source.
-      if (feed.mode === 'gnews') title = title.replace(/\s+-\s+[^-]+$/, '').trim();
-      let link = tag(b, 'link'); if (!link) link = (b.match(/<link[^>]*>([^<]+)</i) || [])[1] || tag(b, 'guid');
-      const desc = tag(b, 'description');
-      const date = tag(b, 'pubDate');
-      const text = `${title} ${desc}`;
-      return { feed: feed.name, feed_region: feed.region, title, link: (link || '').trim(), date, ...priceOf(text), location: locOf(text), type: typeOf(title) };
-    });
+    if (DEEP) {
+      const site = siteOf(feed);
+      if (site) {
+        const q = feed.broker ? GQB : GQ;
+        const seen = new Set(), out = []; let backoff = 0;   // adaptive: thin response = soft-throttle -> back off
+        for (const [a, b] of dateWindows()) {
+          try {
+            const its = parseItems(await fetchXml(gnewsWin(site, q, a, b), feed.ua), { ...feed, _deep: true });
+            let fresh = 0; for (const it of its) { const k = (it.title || '').toLowerCase().slice(0, 50); if (k && !seen.has(k)) { seen.add(k); out.push(it); fresh++; } }
+            backoff = fresh <= 1 ? Math.min(20000, (backoff || 6000) + 6000) : Math.max(0, backoff - 3000);
+          } catch { backoff = Math.min(20000, (backoff || 8000) + 8000); }
+          await nap(jitter(3000, 5000) + backoff);
+        }
+        return out;
+      }
+    }
+    return parseItems(await fetchXml(feed.url, feed.ua), feed);
   } catch (e) { console.error(`  ${feed.name} feed error: ${e.message}`); return []; }
 }
 
-// bounded concurrency — ~55 feeds shouldn't hit Google News with 55 simultaneous requests (429 risk).
-async function pullPooled(feeds, conc = 8) {
-  const out = []; let i = 0;
+// bounded concurrency — low in DEEP mode (serial-ish) to avoid Google News 429s; normal 8 for the daily run.
+async function pullPooled(feeds, conc) {
+  const c = conc || (DEEP ? 3 : 8), out = []; let i = 0;
   const worker = async () => { while (i < feeds.length) { const f = feeds[i++]; out.push(...await pull(f)); } };
-  await Promise.all(Array.from({ length: Math.min(conc, feeds.length) }, worker));
+  await Promise.all(Array.from({ length: Math.min(c, feeds.length) }, worker));
   return out;
 }
+if (DEEP) console.log(`DEEP backfill: ${FEEDS.length} sources × ${dateWindows().length} date-windows, serial+jitter+backoff (slow, ~10-15 min)…`);
 const all = (await pullPooled(FEEDS)).filter(x => x.title && x.link);
 // closed-deal filter: SALE type + a $ amount + in-footprint (regional feeds count as in-footprint)
 // One-stop NATIONAL: any priced closed CRE deal qualifies (geo tag is for display, not a gate).
← f6dff9a sources dashboard: (1) recent-cell + card samples now sort n · back to Re Flyer Aggregator · sources dashboard: expanded article list now renders UNDER t fe4e646 →