`scripts/pull-news.mjs`

52 lines
#!/usr/bin/env node
// Live-data engine for rentv-v1: pulls REAL rentv.com news headlines on a schedule.
// Runs via launchd every 20 min; writes data/news.json the site serves. $0 (plain fetch).
import { writeFileSync, mkdirSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const HERE = dirname(fileURLToPath(import.meta.url));
const OUT = join(HERE, '..', 'data'); mkdirSync(OUT, { recursive: true });

async function fetchDecoded(url) {
  const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 RENTV-v1' }, signal: AbortSignal.timeout(20000) });
  const buf = Buffer.from(await r.arrayBuffer());
  let s = buf.toString('utf8');
  if ((s.match(/�/g) || []).length > 5) s = buf.toString('latin1'); // WIN-1252 fallback
  return s;
}
const clean = (t) => t.replace(/&amp;/g,'&').replace(/&#39;/g,"'").replace(/&quot;/g,'"').replace(/&nbsp;/g,' ').replace(/\s+/g,' ').trim();

try {
  const html = await fetchDecoded('https://www.rentv.com/');
  // Map each story id -> its real property photo (linked-image form: <a href='...news/ID'><img src='.../images/dynamic/news_*.jpg'>)
  const imgMap = {};
  const ire = /href=['"][^'"]*\/news\/(\d+)['"]\s*>\s*<img[^>]+src=['"]([^'"]+\/images\/dynamic\/news_[^'"]+\.(?:jpe?g|png))['"]/gi;
  let im; while ((im = ire.exec(html))) { if (!imgMap[im[1]]) imgMap[im[1]] = im[2].startsWith('http') ? im[2] : 'https://www.rentv.com/' + im[2].replace(/^\//, ''); }
  const re = /<a[^>]+href="([^"]*content\/homepage\/mainnews\/news\/(\d+))"[^>]*>([^<]{20,160})<\/a>/gi;
  const seen = new Set(), items = []; let m;
  while ((m = re.exec(html)) && items.length < 30) {
    const id = m[2]; if (seen.has(id)) continue; seen.add(id);
    const title = clean(m[3]); if (title.length < 20) continue;
    // classify by keywords
    const t = title.toLowerCase();
    const cat = /financ|loan|refi|recap|debt/.test(t) ? 'Financing'
      : /lease|sf lease|tenant/.test(t) ? 'Leases'
      : /develop|breaks ground|project|mixed-use/.test(t) ? 'Development'
      : /retail|shop|center/.test(t) ? 'Retail'
      : /res |multifamily|apartment|unit/.test(t) ? 'Multifamily'
      : /industrial|warehouse|logistics/.test(t) ? 'Industrial'
      : 'Sales';
    items.push({ id, title, url: m[1].startsWith('http') ? m[1] : 'https://www.rentv.com/' + m[1].replace(/^\//,''), cat, image: imgMap[id] || '' });
  }
  const payload = { source: 'rentv.com — live', fetched_at: new Date().toISOString(), count: items.length, items };
  writeFileSync(join(OUT, 'news.json'), JSON.stringify(payload, null, 1));
  console.log('OK: pulled ' + items.length + ' live headlines');
} catch (e) {
  console.error('pull failed:', e.message);
  // never overwrite a good cache with nothing — leave last-good in place
  console.error('(news kept last-good; continuing to ticker refresh)');
}

// Refresh the live market ticker on the same cron (runs regardless of news outcome).
try { await import('./pull-ticker.mjs'); } catch (e) { console.error('ticker refresh failed:', e.message); }