`scripts/refresh.js`

134 lines
// Source refresher — pulls all configured free sources, dedupes by URL hash,
// appends new items to data/news.jsonl. $0: public RSS/HTML only, no paid APIs.
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) DandeanaHub/1.0';

function hash(url) { return crypto.createHash('sha256').update(url.trim()).digest('hex').slice(0, 16); }

function decodeEntities(s) {
  return (s || '')
    .replace(/<!\[CDATA\[(.*?)\]\]>/gs, '$1')
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'").replace(/&nbsp;/g, ' ')
    .replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
}

async function get(url, timeoutMs = 20000) {
  const ctl = new AbortController();
  const t = setTimeout(() => ctl.abort(), timeoutMs);
  try {
    const r = await fetch(url, { headers: { 'User-Agent': UA }, signal: ctl.signal, redirect: 'follow' });
    if (!r.ok) throw new Error(`HTTP ${r.status}`);
    return await r.text();
  } finally { clearTimeout(t); }
}

// Google News RSS <item> blocks → items
function parseRss(xml, sourceLabel, category) {
  const items = [];
  for (const m of xml.matchAll(/<item>([\s\S]*?)<\/item>/g)) {
    const block = m[1];
    const pick = tag => { const mm = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`)); return mm ? mm[1] : ''; };
    const title = decodeEntities(pick('title'));
    const link = decodeEntities(pick('link'));
    if (!title || !link) continue;
    const pub = pick('pubDate');
    const srcTag = decodeEntities(pick('source'));
    items.push({
      title, url: link,
      source: srcTag || sourceLabel,
      category,
      published: pub ? new Date(pub).toISOString() : null,
      snippet: decodeEntities(pick('description')).slice(0, 300),
    });
  }
  return items;
}

// CEQAnet project page → one status item when content changes
async function checkCeqa(src) {
  const html = await get(src.url);
  const docs = [...html.matchAll(/<a[^>]+href="(\/[^"]*(?:Document|Attachment)[^"]*)"[^>]*>([^<]+)<\/a>/gi)]
    .map(m => decodeEntities(m[2])).slice(0, 20);
  const fingerprint = hash(src.url + '|' + docs.join('|'));
  return [{
    title: `CEQAnet — ${src.label}: ${docs.length ? docs.length + ' document(s) on record' : 'record checked'}`,
    url: src.url, source: 'CEQAnet (State Clearinghouse)', category: 'ceqa',
    published: null, snippet: docs.slice(0, 5).join(' · '),
    _fp: fingerprint, // dedupe on content fingerprint, not URL, so a changed record re-surfaces
  }];
}

// Poway page scan → items for links whose text matches the keywords
async function scanPage(src) {
  const html = await get(src.url);
  const kw = src.keywords.map(k => k.toLowerCase());
  const out = [];
  for (const m of html.matchAll(/<a[^>]+href="([^"#]+)"[^>]*>([\s\S]{3,200}?)<\/a>/gi)) {
    const text = decodeEntities(m[2]);
    if (!text) continue;
    const low = text.toLowerCase();
    if (!kw.some(k => low.includes(k))) continue;
    let href = m[1];
    if (href.startsWith('/')) href = new URL(src.url).origin + href;
    if (!href.startsWith('http')) continue;
    out.push({ title: text.slice(0, 180), url: href, source: src.label, category: src.category || 'government', published: null, snippet: `Matched on ${src.url}` });
  }
  return out;
}

async function runRefresh(DATA) {
  const sources = JSON.parse(fs.readFileSync(path.join(DATA, 'sources.json'), 'utf8'));
  const newsPath = path.join(DATA, 'news.jsonl');
  const existing = new Set();
  try {
    for (const line of fs.readFileSync(newsPath, 'utf8').split('\n')) {
      if (!line) continue;
      const it = JSON.parse(line);
      existing.add(it.id);
    }
  } catch { /* first run */ }

  const collected = [];
  const errors = [];
  for (const src of sources) {
    try {
      if (src.type === 'gnews') {
        const url = `https://news.google.com/rss/search?q=${encodeURIComponent(src.query)}&hl=en-US&gl=US&ceid=US:en`;
        collected.push(...parseRss(await get(url), src.label, src.category || 'news'));
      } else if (src.type === 'rss') {
        collected.push(...parseRss(await get(src.url), src.label, src.category || 'news'));
      } else if (src.type === 'ceqa') {
        collected.push(...await checkCeqa(src));
      } else if (src.type === 'scan') {
        collected.push(...await scanPage(src));
      }
    } catch (e) { errors.push(`${src.label}: ${e.message}`); }
  }

  let added = 0;
  const lines = [];
  for (const it of collected) {
    const id = it._fp || hash(it.url);
    if (existing.has(id)) continue;
    existing.add(id);
    delete it._fp;
    lines.push(JSON.stringify({ id, firstSeen: new Date().toISOString(), ...it }));
    added++;
  }
  if (lines.length) fs.appendFileSync(newsPath, lines.join('\n') + '\n');
  const meta = { lastRefresh: new Date().toISOString(), added, checked: collected.length, errors };
  fs.writeFileSync(path.join(DATA, 'refresh-meta.json'), JSON.stringify(meta, null, 2));
  console.log(`refresh: ${added} new / ${collected.length} checked${errors.length ? ' — errors: ' + errors.join('; ') : ''}`);
  return meta;
}

module.exports = { runRefresh };

if (require.main === module) {
  runRefresh(path.join(__dirname, '..', 'data')).catch(e => { console.error(e); process.exit(1); });
}