`lib/aggregate.js`

136 lines
'use strict';
const fs = require('fs');
const path = require('path');
const { fetchFeed } = require('./rss');
const { rewriteAll } = require('./paraphrase');

const FEEDS_PATH = path.join(__dirname, '..', 'data', 'feeds.json');
const PER_FEED = parseInt(process.env.PER_FEED || '5', 10);          // items kept per feed
const PER_COLUMN = parseInt(process.env.PER_COLUMN || '22', 10);      // items shown per column
const PER_OUTLET = parseInt(process.env.PER_OUTLET || '3', 10);       // max items per outlet per column (diversity)

function loadFeeds() {
  return JSON.parse(fs.readFileSync(FEEDS_PATH, 'utf8'));
}

function parseDate(s) {
  const t = Date.parse(s || '');
  return Number.isNaN(t) ? 0 : t;
}

// In-memory wire cache
let WIRE = { columns: [], splash: null, updatedAt: null, building: false, sources: 0, sourcesOk: 0 };

async function buildColumn(colKey, col) {
  const results = await Promise.all(
    col.feeds.map(async (f) => {
      const r = await fetchFeed(f.url);
      return (r.items || []).slice(0, PER_FEED).map((it) => ({ ...it, outlet: f.outlet }));
    })
  );
  // flatten, dedupe by link, sort newest first
  const seen = new Set();
  let items = [];
  for (const arr of results) {
    for (const it of arr) {
      const id = it.link.split('#')[0].split('?')[0];
      if (seen.has(id)) continue;
      seen.add(id);
      items.push(it);
    }
  }
  items.sort((a, b) => parseDate(b.date) - parseDate(a.date));
  // Cap items per outlet so one source can't dominate a column (e.g. all-Apple on event day).
  const perOutlet = {};
  items = items.filter((it) => {
    perOutlet[it.outlet] = (perOutlet[it.outlet] || 0) + 1;
    return perOutlet[it.outlet] <= PER_OUTLET;
  });
  items = items.slice(0, PER_COLUMN);
  const okCount = results.filter((a) => a.length > 0).length;
  return { key: colKey, title: col.title, items, ok: okCount, total: col.feeds.length };
}

async function rebuild() {
  if (WIRE.building) return WIRE;
  WIRE.building = true;
  try {
    const feeds = loadFeeds();
    const cols = await Promise.all(
      Object.entries(feeds.columns).map(([k, c]) => buildColumn(k, c))
    );

    // Rewrite every headline into an original topic sentence (cached by URL).
    const flat = [];
    for (const c of cols) for (const it of c.items) flat.push(it);
    const rewritten = await rewriteAll(flat, { concurrency: 8 });
    // map back by link+title
    const byId = new Map(rewritten.map((r) => [r.link + '|' + r.title, r]));
    for (const c of cols) {
      c.items = c.items.map((it) => {
        const r = byId.get(it.link + '|' + it.title);
        return { outlet: it.outlet, link: it.link, date: it.date, topic: (r && r.topic) || it.title, image: it.image || '' };
      });
    }

    // "1 article per topic": dedupe near-identical stories so each topic appears once across the whole wire.
    const norm = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim().split(' ').slice(0, 10).join(' ');
    const seenTopic = new Set();
    for (const c of cols) {
      c.items = c.items.filter((it) => {
        const k = norm(it.topic);
        if (!k || seenTopic.has(k)) return false;
        seenTopic.add(k);
        return true;
      });
    }

    // Splash = newest image-bearing item across the US + World columns (fallback: newest overall).
    const splashPool = cols.filter((c) => c.key !== 'money_tech').flatMap((c) => c.items);
    splashPool.sort((a, b) => parseDate(b.date) - parseDate(a.date));
    const splash = splashPool.find((it) => it.image) || splashPool[0] || (cols[0] && cols[0].items[0]) || null;
    // Don't repeat the splash story inside its column.
    if (splash) for (const c of cols) c.items = c.items.filter((it) => it.link !== splash.link);

    WIRE = {
      columns: cols,
      splash,
      updatedAt: new Date().toISOString(),
      building: false,
      sources: cols.reduce((n, c) => n + c.total, 0),
      sourcesOk: cols.reduce((n, c) => n + c.ok, 0)
    };
  } catch (e) {
    console.error('[aggregate] rebuild failed', e.message);
    WIRE.building = false;
  }
  return WIRE;
}

function getWire() { return WIRE; }
function meta() { return loadFeeds(); }

const WIRE_JSON = path.join(__dirname, '..', 'data', 'wire.json');

// Persist the current wire to data/wire.json (used by scripts/build-wire.js).
function writeWireFile() {
  const w = getWire();
  const out = { columns: w.columns, splash: w.splash, updatedAt: w.updatedAt, sources: w.sources, sourcesOk: w.sourcesOk };
  fs.writeFileSync(WIRE_JSON, JSON.stringify(out));
  return out;
}

// Load a pre-built wire snapshot into memory (prod static mode — no Ollama needed).
function loadStaticWire() {
  try {
    const w = JSON.parse(fs.readFileSync(WIRE_JSON, 'utf8'));
    if (w && Array.isArray(w.columns)) {
      WIRE = { ...w, building: false };
      return true;
    }
  } catch (e) { /* file missing/invalid — leave WIRE as-is */ }
  return false;
}

module.exports = { rebuild, getWire, meta, writeWireFile, loadStaticWire, WIRE_JSON };