`scripts/short/pick-stories.js`

160 lines
#!/usr/bin/env node
/**
 * STAGE 1 — pick-stories.js  (TK-11342)
 * Read data/wire.json -> select 6 stories (splash first, then strongest column
 * items across the 3 columns), dedupe by outlet (max 2/outlet), skip topics
 * that are empty or < 25 chars. Write data/short/stories.json per CONTRACTS.md
 * and print the 6 chosen headlines.
 *
 * NOTE (verified 2026-09-09): wire.json items have NO `title` field —
 * `item.topic` IS the headline sentence.
 *
 * Cody/DTD red-team hardening (2026-09-09):
 *  - FRESHNESS GUARD (#1): refuse to run if wire.updatedAt is > MAX_STALE_MIN old,
 *    so a dead feed-refresh loop can never render stale news as "today's briefing"
 *    (the silent-wrong-success failure). Exit code 2. Override with --allow-stale.
 *  - CROSS-DAY DEDUPE (#1): exclude links aired in the last HISTORY_DAYS days
 *    (data/short/history.json, written by the orchestrator after a successful
 *    upload). Relaxes to refill if a slow-news day leaves < TARGET eligible.
 *  - DEFENSIVE ENTITY DECODE (#10b): decode HTML entities in headlines so a feed
 *    regression can't put "Apple &amp;amp; Google" on a public card.
 */
'use strict';
const fs = require('fs');
const path = require('path');

const ROOT = path.resolve(__dirname, '..', '..');
const WIRE = path.join(ROOT, 'data', 'wire.json');
const OUT_DIR = path.join(ROOT, 'data', 'short');
const OUT = path.join(OUT_DIR, 'stories.json');
const HISTORY = path.join(OUT_DIR, 'history.json');

const MIN_TOPIC_LEN = 25;
const MAX_PER_OUTLET = 2;
const TARGET = 6;
const MAX_STALE_MIN = Number(process.env.AND_MAX_STALE_MIN || 30); // reject feeds older than this
const HISTORY_DAYS = Number(process.env.AND_HISTORY_DAYS || 2);    // don't re-air links aired in last N days
const ALLOW_STALE = process.argv.includes('--allow-stale');

const ENT = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&#39;': "'", '&apos;': "'" };
function decodeEntities(s) {
  return s
    .replace(/&(amp|lt|gt|quot|#39|apos);/g, (m) => ENT[m])
    .replace(/&#(\d+);/g, (_, d) => { try { return String.fromCodePoint(+d); } catch { return _; } })
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => { try { return String.fromCodePoint(parseInt(h, 16)); } catch { return _; } });
}
function clean(s) {
  return decodeEntities(s == null ? '' : String(s)).replace(/\s+/g, ' ').trim();
}

function dateStamp(iso) {
  const d = iso ? new Date(iso) : new Date();
  const use = isNaN(d.getTime()) ? new Date() : d;
  return use.toISOString().slice(0, 10); // YYYY-MM-DD
}

// Links aired in the last HISTORY_DAYS days (the orchestrator appends after a real upload).
function loadRecentLinks() {
  try {
    const h = JSON.parse(fs.readFileSync(HISTORY, 'utf8'));
    const cutoff = new Date(Date.now() - HISTORY_DAYS * 86400000).toISOString().slice(0, 10);
    const set = new Set();
    (Array.isArray(h) ? h : []).forEach((d) => {
      if (d && d.date >= cutoff) (d.links || []).forEach((l) => set.add(l));
    });
    return set;
  } catch { return new Set(); }
}

function main() {
  if (!fs.existsSync(WIRE)) {
    console.error(`[pick-stories] FATAL: wire.json not found at ${WIRE}`);
    process.exit(1);
  }
  const wire = JSON.parse(fs.readFileSync(WIRE, 'utf8'));

  // --- FRESHNESS GUARD (#1) ---------------------------------------------------
  const upd = Date.parse(wire.updatedAt);
  if (!ALLOW_STALE) {
    if (!Number.isFinite(upd)) {
      console.error(`[pick-stories] FATAL: wire.json has no parseable updatedAt — cannot confirm freshness. Refusing to publish. (override: --allow-stale)`);
      process.exit(2);
    }
    const ageMin = (Date.now() - upd) / 60000;
    if (ageMin > MAX_STALE_MIN) {
      console.error(`[pick-stories] FATAL: wire.json is ${ageMin.toFixed(0)}min stale (> ${MAX_STALE_MIN}min) — the feed-refresh loop is likely dead. Refusing to render stale news as today's briefing. (override: --allow-stale)`);
      process.exit(2);
    }
  }

  const recentLinks = loadRecentLinks();
  const splash = wire.splash || null;
  const columns = Array.isArray(wire.columns) ? wire.columns : [];

  let allowRepeats = false; // flipped to refill if strict pass can't reach TARGET
  const picked = [];
  const outletCount = Object.create(null);
  const seenLinks = new Set();

  const outletKey = (o) => clean(o).toLowerCase();
  function eligible(item) {
    if (!item) return false;
    const topic = clean(item.topic);
    if (topic.length < MIN_TOPIC_LEN) return false;                     // skip short/empty
    const link = clean(item.link);
    if (link && seenLinks.has(link)) return false;                     // dedupe within this run
    if (!allowRepeats && link && recentLinks.has(link)) return false;  // cross-day dedupe (#1)
    const ok = outletKey(item.outlet);
    if (ok && (outletCount[ok] || 0) >= MAX_PER_OUTLET) return false;  // <=2/outlet
    return true;
  }
  function take(item, tag) {
    const ok = outletKey(item.outlet);
    if (ok) outletCount[ok] = (outletCount[ok] || 0) + 1;
    const link = clean(item.link);
    if (link) seenLinks.add(link);
    picked.push({ n: picked.length + 1, headline: clean(item.topic), outlet: clean(item.outlet), link, tag: clean(tag) || 'Top' });
  }

  function selectPass() {
    if (splash && eligible(splash)) take(splash, 'Top');
    const cursors = columns.map(() => 0);
    let progressed = true;
    while (picked.length < TARGET && progressed) {
      progressed = false;
      for (let c = 0; c < columns.length && picked.length < TARGET; c++) {
        const items = Array.isArray(columns[c].items) ? columns[c].items : [];
        while (cursors[c] < items.length) {
          const item = items[cursors[c]++];
          progressed = true;
          if (eligible(item)) { take(item, columns[c].title); break; }
        }
      }
    }
  }

  selectPass();
  // Refill: if cross-day dedupe left us short on a slow-news day, allow repeats.
  if (picked.length < TARGET && recentLinks.size) {
    console.warn(`[pick-stories] WARN: only ${picked.length}/${TARGET} fresh (non-repeat) stories — relaxing cross-day dedupe to refill.`);
    allowRepeats = true;
    selectPass();
  }

  const outDate = dateStamp(wire.updatedAt);
  const result = { date: outDate, generatedAt: new Date().toISOString(), stories: picked };
  fs.mkdirSync(OUT_DIR, { recursive: true });
  fs.writeFileSync(OUT, JSON.stringify(result, null, 2));

  console.log(`[pick-stories] wrote ${picked.length} stories -> ${path.relative(ROOT, OUT)} (date ${outDate}, feed ${Number.isFinite(upd) ? ((Date.now() - upd) / 60000).toFixed(0) + 'min old' : 'age?'})`);
  if (picked.length < TARGET) console.warn(`[pick-stories] WARN: only ${picked.length}/${TARGET} eligible stories found`);
  picked.forEach((s) => console.log(`  ${s.n}. ${s.headline} — via ${s.outlet}  [${s.tag}]`));
  return result;
}

if (require.main === module) {
  try { main(); }
  catch (e) { console.error('[pick-stories] FATAL:', (e && e.stack) || e); process.exit(1); }
}
module.exports = { main };