`scripts/fetch-real-news.mjs`

243 lines
#!/usr/bin/env node
// fetch-real-news.mjs — pulls REAL current headlines from Google News RSS
// (no API key, no cost) and writes them into real-news-data.js as
// `window.P24_REAL_STORIES = [...]`, alongside (not replacing) the invented satire in
// previously lived there. index.html is a static file with no server, so
// it can't fetch news.google.com itself (CORS) — this script is the
// offline refresh step Steve (or a cron) runs locally; the page just loads
// whatever stories-data.js was last written with.
//
// Run: node scripts/fetch-real-news.mjs
//
// Each RSS item becomes a SINGLE-STAGE story (stageIndex 0, one entry in
// `stages`) on purpose — real news doesn't escalate into Onion-style
// absurdity, so giving it only one stage means tick()'s stage-advance loop
// treats it as already at its final stage and never touches it again.
//
// HARD RULE (TK-12165, 2026-09-24): every entry this writes MUST carry a
// non-empty `tags` array — see toStory()'s call into
// scripts/lib/auto-tag-real-news.mjs, the one shared place both this
// fetcher and the one-time backfill read the tag vocabulary from.

import { writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { autoTagRealNews } from "./lib/auto-tag-real-news.mjs";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUT_FILE = path.join(__dirname, "..", "real-news-data.js");
const ITEMS_PER_CATEGORY = 6;
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";

// category key (matches index.html's CATEGORY_LABELS / MOODS) -> RSS source.
// Google News has no dedicated topic feed for politics/weather, so those
// use search RSS instead of a /section/topic/<TOPIC> feed.
const CATEGORY_LABELS = {
  politics: "Politics",
  weather: "Weather",
  scitech: "Sci-Tech",
  sports: "Sports",
  entertainment: "Entertainment",
  business: "Business",
  uncategorized: "Uncategorized",
};

const FEEDS = {
  politics: { kind: "search", q: "politics" },
  weather: { kind: "search", q: "weather" },
  scitech: { kind: "topic", topic: "TECHNOLOGY" },
  sports: { kind: "topic", topic: "SPORTS" },
  entertainment: { kind: "topic", topic: "ENTERTAINMENT" },
  business: { kind: "topic", topic: "BUSINESS" },
  uncategorized: { kind: "topic", topic: "WORLD" },
};

function feedURL(spec) {
  const base = "https://news.google.com/rss";
  const tail = "hl=en-US&gl=US&ceid=US:en";
  return spec.kind === "topic"
    ? `${base}/headlines/section/topic/${spec.topic}?${tail}`
    : `${base}/search?q=${encodeURIComponent(spec.q)}&${tail}`;
}

// --- tiny dependency-free XML/HTML helpers -------------------------------

const ENTITIES = {
  amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", "#39": "'",
};
function decodeEntities(str) {
  return String(str || "").replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (m, ent) => {
    if (ent[0] === "#") {
      const code = ent[1] === "x" || ent[1] === "X"
        ? parseInt(ent.slice(2), 16)
        : parseInt(ent.slice(1), 10);
      return Number.isFinite(code) ? String.fromCodePoint(code) : m;
    }
    return ENTITIES[ent] !== undefined ? ENTITIES[ent] : m;
  });
}
// Google's <description> is XML-entity-encoded HTML (e.g. the literal text
// "&lt;a href=...&gt;Title&lt;/a&gt;&amp;nbsp;&amp;nbsp;&lt;font&gt;Source
// &lt;/font&gt;"), so entities must be decoded BEFORE tags are stripped —
// stripping first finds no literal "<...>" to match (they're still escaped
// as "&lt;"/"&gt;") and leaves raw HTML sitting in the "stripped" text.
// &nbsp; is itself double-escaped ("&amp;nbsp;"), so a second decode pass
// after the tags come out catches what the first pass turned into "&nbsp;".
function stripTags(raw) {
  const onceDecoded = decodeEntities(String(raw || ""));
  // Google's topic-feed <description> is a "further coverage" <ol><li>
  // list of several related headline+source pairs — insert a bullet at
  // each <li> instead of collapsing the whole list into one run-on line.
  const withBullets = onceDecoded.replace(/<li[^>]*>/gi, " • ");
  const noTags = withBullets.replace(/<[^>]*>/g, " ");
  return decodeEntities(noTags).replace(/\s+/g, " ").replace(/^\s*•\s*/, "").trim();
}
function tag(block, name) {
  const m = block.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`, "i"));
  return m ? m[1] : "";
}
function cdataOrText(raw) {
  const m = raw.match(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/);
  return decodeEntities(m ? m[1] : raw).trim();
}

function parseItems(xml) {
  const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => m[1]);
  return items.map((block) => {
    const rawTitle = cdataOrText(tag(block, "title"));
    const link = cdataOrText(tag(block, "link"));
    const pubDate = cdataOrText(tag(block, "pubDate"));
    const sourceMatch = block.match(/<source[^>]*>([\s\S]*?)<\/source>/i);
    const sourceName = sourceMatch ? decodeEntities(sourceMatch[1]).trim() : "";
    const description = stripTags(tag(block, "description"));
    return { rawTitle, link, pubDate, sourceName, description };
  });
}

// Google appends " - <Source Name>" to every item title. Strip it using the
// <source> tag (ground truth) so a headline that legitimately contains
// " - " isn't mangled; falls back to trimming the last " - X" segment only
// when no <source> tag was present.
function cleanTitle(rawTitle, sourceName) {
  if (sourceName && rawTitle.endsWith(` - ${sourceName}`)) {
    return rawTitle.slice(0, -(sourceName.length + 3)).trim();
  }
  const idx = rawTitle.lastIndexOf(" - ");
  return idx > 0 ? rawTitle.slice(0, idx).trim() : rawTitle.trim();
}

function formatPublished(pubDate) {
  const d = new Date(pubDate);
  if (Number.isNaN(d.getTime())) return "";
  return d.toLocaleString("en-US", {
    month: "short", day: "numeric", year: "numeric", hour: "numeric", minute: "2-digit",
  });
}

async function fetchFeed(categoryKey, spec) {
  const url = feedURL(spec);
  const res = await fetch(url, { headers: { "User-Agent": UA, Accept: "application/rss+xml, application/xml, text/xml" } });
  if (!res.ok) throw new Error(`${categoryKey}: HTTP ${res.status} fetching ${url}`);
  const xml = await res.text();
  return parseItems(xml);
}

function toStory(categoryKey, n, raw) {
  const headline = cleanTitle(raw.rawTitle, raw.sourceName);
  const sourceName = raw.sourceName || "wire services";
  // Google News RSS descriptions are not real article summaries — every
  // item's <description> is just its own headline re-wrapped in an <a>
  // plus the source name (verified live 2026-09-24 across both topic and
  // search feeds), so "<headline> <sourceName>" carries zero information
  // beyond what's already in the headline/byline. Strip that same trailing
  // "<sourceName>" off the stripped description before comparing, so a
  // dek is only kept when there's real extra text left over.
  const strippedDesc = raw.description;
  const descWithoutSource = sourceName && strippedDesc.endsWith(sourceName)
    ? strippedDesc.slice(0, -sourceName.length).trim()
    : strippedDesc;
  const dek = descWithoutSource && descWithoutSource !== headline ? descWithoutSource : "";
  const published = formatPublished(raw.pubDate);
  const detail = dek || `Via ${sourceName}${published ? `, ${published}` : ""}.`;
  const paragraphs = [
    detail,
    `Read the full story at ${sourceName}.`,
  ];

  const story = {
    id: `real-${categoryKey}-${n}`,
    category: categoryKey,
    categoryLabel: CATEGORY_LABELS[categoryKey],
    location: "",
    byline: sourceName,
    image: null,
    sourceUrl: raw.link,
    sourceName,
    publishedLabel: published,
    intervalSec: 999999,
    offsetSec: 0,
    stageIndex: 0,
    article: {
      dek,
      photoCaption: "",
      paragraphs,
    },
    // A single stage: stageIndex (0) === stages.length - 1 (0) is true from
    // the moment this loads, so tick()'s `alreadyMax` check holds it there
    // forever — real news never escalates into a satirical spiral.
    stages: [
      { headline, detail },
    ],
  };
  // HARD RULE (TK-12165): every real-news entry MUST carry a non-empty
  // tags array. autoTagRealNews() (scripts/lib/auto-tag-real-news.mjs) is
  // deterministic and always returns 3-6 tags (category + outlet + keyword
  // matches, padded with a category fallback if nothing else matched), so
  // this can never write an untagged entry.
  story.tags = autoTagRealNews(story);
  return story;
}

async function main() {
  const categories = Object.keys(FEEDS);
  const all = [];
  const summary = [];

  for (const categoryKey of categories) {
    const spec = FEEDS[categoryKey];
    let items;
    try {
      items = await fetchFeed(categoryKey, spec);
    } catch (err) {
      console.error(`✗ ${categoryKey}: ${err.message}`);
      summary.push({ categoryKey, count: 0, error: err.message });
      continue;
    }
    const picked = items.slice(0, ITEMS_PER_CATEGORY);
    picked.forEach((raw, i) => all.push(toStory(categoryKey, i + 1, raw)));
    console.log(`✓ ${categoryKey}: ${picked.length} stories from ${feedURL(spec)}`);
    summary.push({ categoryKey, count: picked.length });
  }

  if (!all.length) {
    console.error("No stories fetched from any category — aborting write so stories-data.js is left untouched.");
    process.exit(1);
  }

  const header =
    "// AUTO-GENERATED by scripts/fetch-real-news.mjs — real headlines pulled from\n" +
    "// Google News RSS (no API key, $0). Do not hand-edit; re-run the script to\n" +
    "// refresh. Each story is single-stage (real news doesn't escalate) and links\n" +
    `// out to its real source article. Generated ${new Date().toISOString()}.\n`;
  const body = `window.P24_REAL_STORIES = ${JSON.stringify(all, null, 1)};\n`;
  await writeFile(OUT_FILE, header + body, "utf8");

  console.log(`\nWrote ${all.length} real stories to ${OUT_FILE}`);
  console.log("Per category:", summary.map((s) => `${s.categoryKey}=${s.count}`).join(", "));
}

main().catch((err) => {
  console.error("fetch-real-news.mjs failed:", err);
  process.exit(1);
});