`scripts/match-shadowman-articles.mjs`

217 lines
#!/usr/bin/env node
// match-shadowman-articles.mjs — link each APPROVED Shadow Man cartoon to its best-fitting P24 article
// (TK-12247). $0: keyword shortlist + a LOCAL model judge (Ollama or the MLX server). No paid APIs.
//
// Usage: node scripts/match-shadowman-articles.mjs [--p24=~/Projects/crazy-news-channel] [--dry-run]
//        [--min-score=4] [--shortlist=8] [--relink-manual]
//
// For each approved piece: shortlist the top N stories by keyword overlap (title+caption+scene+theme vs
// headline+summary+tags, outlet-name tags ignored), then ask the local model to pick ONE story id from
// that shortlist or NONE, with a 1-5 fit score and a one-line reason (JSON). A link is written only when
// score >= min-score AND the pick is in the shortlist. Stories that already carry a non-Shadow-Man
// cartoon are excluded (index.html shows the FIRST manifest cartoon per story, so a Shadow Man piece
// there would never appear), and two pieces never share a story (higher score wins, the other is
// re-judged without it). Pieces linked by hand in Inkwell (match_by:"manual") are left alone unless
// --relink-manual. If no local model responds the run reports NOT-MEASURED, writes nothing, exits 3.
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { loadStories, readManifest, storyUrl, storySourceLabel, p24Dir } from './lib-stories.mjs';

const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const args = Object.fromEntries(process.argv.slice(2).map(a => { const m = a.match(/^--([^=]+)=(.*)$/); return m ? [m[1], m[2]] : [a.replace(/^--/, ''), true]; }));
const P24 = p24Dir(args.p24);
const MAIN_P24 = p24Dir('~/Projects/crazy-news-channel');
const MIN = Number(args['min-score'] || 4);
const SHORT = Number(args.shortlist || 8);
const DATA = path.join(ROOT, 'data', 'shadowman.json');

const ENGINES = [
  { name: 'ollama', model: 'hf.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF:Q4_K_M', url: 'http://127.0.0.1:11434/api/chat' },
  { name: 'mlx', model: 'mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit', url: 'http://127.0.0.1:8000/v1/chat/completions' },
];

async function callEngine(e, messages, timeoutMs) {
  const body = e.name === 'ollama'
    ? { model: e.model, messages, stream: false, format: 'json', options: { temperature: 0, num_predict: 200 } }
    : { model: e.model, messages, temperature: 0, max_tokens: 200 };
  const r = await fetch(e.url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs) });
  if (!r.ok) throw new Error(`${e.name} HTTP ${r.status}`);
  const j = await r.json();
  const text = e.name === 'ollama' ? j.message && j.message.content : j.choices && j.choices[0] && j.choices[0].message && j.choices[0].message.content;
  if (typeof text !== 'string') throw new Error(`${e.name}: no content`);
  return text;
}

async function pickEngine() {
  for (const e of ENGINES) {
    try {
      const t = await callEngine(e, [{ role: 'user', content: 'Reply with exactly this JSON: {"ok":true}' }], 120000);
      if (/"ok"\s*:\s*true/.test(t)) return e;
    } catch (err) { console.error(`  engine ${e.name} unavailable: ${err.message}`); }
  }
  return null;
}

const STOP = new Set('the a an and or of to in on at for with by from as is are was were be been it its this that these those his her their our your my he she they we you i me him them who whom what which while into onto over under out up down off about than then so but not no yes all any each every some one two very just only also too more most much many such own same other another here there when where why how can will would should could may might do does did done has have had having being after before again further once per via own while like said says new latest news report reports amid says'.split(/\s+/));
const stem = (w) => w.replace(/(ies)$/, 'y').replace(/(es|s)$/, '').replace(/(ing|ed)$/, '');
function tokens(text) {
  return new Set(String(text || '').toLowerCase().replace(/[’']/g, '').split(/[^a-z0-9]+/).filter(w => w.length >= 3 && !STOP.has(w)).map(stem).filter(w => w.length >= 3));
}

// Outlet names never count as topic words (a tag like "the new york times" says nothing about fit).
function outletNames(stories) {
  const s = new Set();
  for (const st of stories) if (st.sourceName) s.add(st.sourceName.toLowerCase());
  return s;
}

function storyTokens(st, outlets) {
  const tags = (st.tags || []).filter(t => !outlets.has(String(t).toLowerCase()));
  return tokens([st.headline, st.summary.replace(/^Via [^,]+,.*$/i, ''), tags.join(' '), tags.join(' ')].join(' '));
}

function pieceText(p) {
  const scene = String(p.scene || '').replace(/,\s*(1960s|1970s|contemporary)[^,]*props.*$/i, '');
  return [p.title, p.caption, scene, p.theme].join(' ');
}

// Theme / motif words -> the vocabulary P24 stories actually use (tags + headline words), so the keyword
// shortlist can surface e.g. a Senate-race story for an elephant-vs-donkey cartoon. Shortlist only;
// the local model still makes the call.
const EXPAND = {
  'Elections & party politics': 'elections election campaign senate race congress democrats republicans voters political',
  'Corruption & money in politics': 'lobbying lobbyist donors campaign congress ethics political interference money',
  'War & militarism': 'military conflict geopolitics attack airstrikes missiles war nato defense',
  'Presidential power & scandal': 'white house president trump political interference scandal review',
  'Local LA & California politics': 'city council infrastructure bureaucracy road traffic parking commute budget',
  'Religion & politics': 'religion church faith charity donated',
  'Guns & violence': 'crime guns shooting stabbing violence',
  'Memorials & tributes': 'memorial tribute death history',
};
const MOTIF = { pothole: 'road infrastructure repaving', traffic: 'infrastructure commute road', gridlock: 'infrastructure commute', shredding: 'documents records', document: 'documents records', leak: 'white house reporter access', lobbyist: 'lobbying', donor: 'donors campaign', congress: 'congress senate', baby: 'campaign', district: 'elections race', map: 'elections race', elephant: 'republicans elections', donkey: 'democrats elections', tank: 'military', war: 'military conflict', plate: 'donated charity' };
function expandedText(p) {
  const base = pieceText(p);
  const extra = [EXPAND[p.theme] || ''];
  for (const [k, v] of Object.entries(MOTIF)) if (new RegExp(`\\b${k}`, 'i').test(base)) extra.push(v);
  return base + ' ' + extra.join(' ');
}

function shortlist(piece, pool, outlets) {
  const pt = tokens(expandedText(piece));
  return pool.map(st => {
    const tt = storyTokens(st, outlets);
    let score = 0;
    for (const w of pt) if (tt.has(w)) score += 1;
    return { st, score };
  }).sort((a, b) => b.score - a.score || (Date.parse(b.st.publishedAt || 0) || 0) - (Date.parse(a.st.publishedAt || 0) || 0))
    .slice(0, SHORT).map(x => ({ ...x.st, kw: x.score }));
}

function prompt(piece, cands) {
  const lines = cands.map(c => `- id: ${c.id}\n  headline: ${c.headline}\n  summary: ${c.summary.slice(0, 220)}\n  tags: ${(c.tags || []).join(', ')}`).join('\n');
  return [
    { role: 'system', content: 'You are a newspaper editor placing a single-panel editorial cartoon next to the news article it best comments on. Be strict: a cartoon fits only if a reader would immediately see it as commentary on that specific story. Topical overlap alone (both mention politics) is a weak fit. Answer with JSON only.' },
    { role: 'user', content: `CARTOON\ntitle: ${piece.title}\ncaption: ${piece.caption}\nscene: ${piece.scene}\ntheme: ${piece.theme}\n\nCANDIDATE ARTICLES\n${lines}\n\nPick the ONE candidate article this cartoon best comments on, or NONE if none fits. Score fit 1-5 (5 = the cartoon is clearly about this story, 4 = strong natural fit, 3 = loose/thematic only, 1-2 = poor). Reply with JSON exactly like {"story_id":"<candidate id or NONE>","score":<1-5>,"reason":"<one short sentence>"}` },
  ];
}

function parseJson(text) {
  const i = text.indexOf('{');
  const j = text.lastIndexOf('}');
  if (i < 0 || j < i) throw new Error('no JSON object in model reply');
  return JSON.parse(text.slice(i, j + 1));
}

async function judge(engine, piece, cands) {
  let last;
  for (let attempt = 0; attempt < 2; attempt++) {
    try {
      const out = parseJson(await callEngine(engine, prompt(piece, cands), 300000));
      const sid = String(out.story_id || 'NONE').trim();
      const score = Math.max(1, Math.min(5, Math.round(Number(out.score) || 1)));
      return { story_id: sid, score, reason: String(out.reason || '').replace(/\s+/g, ' ').trim().slice(0, 240) };
    } catch (err) { last = err; }
  }
  throw last;
}

(async () => {
  const doc = JSON.parse(fs.readFileSync(DATA, 'utf8'));
  const stories = loadStories(P24);
  const outlets = outletNames(stories);
  // Stories already carrying a non-Shadow-Man cartoon in the target OR the main checkout's manifest.
  const taken = new Set();
  for (const d of new Set([P24, MAIN_P24])) for (const c of readManifest(d)) if (c.story_id && !String(c.id).startsWith('shadowman-')) taken.add(c.story_id);
  const pool = stories.filter(s => !taken.has(s.id));
  console.log(`stories: ${stories.length} total, ${taken.size} already have a cartoon, ${pool.length} eligible (from ${path.relative(process.env.HOME, P24)})`);

  const engine = await pickEngine();
  if (!engine) {
    console.log('NOT-MEASURED: no local model responded (ollama :11434, mlx :8000). Nothing linked, nothing written.');
    process.exit(3);
  }
  console.log(`judge: ${engine.name} ${engine.model} ($0 local)`);

  const manualIds = new Set(doc.items.filter(i => i.match_by === 'manual' && i.story_id && !args['relink-manual']).map(i => i.story_id));
  const targets = doc.items.filter(i => i.status === 'approved' && !(i.match_by === 'manual' && !args['relink-manual']));
  const results = new Map();
  const judgeOne = async (piece, exclude) => {
    const cands = shortlist(piece, pool.filter(s => !exclude.has(s.id) && !manualIds.has(s.id)), outlets);
    const t0 = Date.now();
    const v = await judge(engine, piece, cands);
    const inList = cands.find(c => c.id === v.story_id);
    const linked = !!inList && v.score >= MIN;
    return { piece, cands, verdict: v, story: linked ? inList : null, secs: ((Date.now() - t0) / 1000).toFixed(1), note: !inList && v.story_id !== 'NONE' ? `pick "${v.story_id}" not in shortlist` : '' };
  };
  for (const piece of targets) {
    const r = await judgeOne(piece, new Set());
    results.set(piece.id, r);
    console.log(`  ${piece.title} -> ${r.verdict.story_id} (${r.verdict.score}) ${r.secs}s`);
  }
  // Resolve collisions: two pieces on one story -> the higher score keeps it, the other is re-judged without it.
  for (let round = 0; round < 3; round++) {
    const byStory = new Map();
    for (const r of results.values()) if (r.story) (byStory.get(r.story.id) || byStory.set(r.story.id, []).get(r.story.id)).push(r);
    const losers = [];
    for (const list of byStory.values()) if (list.length > 1) { list.sort((a, b) => b.verdict.score - a.verdict.score); losers.push(...list.slice(1)); }
    if (!losers.length) break;
    const claimed = new Set([...byStory.keys()]);
    for (const l of losers) {
      const r = await judgeOne(l.piece, claimed);
      r.note = `re-judged: ${l.story.id} went to a higher-scoring piece`;
      results.set(l.piece.id, r);
      console.log(`  (collision) ${l.piece.title} -> ${r.verdict.story_id} (${r.verdict.score})`);
    }
  }

  const now = new Date().toISOString();
  const table = [];
  for (const piece of targets) {
    const r = results.get(piece.id);
    const s = r.story;
    Object.assign(piece, {
      story_id: s ? s.id : null,
      story_title: s ? s.headline : null,
      story_url: s ? storyUrl(s) : null,
      story_source: s ? storySourceLabel(s) : null,
      match_score: r.verdict.score,
      match_reason: r.verdict.reason + (r.note ? ` [${r.note}]` : '') + (!s && r.verdict.story_id !== 'NONE' && !r.note ? ` [best pick ${r.verdict.story_id} scored below ${MIN}]` : ''),
      match_by: 'auto',
      match_model: `${engine.name}:${engine.model}`,
      matched_at: now,
    });
    table.push({ id: piece.id, title: piece.title, story_id: piece.story_id, story: piece.story_title, score: r.verdict.score, pick: r.verdict.story_id, reason: piece.match_reason, shortlist: r.cands.map(c => `${c.id}(${c.kw})`).join(' ') });
  }
  console.log('\nMATCH TABLE');
  for (const t of table) console.log(`- ${t.title}\n    ${t.story_id ? `LINKED ${t.story_id} — ${t.story}` : `NONE (model pick: ${t.pick})`}  score ${t.score}\n    reason: ${t.reason}\n    shortlist: ${t.shortlist}`);
  const n = table.filter(t => t.story_id).length;
  console.log(`\n${n}/${table.length} linked (min score ${MIN}); ${doc.items.filter(i => i.match_by === 'manual').length} manual link(s) left untouched`);
  if (args['dry-run']) { console.log('--dry-run: data/shadowman.json NOT written'); return; }
  const tmp = DATA + '.tmp';
  fs.writeFileSync(tmp, JSON.stringify(doc, null, 2) + '\n');
  fs.renameSync(tmp, DATA);
  fs.appendFileSync(path.join(ROOT, 'data', 'shadowman-actions.jsonl'), JSON.stringify({ ts: now, collection: 'shadowman', action: 'auto-link', engine: `${engine.name}:${engine.model}`, linked: table.filter(t => t.story_id).map(t => [t.id, t.story_id]) }) + '\n');
  console.log('wrote data/shadowman.json');
})().catch(e => { console.error('FAIL', e); process.exit(1); });