`scripts/short/fetch-stock.mjs`

130 lines
#!/usr/bin/env node
// fetch-stock.mjs — download ONE commercially-safe stock image per story for the Short's card
// backgrounds. Source: Openverse (no API key), filtered to license=cc0,pdm (public domain / CC0)
// so it's commercial-use safe with NO attribution required. Best-effort: always exits 0; a story
// with no usable image just falls back to the gradient card in render-short.js.
// Writes data/short/img/{intro,<beatN>,outro}.jpg. TK-11342 / TK-11343 (reduce template sameness).
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const DIR = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(DIR, '../..');
const DATA = path.join(ROOT, 'data', 'short');
const IMG = path.join(DATA, 'img');
const LOGO = path.join(DATA, 'logo');
const OPENVERSE = 'https://api.openverse.org/v1/images/';

const STOP = new Set(('a,an,the,of,to,in,on,for,and,or,but,after,before,as,at,by,with,from,into,over,under,is,are,was,were,' +
  'be,been,has,have,had,will,would,could,should,say,says,said,report,reports,reported,new,amid,than,that,this,these,those,' +
  'it,its,their,his,her,they,dozens,several,many,more,most,who,what,when,where,why,how,about,against,between,during,without,' +
  'first,second,two,three,four,five,near,off,out,up,down,per,via,plan,plans,set,gets,get').split(','));

function queryFor(headline) {
  const caps = (headline.match(/\b[A-Z][a-zA-Z]{2,}/g) || []).map((w) => w.toLowerCase()).filter((w) => !STOP.has(w));
  const words = headline.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3 && !STOP.has(w));
  const terms = [...new Set([...caps, ...words])].slice(0, 3);
  return terms.join(' ') || 'breaking news';
}

async function tfetch(url, opts = {}, ms = 9000) {
  const ac = new AbortController(); const to = setTimeout(() => ac.abort(), ms);
  try { return await fetch(url, { ...opts, signal: ac.signal, headers: { 'user-agent': 'allnewsdaily-short/1.0', ...(opts.headers || {}) } }); }
  finally { clearTimeout(to); }
}

async function downloadFirstUsable(results, outPath) {
  for (const it of results) {
    for (const src of [it.url, it.thumbnail]) {   // try original, then the Openverse-proxied thumbnail
      if (!src) continue;
      try {
        const r = await tfetch(src, {}, 10000);
        if (!r.ok) continue;
        const ct = (r.headers.get('content-type') || '').split(';')[0].trim();
        if (!/image\//.test(ct) || ct === 'image/svg+xml') continue; // skip svg — not a raster image; IM will
        // content-sniff it as vector and try to render any embedded <text> via its font engine, which can
        // crash the whole render on an unresolvable font-family (see TK-11458).
        const buf = Buffer.from(await r.arrayBuffer());
        if (buf.length < 6000) continue; // skip tiny/broken
        fs.writeFileSync(outPath, buf);
        return { ok: true, bytes: buf.length };
      } catch { /* try next src */ }
    }
  }
  return { ok: false };
}

// Rotating generic news imagery so a CC0-sparse headline still gets a (thematic) background,
// and consecutive fallbacks differ instead of repeating the same photo.
const GENERIC = ['news studio broadcast', 'city skyline', 'government capitol building', 'world globe earth',
  'newspaper headlines', 'crowd people street', 'flags government', 'stock market chart', 'satellite earth night'];

function domainOf(link) { try { return new URL(link).hostname.replace(/^www\./, ''); } catch { return null; } }
const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/x-icon': 'ico', 'image/vnd.microsoft.icon': 'ico', 'image/svg+xml': 'svg', 'image/webp': 'webp', 'image/gif': 'gif' };

// Each network's real logo: Clearbit (best) → DuckDuckGo icon proxy → Google favicons (always returns something).
async function grabLogo(domain, outBase) {
  if (!domain) return { ok: false, note: 'no domain' };
  const sources = [
    `https://logo.clearbit.com/${domain}?size=256`,
    `https://icons.duckduckgo.com/ip3/${domain}.ico`,
    `https://www.google.com/s2/favicons?sz=256&domain=${domain}`,
  ];
  for (const src of sources) {
    try {
      const r = await tfetch(src, {}, 8000);
      if (!r.ok) continue;
      const ct = (r.headers.get('content-type') || '').split(';')[0].trim();
      if (!/image\//.test(ct) || ct === 'image/svg+xml') continue; // skip svg (IM raster handling varies)
      const buf = Buffer.from(await r.arrayBuffer());
      if (buf.length < 400) continue;
      const out = `${outBase}.${EXT[ct] || 'png'}`;
      fs.writeFileSync(out, buf);
      return { ok: true, domain, via: src.split('/')[2], bytes: buf.length };
    } catch { /* next source */ }
  }
  return { ok: false, domain, note: 'no logo source hit' };
}

async function grabWithFallback(primaryQuery, outPath, idx) {
  let r = await grab(primaryQuery, outPath); if (r.ok) return { ...r, used: primaryQuery };
  const t1 = primaryQuery.split(' ')[0];
  if (t1 && t1 !== primaryQuery) { r = await grab(t1, outPath); if (r.ok) return { ...r, used: t1 }; }
  const g = GENERIC[idx % GENERIC.length];
  r = await grab(g, outPath); if (r.ok) return { ...r, used: g + ' (generic)' };
  r = await grab('news', outPath); return r.ok ? { ...r, used: 'news (generic)' } : { ok: false };
}

async function grab(query, outPath) {
  try {
    const url = `${OPENVERSE}?q=${encodeURIComponent(query)}&license=cc0,pdm&size=large&page_size=8&mature=false`;
    const r = await tfetch(url);
    if (!r.ok) return { ok: false, note: `openverse HTTP ${r.status}` };
    const j = await r.json();
    const res = await downloadFirstUsable(j.results || [], outPath);
    return res.ok ? { ok: true, query, ...res } : { ok: false, note: 'no usable result', query };
  } catch (e) { return { ok: false, note: e.message }; }
}

(async () => {
  const storiesDoc = JSON.parse(fs.readFileSync(path.join(DATA, 'stories.json'), 'utf8'));
  for (const dir of [IMG, LOGO]) { fs.rmSync(dir, { recursive: true, force: true }); fs.mkdirSync(dir, { recursive: true }); }

  const imgJobs = [];
  imgJobs.push(grabWithFallback('news studio broadcast', path.join(IMG, 'intro.jpg'), 0).then((r) => ['img:intro', r]));
  storiesDoc.stories.forEach((s, i) => imgJobs.push(grabWithFallback(queryFor(s.headline), path.join(IMG, `${s.n}.jpg`), i + 1).then((r) => [`img:beat${s.n}`, r])));
  imgJobs.push(grabWithFallback('world globe earth', path.join(IMG, 'outro.jpg'), 8).then((r) => ['img:outro', r]));

  // one real network logo per story (Clearbit → DuckDuckGo → Google favicons)
  const logoJobs = storiesDoc.stories.map((s) => grabLogo(domainOf(s.link), path.join(LOGO, String(s.n))).then((r) => [`logo:${s.outlet}`, r]));

  const results = await Promise.all([...imgJobs, ...logoJobs]);
  let img = 0, logo = 0;
  for (const [tag, r] of results) {
    if (r.ok) { if (tag.startsWith('logo')) logo++; else img++; }
    console.log(`  ${tag}: ${r.ok ? '✓ ' + (r.used || r.via || r.domain || '') : '✗ ' + (r.note || 'fail')}`);
  }
  console.log(`[fetch-stock] ${img} stock images (CC0/public-domain) + ${logo}/${storiesDoc.stories.length} network logos.`);
  process.exit(0); // best-effort — never block the pipeline
})().catch((e) => { console.error('[fetch-stock] non-fatal:', e.message); process.exit(0); });