`db/ingest-wikimedia-images.mjs`

126 lines
// Pulls cast headshots from Wikimedia ONLY when the per-file license is free
// for commercial reuse (PD / CC0 / CC-BY / CC-BY-SA). Rejects NC/ND/fair-use/
// non-free. Records file_url + license + attribution into the assets rights-ledger
// and links cast_people.headshot_asset_id. Never assumes a Commons file is free.
// Run: node db/ingest-wikimedia-images.mjs
import pg from "pg";

const DATABASE_URL =
  process.env.DATABASE_URL ?? "postgresql://localhost/nineoh_guide?host=/tmp";
const pool = new pg.Pool({ connectionString: DATABASE_URL });
const UA = "nineoh-guide/0.1 (unofficial fan project; contact: set-at-launch)";
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function fetchJson(url, tries = 4) {
  for (let i = 0; i < tries; i++) {
    try {
      const r = await fetch(url, { headers: { "User-Agent": UA } });
      const t = await r.text();
      return JSON.parse(t); // throws on rate-limit HTML → caught + retried
    } catch {
      await sleep(800 * (i + 1)); // backoff
    }
  }
  throw new Error("rate-limited after retries");
}

const FREE = /^(cc0|cc-by-\d|cc-by-sa-\d|pd|public domain)/i;
const NONFREE = /(nc|nd|non-free|fair use|copyright)/i;

function stripHtml(s) {
  return (s || "").replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
}

async function pageImageFile(name) {
  // Lead/infobox image filename + original URL from English Wikipedia.
  const u = `https://en.wikipedia.org/w/api.php?action=query&format=json&prop=pageimages&piprop=original|name&titles=${encodeURIComponent(
    name
  )}`;
  const j = await fetchJson(u);
  const pages = j.query?.pages ?? {};
  const page = Object.values(pages)[0];
  if (!page?.pageimage) return null;
  return { file: page.pageimage, url: page.original?.source ?? null };
}

async function commonsLicense(file) {
  const u = `https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(
    "File:" + file
  )}`;
  const j = await fetchJson(u);
  const pages = j.query?.pages ?? {};
  const page = Object.values(pages)[0];
  const info = page?.imageinfo?.[0];
  if (!info) return null;
  const m = info.extmetadata ?? {};
  return {
    license: (m.License?.value || "").toLowerCase(),
    licenseName: m.LicenseShortName?.value || "",
    licenseUrl: m.LicenseUrl?.value || "",
    artist: stripHtml(m.Artist?.value || m.Credit?.value || ""),
    url: info.url,
  };
}

// Only cast that don't already have a headshot — makes re-runs safe (no dup assets).
const cast = (
  await pool.query(`select id, name from cast_people where headshot_asset_id is null order by name`)
).rows;
let accepted = 0,
  skipped = 0;

for (const person of cast) {
  await sleep(700); // be polite to the Wikimedia API
  try {
    const img = await pageImageFile(person.name);
    if (!img) {
      console.log(`SKIP  ${person.name} — no Wikipedia lead image`);
      skipped++;
      continue;
    }
    const lic = await commonsLicense(img.file);
    if (!lic) {
      console.log(`SKIP  ${person.name} — no license metadata`);
      skipped++;
      continue;
    }
    const key = lic.license || lic.licenseName;
    const free = FREE.test(lic.license) || /public domain|cc0|cc by(?!-nc|-nd)/i.test(lic.licenseName);
    const blocked = NONFREE.test(lic.license) || /nc|nd|fair use/i.test(lic.licenseName);
    if (!free || blocked) {
      console.log(`SKIP  ${person.name} — license "${lic.licenseName || key}" not free-for-reuse`);
      skipped++;
      continue;
    }
    const licType = /cc0|public domain|^pd/i.test(key) ? "public-domain"
      : /sa/i.test(key) ? "cc-by-sa" : "cc-by";
    const attribution = `${lic.artist || "Unknown"} / ${lic.licenseName || key} (via Wikimedia Commons)`;
    const url = lic.url || img.url;

    const asset = await pool.query(
      `insert into assets (type, file_url, thumbnail_url, license_type, license_url,
                           attribution_text, usage_scope, source_url, rights_notes, approved_for_marketing)
       values ('image',$1,$1,$2,$3,$4,'editorial',$5,$6,false)
       returning id`,
      [
        url,
        licType,
        lic.licenseUrl || null,
        attribution,
        `https://commons.wikimedia.org/wiki/File:${img.file}`,
        `Wikimedia Commons file, license verified ${new Date().toISOString().slice(0, 10)}`,
      ]
    );
    await pool.query(`update cast_people set headshot_asset_id=$1 where id=$2`, [asset.rows[0].id, person.id]);
    console.log(`OK    ${person.name} — ${licType} (${lic.licenseName})`);
    accepted++;
  } catch (e) {
    console.log(`SKIP  ${person.name} — error ${e.message}`);
    skipped++;
  }
}

console.log(`\naccepted ${accepted}, skipped ${skipped} of ${cast.length}`);
console.log("Only free-for-reuse licenses stored; each asset carries attribution + source.");
await pool.end();