`scrapers/wikimedia-credit-backfill.js`

145 lines
#!/usr/bin/env node
'use strict';
// Backfill image credit + license + source URL for every asim_movies poster
// and asim_people headshot we previously imported from Wikimedia Commons.
//
// Per Steve's fan-art credit policy (memory: feedback_fan_art_credit_policy):
// every fan-art / non-PD image must carry artist + license + source URL or it
// cannot be displayed. We have 783 movie posters + a handful of headshots
// already in place without that metadata. This script fetches it.
//
// Commons API: action=query, titles=File:..., prop=imageinfo,
// iiprop=extmetadata returns Artist, LicenseShortName, LicenseUrl per file.
// Batches up to 50 files per request.

require('dotenv').config();
const { Pool } = require('pg');

const pool = new Pool({
  connectionString: process.env.DATABASE_URL || 'postgresql:///dw_unified?host=/tmp&user=stevestudio2',
  max: 4,
});

const COMMONS_API = 'https://commons.wikimedia.org/w/api.php';
const UA = 'AsSeenInMovies/0.1 (https://asseeninmovies.com; steve@designerwallcoverings.com)';
const BATCH = 50;
const SLEEP_MS = 1100;
const sleep = (ms) => new Promise(r => setTimeout(r, ms));

function fileNameFromCommonsUrl(url) {
  // https://commons.wikimedia.org/wiki/Special:FilePath/Goosebumps%202023%20poster.jpg?width=600
  //                                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  if (!url) return null;
  const m = url.match(/Special:FilePath\/([^?]+)/i);
  if (!m) return null;
  try { return decodeURIComponent(m[1]).replace(/_/g, ' '); }
  catch { return m[1].replace(/_/g, ' '); }
}

function stripHtml(s) {
  if (!s) return null;
  // Commons returns "Artist" wrapped in <a>...</a> HTML
  return String(s).replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').trim();
}

async function commonsImageInfo(filenames) {
  const titles = filenames.map(n => `File:${n}`).join('|');
  const params = new URLSearchParams({
    action: 'query',
    titles,
    prop: 'imageinfo',
    iiprop: 'extmetadata|url|user',
    format: 'json',
    formatversion: '2',
  });
  const r = await fetch(`${COMMONS_API}?${params}`, { headers: { 'User-Agent': UA } });
  if (!r.ok) throw new Error(`commons ${r.status}`);
  const j = await r.json();
  const out = new Map();
  const pages = j.query?.pages || [];
  for (const p of pages) {
    if (!p.imageinfo || !p.imageinfo[0]) continue;
    const ii = p.imageinfo[0];
    const ext = ii.extmetadata || {};
    const title = p.title.replace(/^File:/, '');
    out.set(title, {
      artist:  stripHtml(ext.Artist?.value)            || ii.user || null,
      license: ext.LicenseShortName?.value             || ext.License?.value || null,
      licenseUrl: ext.LicenseUrl?.value                || null,
      descriptionUrl: ii.descriptionurl                || `https://commons.wikimedia.org/wiki/File:${encodeURIComponent(title)}`,
    });
  }
  return out;
}

async function backfillMovies() {
  const sel = await pool.query(`
    SELECT id, poster_url FROM asim_movies
     WHERE poster_source='wikimedia-cc' AND poster_credit IS NULL
     ORDER BY id LIMIT 2000`);
  console.log(`[backfill-movies] ${sel.rows.length} rows`);
  let ok = 0, miss = 0;
  for (let i = 0; i < sel.rows.length; i += BATCH) {
    const slice = sel.rows.slice(i, i + BATCH);
    const idToFile = new Map();
    for (const r of slice) {
      const f = fileNameFromCommonsUrl(r.poster_url);
      if (f) idToFile.set(r.id, f);
    }
    if (!idToFile.size) continue;
    let meta;
    try { meta = await commonsImageInfo([...idToFile.values()]); }
    catch (e) { console.error(`  batch ${i / BATCH}: ${e.message}`); await sleep(5000); continue; }
    for (const [id, file] of idToFile) {
      const m = meta.get(file);
      if (!m) { miss++; continue; }
      await pool.query(
        `UPDATE asim_movies SET poster_credit=$1, poster_license=$2, poster_source_url=$3, updated_at=NOW() WHERE id=$4`,
        [m.artist, m.license, m.descriptionUrl, id]
      );
      ok++;
    }
    if ((i / BATCH) % 5 === 0) console.log(`  …batch ${Math.floor(i/BATCH)+1}/${Math.ceil(sel.rows.length/BATCH)} ok=${ok} miss=${miss}`);
    await sleep(SLEEP_MS);
  }
  console.log(`[backfill-movies] done. ok=${ok} miss=${miss}`);
}

async function backfillPeople() {
  const sel = await pool.query(`
    SELECT id, headshot_url FROM asim_people
     WHERE headshot_source='wikimedia-cc' AND headshot_credit IS NULL
     ORDER BY id LIMIT 1000`);
  console.log(`[backfill-people] ${sel.rows.length} rows`);
  let ok = 0, miss = 0;
  for (let i = 0; i < sel.rows.length; i += BATCH) {
    const slice = sel.rows.slice(i, i + BATCH);
    const idToFile = new Map();
    for (const r of slice) {
      const f = fileNameFromCommonsUrl(r.headshot_url);
      if (f) idToFile.set(r.id, f);
    }
    if (!idToFile.size) continue;
    let meta;
    try { meta = await commonsImageInfo([...idToFile.values()]); }
    catch (e) { console.error(`  batch ${i / BATCH}: ${e.message}`); await sleep(5000); continue; }
    for (const [id, file] of idToFile) {
      const m = meta.get(file);
      if (!m) { miss++; continue; }
      await pool.query(
        `UPDATE asim_people SET headshot_credit=$1, headshot_license=$2, headshot_source_url=$3, updated_at=NOW() WHERE id=$4`,
        [m.artist, m.license, m.descriptionUrl, id]
      );
      ok++;
    }
    await sleep(SLEEP_MS);
  }
  console.log(`[backfill-people] done. ok=${ok} miss=${miss}`);
}

(async () => {
  await backfillMovies();
  await backfillPeople();
  await pool.end();
})().catch(e => { console.error('fatal:', e); process.exit(1); });