`scrapers/wikidata-commons-fallback.js`

154 lines
#!/usr/bin/env node
'use strict';
// Commons-search fallback for asim_people who have a wikidata_id but no
// headshot_url (P18 was empty on the wikidata entity, but Commons often
// still has a "Category:Person Name" with usable photos).
//
// Strategy: for each (wd_id, full_name) row missing headshot_url, search
// Commons for `"<full_name>" filemime:image/jpeg|png` in namespace 6, take
// the first result, fetch extmetadata, only apply if license is CC-anything
// (rejects "Fair use" and "Non-free copyright").
//
// Polite: batches 1 search/req, 1.1s sleep. UA per Wikimedia policy.

require('dotenv').config();
const { Pool } = require('pg');

const args = process.argv.slice(2).reduce((m, a) => {
  const [k, v] = a.replace(/^--/, '').split('=');
  m[k] = v ?? true; return m;
}, {});

const LIMIT    = parseInt(args.limit || '200', 10);
const DRY      = !!args.dry;
const SLEEP_MS = parseInt(args.sleep || '1100', 10);
const UA       = 'AsSeenInMovies/0.1 (https://asseeninmovies.com; steve@designerwallcoverings.com)';
const COMMONS  = 'https://commons.wikimedia.org/w/api.php';

const pool = new Pool({
  connectionString: process.env.DATABASE_URL || 'postgresql:///dw_unified?host=/tmp&user=stevestudio2',
  max: 4,
});

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

// CC-* / public-domain only. Reject anything with "fair use", "non-free",
// "all rights reserved", or no license name at all.
function isFree(licenseShortName) {
  if (!licenseShortName) return false;
  const s = String(licenseShortName).toLowerCase();
  if (/fair use|non-?free|all rights|copyright/i.test(s)) return false;
  return /\b(cc|public domain|pd|cc0|cc by|cc-by)\b/i.test(s);
}

function stripHtml(s) {
  if (!s) return null;
  return String(s).replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim() || null;
}

async function searchOne(name) {
  // Commons search; namespace 6 = File. Quote name for phrase match.
  const url = new URL(COMMONS);
  url.search = new URLSearchParams({
    action: 'query',
    list: 'search',
    srsearch: `"${name}" filemime:image/jpeg|filemime:image/png`,
    srnamespace: '6',
    srlimit: '3',
    format: 'json',
    origin: '*',
  }).toString();
  const r = await fetch(url.toString(), { headers: { 'user-agent': UA } });
  if (!r.ok) throw new Error(`search ${r.status}`);
  const j = await r.json();
  const hits = (j.query && j.query.search) || [];
  return hits.map(h => h.title);  // "File:Foo Bar.jpg"
}

async function imageInfo(title) {
  const url = new URL(COMMONS);
  url.search = new URLSearchParams({
    action: 'query',
    titles: title,
    prop: 'imageinfo',
    iiprop: 'url|extmetadata',
    format: 'json',
    origin: '*',
  }).toString();
  const r = await fetch(url.toString(), { headers: { 'user-agent': UA } });
  if (!r.ok) throw new Error(`info ${r.status}`);
  const j = await r.json();
  const pages = (j.query && j.query.pages) || {};
  const first = Object.values(pages)[0];
  if (!first || !first.imageinfo || !first.imageinfo[0]) return null;
  const ii = first.imageinfo[0];
  const md = ii.extmetadata || {};
  return {
    url: ii.url,
    title,
    artist: stripHtml((md.Artist && md.Artist.value) || null),
    license: stripHtml((md.LicenseShortName && md.LicenseShortName.value) || null),
    licenseUrl: (md.LicenseUrl && md.LicenseUrl.value) || null,
    descriptionUrl: ii.descriptionurl,
  };
}

(async () => {
  // Order by credit count DESC — most-credited (== most likely to have a
  // Commons photo) first. Random ordering put set decorators at the top.
  const sel = await pool.query(`
    SELECT p.id, p.full_name, p.wikidata_id, COALESCE(cc.c, 0) AS credit_count
      FROM asim_people p
      LEFT JOIN (SELECT person_id, count(*) AS c FROM asim_credits GROUP BY person_id) cc
        ON cc.person_id = p.id
     WHERE p.wikidata_id IS NOT NULL
       AND p.headshot_url IS NULL
       AND p.full_name IS NOT NULL
       AND length(p.full_name) BETWEEN 4 AND 80
     ORDER BY cc.c DESC NULLS LAST, p.id
     LIMIT $1`, [LIMIT]);

  console.log(`[commons-fallback] candidates=${sel.rows.length} dry=${DRY}`);
  let tried = 0, found = 0, applied = 0, skip = 0;

  for (const row of sel.rows) {
    tried++;
    let titles;
    try { titles = await searchOne(row.full_name); }
    catch (e) { console.error(`  search ${row.full_name}: ${e.message}`); await sleep(SLEEP_MS); continue; }

    if (!titles.length) { await sleep(SLEEP_MS); continue; }

    // Take the first hit only. Future: rank by file name match strength.
    const title = titles[0];
    let info;
    try { info = await imageInfo(title); }
    catch (e) { console.error(`  info ${title}: ${e.message}`); await sleep(SLEEP_MS); continue; }
    if (!info) { await sleep(SLEEP_MS); continue; }

    found++;
    if (!isFree(info.license)) {
      skip++;
      if (tried % 25 === 0) console.log(`  ${tried}/${sel.rows.length}  found=${found} applied=${applied} skip-license=${skip}`);
      await sleep(SLEEP_MS); continue;
    }

    if (!DRY) {
      await pool.query(`
        UPDATE asim_people
           SET headshot_url    = $1,
               headshot_credit = $2,
               headshot_license = $3,
               headshot_source_url = $4
         WHERE id = $5
           AND headshot_url IS NULL`,
        [info.url, info.artist || 'Wikimedia Commons', info.license, info.descriptionUrl, row.id]);
      applied++;
    }
    if (tried % 25 === 0) console.log(`  ${tried}/${sel.rows.length}  found=${found} applied=${applied} skip-license=${skip}`);
    await sleep(SLEEP_MS);
  }
  console.log(`[commons-fallback] done. tried=${tried} found=${found} applied=${applied} skip-license=${skip}`);
  await pool.end();
})().catch(e => { console.error(e); process.exit(1); });