`scripts/fetch-wikidata.mjs`

205 lines
#!/usr/bin/env node
// CelebritySignatures — verified-signature fetcher.
// Spine: Wikidata P109 ("signature") links a real person to a real signature
// FILE on Wikimedia Commons. We query the public SPARQL endpoint (no key, free),
// rank by cross-wiki sitelink count, then pull each file's license from the
// Commons API. Nothing is invented — every row traces to a fetchable file.
//
// Usage: node scripts/fetch-wikidata.mjs [category|all]
// Output: data/<category>.json  +  data/celebrity_signatures.{json,csv}

import { writeFile, mkdir } from 'node:fs/promises';
import {
  runSparql, fileTitleFromUrl, fetchLicenses, licenseRisk, assess,
  fetchLabels, csvCell,
} from './lib/wikidata-common.mjs';

const PER_CAT = 320;            // fetch headroom (cross-category dedupe is hungry), trimmed to 100
const TARGET = 100;

// Authoritative human-curated ranking lists per category (cited as guidance;
// the automated `rank` is the Wikidata sitelink-sorted position).
const RANKING_SOURCES = {
  politics: ['https://www.gallup.com/poll/most-admired-man-woman.aspx', 'https://time.com/time100/'],
  sports: ['https://www.espn.com/', 'https://www.olympics.com/en/athletes', 'https://www.laureus.com/'],
  movies: ['https://www.imdb.com/chart/starmeter/', 'https://www.forbes.com/celebrities/'],
  tv: ['https://www.emmys.com/', 'https://yougov.com/ratings/'],
  hollywood: ['https://www.afi.com/afis-100-years-100-stars/', 'https://walkoffame.com/'],
  declaration: ['https://www.archives.gov/founding-docs/declaration', 'https://founders.archives.gov/'],
};

// SPARQL WHERE body per category. Must bind ?person, ?links; ?sig where available.
const CATEGORIES = {
  politics: {
    label: 'Politics',
    body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q82955 ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
  },
  sports: {
    label: 'Sports',
    body: `?person wdt:P31 wd:Q5 ; wdt:P641 ?sport ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
  },
  movies: {
    label: 'Movies',
    body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
  },
  tv: {
    label: 'TV',
    body: `?person wdt:P31 wd:Q5 ; wdt:P106 ?occ ; wdt:P109 ?sig ; wikibase:sitelinks ?links . VALUES ?occ { wd:Q10798782 wd:Q947873 }`,
  },
  hollywood: {
    // Golden-Age / iconic: film actors who are deceased (cleanest publicity posture).
    label: 'Hollywood',
    body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links ; wdt:P570 ?deadrequired .`,
  },
  moviesclassic: {
    // Deceased film actors beyond the Hollywood top tier (dedupe gives the next 100) —
    // a usable, publicity-safe "Movies" set (living movie stars are permission-needed).
    label: 'Movies (Classic)',
    body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links ; wdt:P570 ?deadrequired .`,
  },
  // NOTE: the Declaration of Independence set is owned by scripts/fetch-declaration.mjs
  // (MediaWiki/Commons APIs, outage-proof). The combiner below merges its output in.
};

function buildQuery(body, limit) {
  return `SELECT DISTINCT ?person ?personLabel ?sig ?dod ?links WHERE {
    ${body}
    OPTIONAL { ?person wdt:P570 ?dod . }
    SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
  } ORDER BY DESC(?links) LIMIT ${limit}`;
}

async function fetchCategory(catKey) {
  const cat = CATEGORIES[catKey];
  const rows = await runSparql(buildQuery(cat.body, cat.limit || PER_CAT));
  // dedupe by person QID, keep highest sitelinks (already sorted)
  const seen = new Set();
  const people = [];
  for (const b of rows) {
    const qid = b.person.value.split('/').pop();
    if (seen.has(qid)) continue;
    seen.add(qid);
    people.push({
      qid,
      name: b.personLabel?.value || qid,
      sigUrl: b.sig?.value || null,
      dod: b.dod?.value || null,
      links: parseInt(b.links?.value || '0', 10),
    });
  }
  return people;
}

async function main() {
  const arg = (process.argv[2] || 'all').toLowerCase();
  const keys = arg === 'all' ? Object.keys(CATEGORIES) : [arg];
  await mkdir('data', { recursive: true });

  const globalSeen = new Set();
  const all = [];
  // priority order keeps a deceased icon in Hollywood/declaration over movies/tv
  const priority = ['declaration', 'politics', 'sports', 'hollywood', 'moviesclassic', 'movies', 'tv'];
  const order = keys.sort((a, b) => priority.indexOf(a) - priority.indexOf(b));

  for (const key of order) {
    process.stdout.write(`\n[${key}] querying Wikidata… `);
    let people;
    try {
      people = await fetchCategory(key);
    } catch (e) {
      console.log(`FAILED: ${e.message}`);
      continue;
    }
    console.log(`${people.length} candidates with a signature on file`);

    // global dedupe across categories
    people = people.filter((p) => !globalSeen.has(p.qid));

    // backfill any name that came back as a raw QID (label-service flakiness)
    const needLabel = people.filter((p) => /^Q\d+$/.test(p.name)).map((p) => p.qid);
    if (needLabel.length) {
      const labels = await fetchLabels(needLabel);
      for (const p of people) if (labels[p.qid]) p.name = labels[p.qid];
    }

    // license lookup
    const titles = [...new Set(people.map((p) => fileTitleFromUrl(p.sigUrl)).filter(Boolean))];
    process.stdout.write(`[${key}] fetching ${titles.length} Commons licenses… `);
    const lic = titles.length ? await fetchLicenses(titles) : {};
    console.log('done');

    const isDecl = key === 'declaration';
    const catRows = [];
    for (const p of people) {
      if (catRows.length >= (isDecl ? 56 : TARGET)) break;
      globalSeen.add(p.qid);
      const title = fileTitleFromUrl(p.sigUrl);
      const meta = title ? lic[title] : null;
      const deceased = !!p.dod || isDecl; // all DoI signers deceased
      const hasFile = !!p.sigUrl;
      const fromArchive = isDecl && !hasFile;
      const license = fromArchive
        ? 'Public domain (US Gov work / pre-1929)'
        : (meta?.license || (hasFile ? 'unknown (verify on Commons)' : 'n/a'));
      const { risk_level, usable } = assess({ deceased, lic: license, fromArchive });

      const sigImg = fromArchive
        ? 'https://www.archives.gov/founding-docs/declaration-transcript'
        : (meta?.directUrl || p.sigUrl);
      const sourceType = fromArchive
        ? 'US National Archives — engrossed Declaration of Independence (PD)'
        : 'Wikimedia Commons (Wikidata P109 signature file)';
      const filePage = meta?.filePage
        || (title ? `https://commons.wikimedia.org/wiki/${encodeURIComponent(title)}` : null);

      catRows.push({
        category: CATEGORIES[key].label,
        rank: catRows.length + 1,
        full_name: p.name,
        wikidata: `https://www.wikidata.org/wiki/${p.qid}`,
        reason_for_ranking: `Cross-wiki notability: present on ${p.links} language Wikipedias`,
        ranking_source_urls: (RANKING_SOURCES[key] || []).join(' | '),
        signature_image_url: sigImg,
        signature_source_type: sourceType,
        image_license: license,
        image_author: meta?.artist || (fromArchive ? 'US Government (1776)' : ''),
        deceased: deceased ? 'yes' : 'no',
        death_date: p.dod ? p.dod.slice(0, 10) : (isDecl ? 'pre-1830' : ''),
        risk_level,
        usable_in_commercial_collage: usable,
        notes: [
          deceased ? 'Deceased — low publicity-rights risk' : 'LIVING — right-of-publicity exposure for commercial use',
          /share|sa/i.test(license) && /cc by-?sa/i.test(license) ? 'CC BY-SA: derivative collage may need share-alike + attribution' : '',
          fromArchive ? 'No standalone Commons signature file; signature appears on the engrossed Declaration (NARA scan, PD)' : '',
        ].filter(Boolean).join('; '),
        backup_source: isDecl
          ? 'https://founders.archives.gov/'
          : (filePage || 'https://catalog.archives.gov/'),
      });
    }
    await writeFile(`data/${key}.json`, JSON.stringify(catRows, null, 2));
    console.log(`[${key}] -> data/${key}.json (${catRows.length} rows, ${catRows.filter(r => r.usable_in_commercial_collage === 'yes').length} collage-usable)`);
    all.push(...catRows);
  }

  // merge in the dedicated Declaration of Independence set (if built)
  try {
    const { readFile } = await import('node:fs/promises');
    const decl = JSON.parse(await readFile('data/declaration.json', 'utf8'));
    if (decl.length) { all.unshift(...decl); console.log(`merged ${decl.length} Declaration signers`); }
  } catch { /* declaration not built yet */ }

  // combined outputs
  await writeFile('data/celebrity_signatures.json', JSON.stringify(all, null, 2));
  const cols = ['category', 'rank', 'full_name', 'wikidata', 'reason_for_ranking', 'ranking_source_urls', 'signature_image_url', 'signature_source_type', 'image_license', 'image_author', 'deceased', 'death_date', 'risk_level', 'usable_in_commercial_collage', 'notes', 'backup_source'];
  const csv = [cols.join(',')].concat(all.map((r) => cols.map((c) => csvCell(r[c])).join(','))).join('\n');
  await writeFile('data/celebrity_signatures.csv', csv);

  console.log(`\n=== TOTAL ${all.length} rows ===`);
  const byUsable = all.reduce((a, r) => ((a[r.usable_in_commercial_collage] = (a[r.usable_in_commercial_collage] || 0) + 1), a), {});
  console.log('Collage usability:', byUsable);
  console.log('Wrote data/celebrity_signatures.{json,csv}');
}

main().catch((e) => { console.error(e); process.exit(1); });