`scripts/fetch-authors.mjs`

116 lines
#!/usr/bin/env node
// "Authors" category — famous deceased writers with REAL P109 signatures.
// Same architecture as the Artists set: Wikidata universe by literary
// occupation (writer, novelist, poet, playwright, essayist, short-story
// writer, children's writer, science-fiction writer), deceased >50yr
// (death < 1974, the site's publicity/copyright posture), famous-first by
// cross-wiki sitelinks, every signature verified on Wikimedia Commons with a
// license gate. Output: data/authors.json (regenerable; server merges it into
// /api/signatures — the hand-curated file is never touched).
// Dedupe: anyone already in celebrity_signatures.json or artists.json keeps
// their existing category; they are skipped here.
//
// Usage: node scripts/fetch-authors.mjs
// Cost: $0 — WDQS + Commons, no keys.

import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
  runSparql, fileTitleFromUrl, fetchLicenses, licenseRisk, assess, fetchLabels,
  sleep,
} from './lib/wikidata-common.mjs';

const DATA = join(fileURLToPath(new URL('..', import.meta.url)), 'data');
const DEATH_CUTOFF = 1974;
const OCCUPATIONS = {
  Q36180: 'writer', Q6625963: 'novelist', Q49757: 'poet', Q214917: 'playwright',
  Q11774202: 'essayist', Q15949613: 'short-story writer',
  Q4853732: "children's writer", Q18844224: 'science-fiction writer',
};

// ── 1) universe: deceased authors with a P109 signature ─────────────────────
const byQid = new Map();
for (const [occ, label] of Object.entries(OCCUPATIONS)) {
  const q = `SELECT ?person ?personLabel ?sig ?dod ?links WHERE {
    ?person wdt:P31 wd:Q5 ; wdt:P106 wd:${occ} ; wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
    FILTER(YEAR(?dod) < ${DEATH_CUTOFF})
    SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
  }`;
  process.stdout.write(`[authors] ${label} … `);
  const rows = await runSparql(q);
  let added = 0;
  for (const b of rows) {
    const qid = b.person.value.split('/').pop();
    if (byQid.has(qid)) continue;
    byQid.set(qid, {
      qid,
      name: b.personLabel?.value || qid,
      sigUrl: b.sig?.value || null,
      dod: b.dod?.value || null,
      links: parseInt(b.links?.value || '0', 10),
      role: label,
    });
    added++;
  }
  console.log(`${rows.length} rows (+${added}, total ${byQid.size})`);
  await sleep(1500);
}

// ── 2) cross-category dedupe (curated + artists keep their people) ──────────
const loadJson = async (f, fb) => { try { return JSON.parse(await readFile(join(DATA, f), 'utf8')); } catch { return fb; } };
const qidOf = (r) => (r.wikidata || '').split('/').pop();
const taken = new Set([
  ...(await loadJson('celebrity_signatures.json', [])).map(qidOf),
  ...(await loadJson('artists.json', [])).map(qidOf),
].filter(Boolean));
const candidates = [...byQid.values()].filter((p) => !taken.has(p.qid) && p.sigUrl && p.dod);
console.log(`[authors] ${byQid.size} in universe → ${candidates.length} after cross-category dedupe`);

// label backfill for raw QIDs (WDQS label-service flakiness)
const needLabel = candidates.filter((p) => /^Q\d+$/.test(p.name)).map((p) => p.qid);
if (needLabel.length) {
  const labels = await fetchLabels(needLabel);
  for (const p of candidates) if (labels[p.qid]) p.name = labels[p.qid];
}

// ── 3) Commons license verify (batched; PD/CC kept, unknown dropped) ────────
const titles = [...new Set(candidates.map((p) => fileTitleFromUrl(p.sigUrl)).filter(Boolean))];
console.log(`[authors] fetching ${titles.length} Commons licenses …`);
const lic = await fetchLicenses(titles);

candidates.sort((a, b) => b.links - a.links);
const out = [];
let droppedLic = 0;
for (const p of candidates) {
  const title = fileTitleFromUrl(p.sigUrl);
  const meta = title ? lic[title] : null;
  if (!meta || !meta.directUrl) { droppedLic++; continue; }         // missing file
  const license = meta.license || 'unknown';
  if (licenseRisk(license) === 'high') { droppedLic++; continue; }  // unknown license
  const { risk_level, usable } = assess({ deceased: true, lic: license, fromArchive: false });
  out.push({
    category: 'Authors',
    rank: out.length + 1,
    full_name: p.name,
    wikidata: `https://www.wikidata.org/wiki/${p.qid}`,
    reason_for_ranking: `${p.role[0].toUpperCase() + p.role.slice(1)}; cross-wiki notability: present on ${p.links} language Wikipedias`,
    ranking_source_urls: 'https://www.nobelprize.org/prizes/literature/ | https://www.loc.gov/',
    signature_image_url: meta.directUrl,
    signature_source_type: 'Wikimedia Commons (Wikidata P109 signature file)',
    image_license: license,
    image_author: meta.artist || '',
    deceased: 'yes',
    death_date: p.dod.slice(0, 10),
    risk_level,
    usable_in_commercial_collage: usable,
    notes: `Deceased ${p.dod.slice(0, 4)} — low publicity-rights risk${/sa/i.test(license) && /cc by-?sa/i.test(license) ? '; CC BY-SA: derivative collage may need share-alike + attribution' : ''}`,
    backup_source: meta.filePage || (title ? `https://commons.wikimedia.org/wiki/${encodeURIComponent(title)}` : ''),
  });
}

await writeFile(join(DATA, 'authors.json'), JSON.stringify(out, null, 2));
console.log(`[authors] wrote data/authors.json — ${out.length} authors (${droppedLic} dropped: missing file/unknown license)`);
console.log('top 12:');
out.slice(0, 12).forEach((r) => console.log(`  ${r.rank}. ${r.full_name} (${r.death_date.slice(0, 4)}) — ${r.image_license}`));