`scripts/fetch-artists.mjs`

202 lines
#!/usr/bin/env node
// Fetch long-deceased museum-collection artists that have a public-domain
// Wikidata P109 signature file. Source = Wikidata SPARQL (free) + Wikimedia
// Commons (free). NO paid APIs, NO LLM in this step — pure structured query.
//
// "Artists at the MET, Smithsonian, etc" = creators (P170) of works whose
// collection (P195) is one of the major museums below.
// PD guardrail = deceased (P570) with death year < 1974 (dead >50yr), mirroring
// the catalog's existing deceased-only rule.
// "Famous first" = ORDER BY DESC(sitelinks) — Wikipedia language-edition count.

import { writeFile, readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';

const ROOT = fileURLToPath(new URL('..', import.meta.url));
const ENDPOINT = 'https://query.wikidata.org/sparql';

// Met + Smithsonian family + the "etc" major-museum canon.
const MUSEUMS = {
  Q160236: 'The Metropolitan Museum of Art',
  Q1192305: 'Smithsonian American Art Museum',
  Q1967668: 'National Portrait Gallery (Smithsonian)',
  Q131626: 'Smithsonian Institution',
  Q1215884: 'Cooper Hewitt, Smithsonian Design Museum',
  Q214867: 'National Gallery of Art',
  Q188740: 'Museum of Modern Art',
  Q239303: 'Art Institute of Chicago',
  Q180788: 'National Gallery, London',
  Q19675: 'Louvre',
  Q194235: 'Tate',
  Q23402: "Musée d'Orsay",
  Q190804: 'Rijksmuseum',
  Q51252: 'Uffizi Gallery',
  Q160112: 'Museo del Prado',
  // West-coast + open-collection expansion (Steve 2026-08-03: "all public
  // museums — the Broad, MOCA, LACMA… keep building"). QIDs verified via
  // wbsearchentities same day.
  Q1641836: 'Los Angeles County Museum of Art',
  Q17060100: 'The Broad',
  Q1134646: 'Museum of Contemporary Art, Los Angeles',
  Q731126: 'J. Paul Getty Museum',
  Q913672: 'San Francisco Museum of Modern Art',
  Q639791: 'Whitney Museum of American Art',
  Q201469: 'Solomon R. Guggenheim Museum',
  Q657415: 'Cleveland Museum of Art',
  Q49133: 'Museum of Fine Arts Boston',
  Q510324: 'Philadelphia Museum of Art',
  // Amsterdam / Dutch expansion (Steve 2026-08-03: "more from Amsterdam
  // museums like the Rijks"). Rijksmuseum already above. QIDs verified same day.
  Q224124: 'Van Gogh Museum',
  Q924335: 'Stedelijk Museum Amsterdam',
  Q277316: 'Rembrandt House Museum',
  Q1820897: 'Amsterdam Museum',
  Q221092: 'Mauritshuis',
  // World-collections expansion (Steve 2026-08-04: "keep going"). The dense
  // P6379 path makes any added museum pull its signature-artists. QIDs verified.
  Q132783: 'Hermitage Museum',
  Q6373: 'British Museum',
  Q95569: 'Kunsthistorisches Museum',
  Q178065: 'Centre Georges Pompidou',
  Q460889: 'Museo Nacional Centro de Arte Reina Sofía',
  Q650519: 'Musée Rodin',
  Q213322: 'Victoria and Albert Museum',
  Q165631: 'Gemäldegalerie Berlin',
  Q238587: 'National Portrait Gallery, London',
  Q154568: 'Alte Pinakothek',
  Q163804: 'Städel Museum',
  Q1362629: 'Tokyo National Museum',
};
// Per-museum-batch query. The all-museums-in-one-VALUES query 504s on WDQS now
// (too heavy across 25 museums), so we query in small batches with retry/backoff
// and union the rows (dedup happens in the union step below).
function queryFor(museumQids) {
  const vals = museumQids.map(q => `wd:${q}`).join(' ');
  return `
SELECT ?person ?personLabel ?sig ?dod ?links
       (GROUP_CONCAT(DISTINCT ?museumLabel; separator="|") AS ?museums)
       (GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs)
WHERE {
  VALUES ?museum { ${vals} }
  ?work wdt:P170 ?person ; wdt:P195 ?museum .
  ?person wdt:P109 ?sig .
  ?person wdt:P570 ?dod .
  FILTER(YEAR(?dod) < 1974)
  ?person wikibase:sitelinks ?links .
  OPTIONAL { ?person wdt:P106 ?occ .
    ?occ rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
  ?museum rdfs:label ?museumLabel . FILTER(LANG(?museumLabel)="en")
  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
}
GROUP BY ?person ?personLabel ?sig ?dod ?links
ORDER BY DESC(?links)`;
}
const sleep = ms => new Promise(r => setTimeout(r, ms));
async function sparql(q, tries = 5) {
  const url = `${ENDPOINT}?format=json&query=${encodeURIComponent(q)}`;
  for (let i = 1; ; i++) {
    let res;
    try {
      res = await fetch(url, {
        headers: {
          'Accept': 'application/sparql-results+json',
          'User-Agent': 'CelebritySignatures/1.0 (designerwallcoverings.com; PD-signature artist catalog build)',
        },
      });
    } catch (e) { if (i > tries) throw e; await sleep(8000 * i); continue; }
    if (res.ok) return (await res.json()).results.bindings;
    if ((res.status === 504 || res.status === 429 || res.status >= 500) && i <= tries) {
      process.stdout.write(`(WDQS ${res.status}; backoff ${8 * i}s, try ${i}/${tries}) `);
      await sleep(8000 * i); continue;
    }
    throw new Error(`SPARQL ${res.status}: ${(await res.text()).slice(0, 200)}`);
  }
}

// Second harvest path: P6379 "has works in the collection" is a DIRECT
// person→museum link, far denser than artwork→P195 (which is sparsely itemized,
// esp. for European museums like the Rijksmuseum). This is what actually pulls
// the Amsterdam/Dutch masters in. Same output columns; unioned with P195.
function queryFor6379(museumQids) {
  const vals = museumQids.map(q => `wd:${q}`).join(' ');
  return `
SELECT ?person ?personLabel ?sig ?dod ?links
       (GROUP_CONCAT(DISTINCT ?museumLabel; separator="|") AS ?museums)
       (GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs)
WHERE {
  VALUES ?museum { ${vals} }
  ?person wdt:P6379 ?museum ; wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
  FILTER(YEAR(?dod) < 1974)
  OPTIONAL { ?person wdt:P106 ?occ . ?occ rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
  ?museum rdfs:label ?museumLabel . FILTER(LANG(?museumLabel)="en")
  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
}
GROUP BY ?person ?personLabel ?sig ?dod ?links
ORDER BY DESC(?links)`;
}

const ALL_QIDS = Object.keys(MUSEUMS);
const rows = [];
for (let i = 0; i < ALL_QIDS.length; i += 3) {
  const batch = ALL_QIDS.slice(i, i + 3);
  process.stdout.write(`[P195 ${i + 1}-${i + batch.length}/${ALL_QIDS.length}] ${batch.map(q => MUSEUMS[q]).join(', ')} … `);
  const r = await sparql(queryFor(batch));
  rows.push(...r);
  console.log(`${r.length} rows (total ${rows.length})`);
  await sleep(1500);
}
for (let i = 0; i < ALL_QIDS.length; i += 3) {
  const batch = ALL_QIDS.slice(i, i + 3);
  process.stdout.write(`[P6379 ${i + 1}-${i + batch.length}/${ALL_QIDS.length}] ${batch.map(q => MUSEUMS[q]).join(', ')} … `);
  const r = await sparql(queryFor6379(batch));
  rows.push(...r);
  console.log(`${r.length} rows (total ${rows.length})`);
  await sleep(1500);
}
console.log(`raw rows (P195 + P6379): ${rows.length}`);

// One record per person; Special:FilePath URL is directly usable as an <img> src.
const seen = new Set();
const artists = [];
for (const r of rows) {
  const qid = r.person.value.split('/').pop();
  if (seen.has(qid)) continue;
  seen.add(qid);
  artists.push({
    qid,
    wikidata: r.person.value,
    full_name: r.personLabel.value,
    sitelinks: +r.links.value,
    death_date: r.dod.value.slice(0, 10),
    death_year: +r.dod.value.slice(0, 4),
    signature_image_url: r.sig.value, // http://commons.wikimedia.org/wiki/Special:FilePath/<file>
    museums: (r.museums?.value || '').split('|').filter(Boolean),
    occupations: (r.occs?.value || '').split('|').filter(Boolean),
  });
}

console.log(`unique artists: ${artists.length}`);
console.log('top 15 by fame:');
artists.slice(0, 15).forEach((a, i) =>
  console.log(`  ${i + 1}. ${a.full_name} — ${a.sitelinks} wikis, d.${a.death_year}, ${a.museums.length} museums`));

// UNION into the existing raw set (never overwrite — the roster gap-fill
// harvester also feeds artists-raw.json). Idempotent by QID; museum
// provenance merges; fresher sitelink/occupation data wins.
const rawPath = join(ROOT, 'data', 'artists-raw.json');
let existing = [];
try { existing = JSON.parse(await readFile(rawPath, 'utf8')); } catch { /* first run */ }
const byQid = new Map(existing.map((a) => [a.qid, a]));
let added = 0, updated = 0;
for (const a of artists) {
  const cur = byQid.get(a.qid);
  if (!cur) { byQid.set(a.qid, a); added++; continue; }
  for (const m of a.museums) if (!cur.museums.includes(m)) { cur.museums.push(m); updated++; }
  if (a.occupations.length > (cur.occupations || []).length) cur.occupations = a.occupations;
  if (a.sitelinks > (cur.sitelinks || 0)) cur.sitelinks = a.sitelinks;
}
const unioned = [...byQid.values()].sort((x, y) => (y.sitelinks || 0) - (x.sitelinks || 0));
await writeFile(rawPath, JSON.stringify(unioned, null, 2));
console.log(`\nartists-raw.json: ${existing.length} -> ${unioned.length} (+${added} new, ${updated} museum-tag merges)`);