← back to CelebritySignatures
scripts/fetch-artists-rosters.mjs
384 lines
#!/usr/bin/env node
// Roster GAP-FILL for the Artists category. Companion to fetch-artists.mjs
// (the P195 museum-works harvest): Wikidata's P195 itemization is thin for the
// National Gallery of Art (DC), NPG, Cooper Hewitt and most Smithsonian units,
// so famous-enough artists with real P109 signatures get missed. This script
// harvests the museums' OWN rosters —
// MET : GitHub CSV dump (Artist Wikidata URL column → direct QIDs)
// NGA : opendata constituents.csv (wikidataid column → direct QIDs)
// SI : api.si.edu Open Access (names; needs SI_API_KEY, free api.data.gov)
// — resolves them against Wikidata P109 + deceased(<1974), and UNIONS the
// survivors into data/artists-raw.json in fetch-artists.mjs's exact shape
// (idempotent by QID; safe to re-run after either harvester).
//
// Usage: node scripts/fetch-artists-rosters.mjs [--force] [--skip-met]
// Cost: $0 — free public APIs. Caches in tmp_artists_cache/.
import { writeFile, readFile, mkdir, stat } from 'node:fs/promises';
import { createWriteStream, createReadStream, existsSync, readFileSync } from 'node:fs';
import { pipeline } from 'node:stream/promises';
import { Readable } from 'node:stream';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { UA, runSparql, fetchLabels, sleep } from './lib/wikidata-common.mjs';
const ROOT = fileURLToPath(new URL('..', import.meta.url));
const DATA = join(ROOT, 'data');
const CACHE = join(ROOT, 'tmp_artists_cache');
const FORCE = process.argv.includes('--force');
const SKIP_MET = process.argv.includes('--skip-met');
const DEATH_CUTOFF = 1974; // repo convention: deceased >50yr (see fetch-artists.mjs)
const MET_CSV_URL = 'https://media.githubusercontent.com/media/metmuseum/openaccess/master/MetObjects.csv';
const NGA_CSV_URL = 'https://raw.githubusercontent.com/NationalGalleryOfArt/opendata/main/data/constituents.csv';
const SI_API = 'https://api.si.edu/openaccess/api/v1.0/search';
// unit code → museum display name (matches fetch-artists.mjs label style)
const SI_UNIT_NAMES = {
SAAM: 'Smithsonian American Art Museum',
NPG: 'National Portrait Gallery (Smithsonian)',
CHNDM: 'Cooper Hewitt, Smithsonian Design Museum',
FSG: 'Freer Gallery of Art and Arthur M. Sackler Gallery',
};
const SI_MAX_PAGES_PER_UNIT = 60; // 60k records/unit; truncation LOGGED, never silent
// Visual-arts occupations for the name-match universe (SI rows have no QIDs).
const OCCUPATIONS = ['Q1028181', 'Q1281618', 'Q483501', 'Q11569986', 'Q329439', 'Q644687', 'Q33231', 'Q3391743'];
// ── small utils ──────────────────────────────────────────────────────────────
const stripDiacritics = (s) => s.normalize('NFD').replace(/[̀-ͯ]/g, '');
const normName = (s) => stripDiacritics(String(s || '')).toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
function deinvert(s) { // "Gogh, Vincent van" → "Vincent van Gogh"
const p = String(s || '').split(', ');
return (p.length === 2 && p[1] && !/\d/.test(p[1])) ? `${p[1]} ${p[0]}` : s;
}
const QUALIFIERS = /^(attributed to|after|workshop of|studio of|school of|style of|manner of|circle of|follower of|copy after|possibly|probably|imitator of|formerly attributed to)\s+/i;
const cleanName = (raw) => String(raw || '').trim().replace(QUALIFIERS, '').replace(/\s*\([^)]*\)\s*$/, '').trim();
function deathYearFrom(str) {
const s = String(str || '');
let m = s.match(/(?:d\.|died)\s*(\d{4})/i);
if (m) return +m[1];
m = s.match(/(\d{4})\s*[-–—]\s*(\d{4})/);
if (m && +m[2] > +m[1] && +m[2] < 2030) return +m[2];
return null;
}
const cachedFile = (f) => !FORCE && existsSync(join(CACHE, f));
const readCache = async (f) => JSON.parse(await readFile(join(CACHE, f), 'utf8'));
const writeCache = async (f, v) => writeFile(join(CACHE, f), JSON.stringify(v));
function siApiKey() {
if (process.env.SI_API_KEY) return process.env.SI_API_KEY;
for (const p of [join(ROOT, '.env'), `${process.env.HOME}/Projects/secrets-manager/.env`]) {
try {
const m = readFileSync(p, 'utf8').match(/^SI_API_KEY=(.+)$/m);
if (m) return m[1].trim().replace(/^["']|["']$/g, '');
} catch { /* next */ }
}
return null;
}
// Streaming CSV parser (quote-aware, newlines-in-cells safe, zero-dep).
async function parseCsvStream(readable, onRow) {
let cell = '', row = [], inQ = false, prevQuote = false;
const flushCell = () => { row.push(cell); cell = ''; };
const flushRow = () => { flushCell(); if (row.length > 1 || row[0] !== '') onRow(row); row = []; };
for await (const chunk of readable) {
const s = chunk.toString('utf8');
for (let i = 0; i < s.length; i++) {
const c = s[i];
if (inQ) { if (c === '"') { inQ = false; prevQuote = true; } else cell += c; }
else if (prevQuote && c === '"') { cell += '"'; inQ = true; prevQuote = false; }
else {
prevQuote = false;
if (c === '"') inQ = true;
else if (c === ',') flushCell();
else if (c === '\n') flushRow();
else if (c !== '\r') cell += c;
}
}
}
if (cell !== '' || row.length) flushRow();
}
async function download(url, dest, minBytes) {
if (!FORCE && existsSync(dest) && (await stat(dest)).size >= minBytes) {
console.log(` cached ${dest.split('/').pop()}`); return;
}
console.log(` downloading ${url.split('/').pop()} …`);
const res = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow' });
if (!res.ok) throw new Error(`download ${res.status}: ${url}`);
await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
const size = (await stat(dest)).size;
if (size < minBytes) throw new Error(`${dest} too small (${size}B) — LFS pointer instead of content?`);
console.log(` ok (${(size / 1e6).toFixed(1)}MB)`);
}
// ── harvesters: {name, deathYear, qid|null, museum} ─────────────────────────
async function harvestMet() {
if (cachedFile('roster-met.json')) { console.log('[met] cached'); return readCache('roster-met.json'); }
if (SKIP_MET) { console.log('[met] skipped (--skip-met)'); return []; }
await download(MET_CSV_URL, join(CACHE, 'MetObjects.csv'), 50e6);
const byKey = new Map();
let header = null; const idx = {}; let objects = 0;
await parseCsvStream(createReadStream(join(CACHE, 'MetObjects.csv')), (cells) => {
if (!header) {
header = cells;
for (const h of ['Artist Display Name', 'Artist End Date', 'Artist Wikidata URL']) {
idx[h] = header.findIndex((x) => x.trim() === h);
if (idx[h] < 0) throw new Error(`MET CSV missing column ${h} (headers: ${header.slice(0, 30).join(';')})`);
}
return;
}
objects++;
const names = (cells[idx['Artist Display Name']] || '').split('|');
const ends = (cells[idx['Artist End Date']] || '').split('|');
const wds = (cells[idx['Artist Wikidata URL']] || '').split('|');
for (let i = 0; i < names.length; i++) {
const name = cleanName(names[i]);
if (!name || name.length < 3) continue;
const endRaw = (ends[i] || '').trim();
const end = /^\d{4}$/.test(endRaw) && endRaw !== '9999' ? +endRaw : null;
const qm = (wds[i] || '').match(/Q\d+/);
const key = qm ? qm[0] : normName(name) + '|' + (end || '');
if (!byKey.has(key)) byKey.set(key, { name, deathYear: end, qid: qm ? qm[0] : null, museum: 'Metropolitan Museum of Art' });
}
});
const rows = [...byKey.values()];
console.log(`[met] ${objects} objects → ${rows.length} unique artists (${rows.filter((r) => r.qid).length} QID'd)`);
await writeCache('roster-met.json', rows);
return rows;
}
async function harvestNga() {
if (cachedFile('roster-nga.json')) { console.log('[nga] cached'); return readCache('roster-nga.json'); }
await download(NGA_CSV_URL, join(CACHE, 'nga-constituents.csv'), 1e6);
const rows = [];
let header = null; const idx = {};
await parseCsvStream(createReadStream(join(CACHE, 'nga-constituents.csv')), (cells) => {
if (!header) {
header = cells.map((h) => h.trim().toLowerCase());
for (const h of ['preferreddisplayname', 'forwarddisplayname', 'endyear', 'artistofngaobject', 'constituenttype', 'wikidataid']) idx[h] = header.indexOf(h);
return;
}
if (idx.artistofngaobject >= 0 && cells[idx.artistofngaobject] !== '1') return;
if (idx.constituenttype >= 0 && cells[idx.constituenttype] && cells[idx.constituenttype] !== 'individual') return;
const name = cleanName((idx.forwarddisplayname >= 0 && cells[idx.forwarddisplayname]) || deinvert(cells[idx.preferreddisplayname]));
if (!name || name.length < 3) return;
const end = /^\d{3,4}$/.test(cells[idx.endyear]) ? +cells[idx.endyear] : null;
const qm = (cells[idx.wikidataid] || '').match(/Q\d+/);
rows.push({ name, deathYear: end, qid: qm ? qm[0] : null, museum: 'National Gallery of Art' });
});
console.log(`[nga] ${rows.length} artist constituents (${rows.filter((r) => r.qid).length} QID'd)`);
await writeCache('roster-nga.json', rows);
return rows;
}
const SI_NAME_LABELS = /artist|maker|painter|sculptor|engraver|printmaker|designer|illustrator|photographer|etcher|draftsman/i;
async function harvestSi() {
if (cachedFile('roster-si.json')) { console.log('[si] cached'); return readCache('roster-si.json'); }
const key = siApiKey();
if (!key) { console.log('[si] NO SI_API_KEY — skipping Smithsonian'); return []; }
const byKey = new Map();
for (const [unit, museumName] of Object.entries(SI_UNIT_NAMES)) {
let start = 0, page = 0, total = Infinity, got = 0;
while (start < total && page < SI_MAX_PAGES_PER_UNIT) {
const url = `${SI_API}?q=${encodeURIComponent('unit_code:' + unit)}&start=${start}&rows=1000&api_key=${key}`;
let res;
try { res = await fetch(url, { headers: { 'User-Agent': UA } }); }
catch { await sleep(5000); continue; }
if (res.status === 429) { process.stdout.write('(429→60s) '); await sleep(60000); continue; }
if (!res.ok) { console.log(`[si:${unit}] HTTP ${res.status} — stopping unit`); break; }
const j = await res.json();
total = j.response?.rowCount ?? 0;
const recs = j.response?.rows || [];
if (!recs.length) break;
for (const r of recs) {
const entries = []
.concat((r.content?.freetext?.name || []).filter((n) => SI_NAME_LABELS.test(n.label || '')).map((n) => n.content))
.concat((r.content?.indexedStructured?.name || []).filter((s) => typeof s === 'string'));
for (const raw of entries) {
const dy = deathYearFrom(raw);
const name = cleanName(deinvert(String(raw)
.replace(/,?\s*\d{4}\s*[-–—]?\s*(\d{4})?\s*$/, '')
.replace(/,?\s*(b\.|d\.|born|died|active|ca\.|fl\.)[^,]*$/i, '')));
if (!name || name.length < 3 || /\d/.test(name)) continue;
const k = normName(name) + '|' + (dy || '');
if (!byKey.has(k)) byKey.set(k, { name, deathYear: dy, qid: null, museum: museumName });
}
}
got += recs.length; start += 1000; page++;
if (page % 10 === 0) process.stdout.write(`[si:${unit}] ${got}/${total} … `);
await sleep(350);
}
console.log(start < total
? `\n[si:${unit}] TRUNCATED ${got}/${total} (page cap ${SI_MAX_PAGES_PER_UNIT}; unique-name set saturates early — raise cap to go deeper)`
: `\n[si:${unit}] complete: ${got} records`);
}
const rows = [...byKey.values()];
console.log(`[si] ${rows.length} unique names across ${Object.keys(SI_UNIT_NAMES).join('/')}`);
await writeCache('roster-si.json', rows);
return rows;
}
// ── P109 universe (for name-matching the QID-less SI rows) ──────────────────
async function fetchUniverse() {
if (cachedFile('p109-universe.json')) { console.log('[universe] cached'); return readCache('p109-universe.json'); }
const byQid = new Map();
for (const occ of OCCUPATIONS) {
const q = `SELECT ?person ?personLabel ?sig ?dod ?links
(GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs) WHERE {
?person wdt:P31 wd:Q5 ; wdt:P106 wd:${occ} ; wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
FILTER(YEAR(?dod) < ${DEATH_CUTOFF})
OPTIONAL { ?person wdt:P106 ?o . ?o rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
} GROUP BY ?person ?personLabel ?sig ?dod ?links`;
process.stdout.write(`[universe] ${occ} … `);
const rows = await runSparql(q);
let added = 0;
for (const b of rows) {
const qid = b.person.value.split('/').pop();
if (byQid.has(qid)) continue;
byQid.set(qid, {
qid,
name: b.personLabel?.value || qid,
sig: b.sig?.value || null,
dod: b.dod?.value || null,
links: parseInt(b.links?.value || '0', 10),
occs: (b.occs?.value || '').split('|').filter(Boolean),
});
added++;
}
console.log(`${rows.length} rows (+${added}, total ${byQid.size})`);
await sleep(1500);
}
const rows = [...byQid.values()];
await writeCache('p109-universe.json', rows);
return rows;
}
// ── batched VALUES check for roster QIDs missing from artists-raw ───────────
async function valuesCheck(qids) {
const out = [];
for (let i = 0; i < qids.length; i += 400) {
const batch = qids.slice(i, i + 400);
const bf = `values-${i / 400}.json`;
let rows;
if (cachedFile(bf)) rows = await readCache(bf);
else {
const q = `SELECT ?person ?personLabel ?sig ?dod ?links
(GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs) WHERE {
VALUES ?person { ${batch.map((q2) => `wd:${q2}`).join(' ')} }
?person wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
FILTER(YEAR(?dod) < ${DEATH_CUTOFF})
OPTIONAL { ?person wdt:P106 ?o . ?o rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
} GROUP BY ?person ?personLabel ?sig ?dod ?links`;
rows = await runSparql(q);
await writeCache(bf, rows);
await sleep(1200);
}
out.push(...rows);
process.stdout.write(`[values] batch ${i / 400 + 1}/${Math.ceil(qids.length / 400)} → ${out.length} P109 hits\r`);
}
console.log();
return out;
}
// ── main ─────────────────────────────────────────────────────────────────────
async function main() {
await mkdir(CACHE, { recursive: true });
const rawPath = join(DATA, 'artists-raw.json');
const existing = existsSync(rawPath) ? JSON.parse(await readFile(rawPath, 'utf8')) : [];
const byQid = new Map(existing.map((a) => [a.qid, a]));
console.log(`artists-raw.json: ${existing.length} existing artists (P195 harvest)`);
// A) harvest museum rosters
const rosters = [...await harvestNga(), ...await harvestSi(), ...await harvestMet()];
// provenance merge for QIDs we already have (adds e.g. "National Gallery of Art" tag)
let provenanceAdds = 0;
for (const r of rosters) {
if (r.qid && byQid.has(r.qid)) {
const a = byQid.get(r.qid);
if (!a.museums.includes(r.museum)) { a.museums.push(r.museum); provenanceAdds++; }
}
}
// B) roster QIDs not yet in artists-raw → VALUES P109 check
const newQids = [...new Set(rosters.filter((r) => r.qid && !byQid.has(r.qid)).map((r) => r.qid))];
const museumsByQid = new Map();
for (const r of rosters) {
if (!r.qid) continue;
if (!museumsByQid.has(r.qid)) museumsByQid.set(r.qid, new Set());
museumsByQid.get(r.qid).add(r.museum);
}
console.log(`[resolve] ${newQids.length} roster QIDs not in artists-raw → VALUES P109 check`);
const hits = newQids.length ? await valuesCheck(newQids) : [];
let added = 0;
const addArtist = (qid, name, sig, dod, links, occs, museums) => {
byQid.set(qid, {
qid,
wikidata: `http://www.wikidata.org/entity/${qid}`,
full_name: name,
sitelinks: links,
death_date: dod.slice(0, 10),
death_year: +dod.slice(0, 4),
signature_image_url: sig,
museums: [...museums],
occupations: occs,
});
added++;
};
for (const b of hits) {
const qid = b.person.value.split('/').pop();
if (byQid.has(qid)) continue;
addArtist(qid, b.personLabel?.value || qid, b.sig.value, b.dod.value,
parseInt(b.links?.value || '0', 10), (b.occs?.value || '').split('|').filter(Boolean),
museumsByQid.get(qid) || []);
}
// C) QID-less rows (mostly SI) → name+deathyear match against the P109 universe
const universe = await fetchUniverse();
const uniByName = new Map();
for (const u of universe) {
const k = normName(u.name);
if (!uniByName.has(k)) uniByName.set(k, []);
uniByName.get(k).push(u);
}
let nameHits = 0, ambiguous = 0;
for (const r of rosters) {
if (r.qid) continue;
const cands = uniByName.get(normName(r.name)) || [];
if (!cands.length) continue;
let pick = null;
if (r.deathYear) {
const dy = cands.filter((c) => c.dod && Math.abs(+c.dod.slice(0, 4) - r.deathYear) <= 1);
if (dy.length === 1) pick = dy[0]; else if (dy.length > 1) { ambiguous++; continue; }
} else if (cands.length === 1) pick = cands[0]; // unique in the deceased-artist P109 universe
else { ambiguous++; continue; }
if (!pick) continue;
if (byQid.has(pick.qid)) {
const a = byQid.get(pick.qid);
if (!a.museums.includes(r.museum)) a.museums.push(r.museum);
} else { addArtist(pick.qid, pick.name, pick.sig, pick.dod, pick.links, pick.occs, [r.museum]); nameHits++; }
}
// backfill raw-QID names (label-service flakiness)
const needLabel = [...byQid.values()].filter((a) => /^Q\d+$/.test(a.full_name)).map((a) => a.qid);
if (needLabel.length) {
const labels = await fetchLabels(needLabel);
for (const a of byQid.values()) if (labels[a.qid]) a.full_name = labels[a.qid];
}
const all = [...byQid.values()].sort((a, b) => b.sitelinks - a.sitelinks);
await writeFile(rawPath, JSON.stringify(all, null, 2));
console.log(`\n[done] artists-raw.json: ${existing.length} → ${all.length} artists`
+ ` (+${added} QID-resolved, +${nameHits} name-matched, ${provenanceAdds} provenance tags added, ${ambiguous} ambiguous dropped)`);
console.log('next: node scripts/build-artists.mjs (Commons verify + license gate → data/artists.json)');
}
main().catch((e) => { console.error(e); process.exit(1); });