← back to CelebritySignatures
Artists groundwork: shared wikidata-common lib (extracted from fetch-wikidata), roster gap-fill harvester (NGA/MET-CSV/SI -> P109), idempotent celebrity_signatures merger, museum tags on grid cards (TK-10181)
4d73f5005b9df762104baa8cf70e5d7c54337fe4 · 2026-08-03 10:19:13 -0700 · Steve Abrams
Files touched
M .gitignoreM public/index.htmlA scripts/fetch-artists-rosters.mjsM scripts/fetch-wikidata.mjsA scripts/lib/wikidata-common.mjsA scripts/merge-artists.mjs
Diff
commit 4d73f5005b9df762104baa8cf70e5d7c54337fe4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Aug 3 10:19:13 2026 -0700
Artists groundwork: shared wikidata-common lib (extracted from fetch-wikidata), roster gap-fill harvester (NGA/MET-CSV/SI -> P109), idempotent celebrity_signatures merger, museum tags on grid cards (TK-10181)
---
.gitignore | 1 +
public/index.html | 7 +-
scripts/fetch-artists-rosters.mjs | 383 ++++++++++++++++++++++++++++++++++++++
scripts/fetch-wikidata.mjs | 114 +-----------
scripts/lib/wikidata-common.mjs | 115 ++++++++++++
scripts/merge-artists.mjs | 58 ++++++
6 files changed, 565 insertions(+), 113 deletions(-)
diff --git a/.gitignore b/.gitignore
index e966f20..6fc2d95 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,4 @@ data/sessions.json
.deploy.conf
tmp_oldest_cache/
tmp_theme_cache/
+tmp_artists_cache/
diff --git a/public/index.html b/public/index.html
index 3b6b03e..65259e3 100644
--- a/public/index.html
+++ b/public/index.html
@@ -107,7 +107,7 @@ function render() {
let rows = DATA.filter(r => (prefs.cat==='All' || r.category===prefs.cat)
&& (prefs.use==='all' || r.usable_in_commercial_collage===prefs.use));
- const cats = ['Oldest Signatures','Declaration of Independence','Politics'];
+ const cats = ['Oldest Signatures','Declaration of Independence','Politics','Artists'];
const sorters = {
cat: (a,b) => cats.indexOf(a.category)-cats.indexOf(b.category) || a.rank-b.rank,
name: (a,b) => a.full_name.localeCompare(b.full_name),
@@ -129,13 +129,14 @@ function render() {
<div class="name">${r.full_name}</div>
<div class="row"><span class="tag">${drill('cat',r.category)}</span><span class="badge b-${r.risk_level}">${r.risk_level}</span><span class="badge ${uClass}">${drill('use',ub,uText)}</span></div>
<div class="sub">${r.deceased==='yes'?'† ':'● living · '}${(r.image_license||'').slice(0,28)}</div>
+ ${Array.isArray(r.museums)&&r.museums.length?`<div class="sub" title="${esc(r.museums.join(' · '))}">🏛 ${esc(r.museums.slice(0,2).join(' · '))}${r.museums.length>2?` +${r.museums.length-2}`:''}</div>`:''}
<div class="sub"><a href="${r.wikidata}" target="_blank" rel="noopener noreferrer">Wikidata</a> · <a href="${r.backup_source}" target="_blank" rel="noopener noreferrer">source</a></div>
</div></div>`;
}).join('');
}
function buildChips() {
- const cats = ['All','Oldest Signatures','Declaration of Independence','Politics'];
+ const cats = ['All','Oldest Signatures','Declaration of Independence','Politics','Artists'];
$('#catChips').innerHTML = cats.map(c => `<span class="chip${c===prefs.cat?' on':''}" data-c="${c}">${c}</span>`).join('');
$('#catChips').onclick = (e) => { const c=e.target.dataset.c; if(!c) return; prefs.cat=c; save(); writeURL(true);
document.querySelectorAll('.chip').forEach(x=>x.classList.toggle('on', x.dataset.c===c)); render(); };
@@ -155,7 +156,7 @@ const slug = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g,'-').replace(/^-|-$/g,
let collagesBuilt = false;
function buildCollages() {
if (collagesBuilt) return; collagesBuilt = true;
- const order = ['Oldest Signatures','Declaration of Independence','Politics'];
+ const order = ['Oldest Signatures','Declaration of Independence','Politics','Artists'];
const usable = {}; DATA.forEach(r => { if(r.usable_in_commercial_collage==='yes') usable[r.category]=(usable[r.category]||0)+1; });
const cats = [...new Set([...order, ...Object.keys(usable)])].filter(c => order.includes(c) || usable[c]);
$('#collagesView').innerHTML = cats.map(c => {
diff --git a/scripts/fetch-artists-rosters.mjs b/scripts/fetch-artists-rosters.mjs
new file mode 100644
index 0000000..1fa3282
--- /dev/null
+++ b/scripts/fetch-artists-rosters.mjs
@@ -0,0 +1,383 @@
+#!/usr/bin/env node
+// Roster GAP-FILL for the Artists category. Companion to fetch-artists.mjs
+// (the P195 museum-works harvest): Wikidata's P195 itemization is thin for the
+// National Gallery of Art (DC), NPG, Cooper Hewitt and most Smithsonian units,
+// so famous-enough artists with real P109 signatures get missed. This script
+// harvests the museums' OWN rosters —
+// MET : GitHub CSV dump (Artist Wikidata URL column → direct QIDs)
+// NGA : opendata constituents.csv (wikidataid column → direct QIDs)
+// SI : api.si.edu Open Access (names; needs SI_API_KEY, free api.data.gov)
+// — resolves them against Wikidata P109 + deceased(<1974), and UNIONS the
+// survivors into data/artists-raw.json in fetch-artists.mjs's exact shape
+// (idempotent by QID; safe to re-run after either harvester).
+//
+// Usage: node scripts/fetch-artists-rosters.mjs [--force] [--skip-met]
+// Cost: $0 — free public APIs. Caches in tmp_artists_cache/.
+
+import { writeFile, readFile, mkdir, stat } from 'node:fs/promises';
+import { createWriteStream, createReadStream, existsSync, readFileSync } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { UA, runSparql, fetchLabels, sleep } from './lib/wikidata-common.mjs';
+
+const ROOT = fileURLToPath(new URL('..', import.meta.url));
+const DATA = join(ROOT, 'data');
+const CACHE = join(ROOT, 'tmp_artists_cache');
+const FORCE = process.argv.includes('--force');
+const SKIP_MET = process.argv.includes('--skip-met');
+const DEATH_CUTOFF = 1974; // repo convention: deceased >50yr (see fetch-artists.mjs)
+
+const MET_CSV_URL = 'https://media.githubusercontent.com/media/metmuseum/openaccess/master/MetObjects.csv';
+const NGA_CSV_URL = 'https://raw.githubusercontent.com/NationalGalleryOfArt/opendata/main/data/constituents.csv';
+const SI_API = 'https://api.si.edu/openaccess/api/v1.0/search';
+// unit code → museum display name (matches fetch-artists.mjs label style)
+const SI_UNIT_NAMES = {
+ SAAM: 'Smithsonian American Art Museum',
+ NPG: 'National Portrait Gallery (Smithsonian)',
+ CHNDM: 'Cooper Hewitt, Smithsonian Design Museum',
+ FSG: 'Freer Gallery of Art and Arthur M. Sackler Gallery',
+};
+const SI_MAX_PAGES_PER_UNIT = 60; // 60k records/unit; truncation LOGGED, never silent
+
+// Visual-arts occupations for the name-match universe (SI rows have no QIDs).
+const OCCUPATIONS = ['Q1028181', 'Q1281618', 'Q483501', 'Q11569986', 'Q329439', 'Q644687', 'Q33231', 'Q3391743'];
+
+// ── small utils ──────────────────────────────────────────────────────────────
+const stripDiacritics = (s) => s.normalize('NFD').replace(/[̀-ͯ]/g, '');
+const normName = (s) => stripDiacritics(String(s || '')).toLowerCase().replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
+function deinvert(s) { // "Gogh, Vincent van" → "Vincent van Gogh"
+ const p = String(s || '').split(', ');
+ return (p.length === 2 && p[1] && !/\d/.test(p[1])) ? `${p[1]} ${p[0]}` : s;
+}
+const QUALIFIERS = /^(attributed to|after|workshop of|studio of|school of|style of|manner of|circle of|follower of|copy after|possibly|probably|imitator of|formerly attributed to)\s+/i;
+const cleanName = (raw) => String(raw || '').trim().replace(QUALIFIERS, '').replace(/\s*\([^)]*\)\s*$/, '').trim();
+function deathYearFrom(str) {
+ const s = String(str || '');
+ let m = s.match(/(?:d\.|died)\s*(\d{4})/i);
+ if (m) return +m[1];
+ m = s.match(/(\d{4})\s*[-–—]\s*(\d{4})/);
+ if (m && +m[2] > +m[1] && +m[2] < 2030) return +m[2];
+ return null;
+}
+const cachedFile = (f) => !FORCE && existsSync(join(CACHE, f));
+const readCache = async (f) => JSON.parse(await readFile(join(CACHE, f), 'utf8'));
+const writeCache = async (f, v) => writeFile(join(CACHE, f), JSON.stringify(v));
+
+function siApiKey() {
+ if (process.env.SI_API_KEY) return process.env.SI_API_KEY;
+ for (const p of [join(ROOT, '.env'), `${process.env.HOME}/Projects/secrets-manager/.env`]) {
+ try {
+ const m = readFileSync(p, 'utf8').match(/^SI_API_KEY=(.+)$/m);
+ if (m) return m[1].trim().replace(/^["']|["']$/g, '');
+ } catch { /* next */ }
+ }
+ return null;
+}
+
+// Streaming CSV parser (quote-aware, newlines-in-cells safe, zero-dep).
+async function parseCsvStream(readable, onRow) {
+ let cell = '', row = [], inQ = false, prevQuote = false;
+ const flushCell = () => { row.push(cell); cell = ''; };
+ const flushRow = () => { flushCell(); if (row.length > 1 || row[0] !== '') onRow(row); row = []; };
+ for await (const chunk of readable) {
+ const s = chunk.toString('utf8');
+ for (let i = 0; i < s.length; i++) {
+ const c = s[i];
+ if (inQ) { if (c === '"') { inQ = false; prevQuote = true; } else cell += c; }
+ else if (prevQuote && c === '"') { cell += '"'; inQ = true; prevQuote = false; }
+ else {
+ prevQuote = false;
+ if (c === '"') inQ = true;
+ else if (c === ',') flushCell();
+ else if (c === '\n') flushRow();
+ else if (c !== '\r') cell += c;
+ }
+ }
+ }
+ if (cell !== '' || row.length) flushRow();
+}
+
+async function download(url, dest, minBytes) {
+ if (!FORCE && existsSync(dest) && (await stat(dest)).size >= minBytes) {
+ console.log(` cached ${dest.split('/').pop()}`); return;
+ }
+ console.log(` downloading ${url.split('/').pop()} …`);
+ const res = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow' });
+ if (!res.ok) throw new Error(`download ${res.status}: ${url}`);
+ await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
+ const size = (await stat(dest)).size;
+ if (size < minBytes) throw new Error(`${dest} too small (${size}B) — LFS pointer instead of content?`);
+ console.log(` ok (${(size / 1e6).toFixed(1)}MB)`);
+}
+
+// ── harvesters: {name, deathYear, qid|null, museum} ─────────────────────────
+
+async function harvestMet() {
+ if (cachedFile('roster-met.json')) { console.log('[met] cached'); return readCache('roster-met.json'); }
+ if (SKIP_MET) { console.log('[met] skipped (--skip-met)'); return []; }
+ await download(MET_CSV_URL, join(CACHE, 'MetObjects.csv'), 50e6);
+ const byKey = new Map();
+ let header = null; const idx = {}; let objects = 0;
+ await parseCsvStream(createReadStream(join(CACHE, 'MetObjects.csv')), (cells) => {
+ if (!header) {
+ header = cells;
+ for (const h of ['Artist Display Name', 'Artist End Date', 'Artist Wikidata URL']) {
+ idx[h] = header.findIndex((x) => x.trim() === h);
+ if (idx[h] < 0) throw new Error(`MET CSV missing column ${h} (headers: ${header.slice(0, 30).join(';')})`);
+ }
+ return;
+ }
+ objects++;
+ const names = (cells[idx['Artist Display Name']] || '').split('|');
+ const ends = (cells[idx['Artist End Date']] || '').split('|');
+ const wds = (cells[idx['Artist Wikidata URL']] || '').split('|');
+ for (let i = 0; i < names.length; i++) {
+ const name = cleanName(names[i]);
+ if (!name || name.length < 3) continue;
+ const endRaw = (ends[i] || '').trim();
+ const end = /^\d{4}$/.test(endRaw) && endRaw !== '9999' ? +endRaw : null;
+ const qm = (wds[i] || '').match(/Q\d+/);
+ const key = qm ? qm[0] : normName(name) + '|' + (end || '');
+ if (!byKey.has(key)) byKey.set(key, { name, deathYear: end, qid: qm ? qm[0] : null, museum: 'Metropolitan Museum of Art' });
+ }
+ });
+ const rows = [...byKey.values()];
+ console.log(`[met] ${objects} objects → ${rows.length} unique artists (${rows.filter((r) => r.qid).length} QID'd)`);
+ await writeCache('roster-met.json', rows);
+ return rows;
+}
+
+async function harvestNga() {
+ if (cachedFile('roster-nga.json')) { console.log('[nga] cached'); return readCache('roster-nga.json'); }
+ await download(NGA_CSV_URL, join(CACHE, 'nga-constituents.csv'), 1e6);
+ const rows = [];
+ let header = null; const idx = {};
+ await parseCsvStream(createReadStream(join(CACHE, 'nga-constituents.csv')), (cells) => {
+ if (!header) {
+ header = cells.map((h) => h.trim().toLowerCase());
+ for (const h of ['preferreddisplayname', 'forwarddisplayname', 'endyear', 'artistofngaobject', 'constituenttype', 'wikidataid']) idx[h] = header.indexOf(h);
+ return;
+ }
+ if (idx.artistofngaobject >= 0 && cells[idx.artistofngaobject] !== '1') return;
+ if (idx.constituenttype >= 0 && cells[idx.constituenttype] && cells[idx.constituenttype] !== 'individual') return;
+ const name = cleanName((idx.forwarddisplayname >= 0 && cells[idx.forwarddisplayname]) || deinvert(cells[idx.preferreddisplayname]));
+ if (!name || name.length < 3) return;
+ const end = /^\d{3,4}$/.test(cells[idx.endyear]) ? +cells[idx.endyear] : null;
+ const qm = (cells[idx.wikidataid] || '').match(/Q\d+/);
+ rows.push({ name, deathYear: end, qid: qm ? qm[0] : null, museum: 'National Gallery of Art' });
+ });
+ console.log(`[nga] ${rows.length} artist constituents (${rows.filter((r) => r.qid).length} QID'd)`);
+ await writeCache('roster-nga.json', rows);
+ return rows;
+}
+
+const SI_NAME_LABELS = /artist|maker|painter|sculptor|engraver|printmaker|designer|illustrator|photographer|etcher|draftsman/i;
+async function harvestSi() {
+ if (cachedFile('roster-si.json')) { console.log('[si] cached'); return readCache('roster-si.json'); }
+ const key = siApiKey();
+ if (!key) { console.log('[si] NO SI_API_KEY — skipping Smithsonian'); return []; }
+ const byKey = new Map();
+ for (const [unit, museumName] of Object.entries(SI_UNIT_NAMES)) {
+ let start = 0, page = 0, total = Infinity, got = 0;
+ while (start < total && page < SI_MAX_PAGES_PER_UNIT) {
+ const url = `${SI_API}?q=${encodeURIComponent('unit_code:' + unit)}&start=${start}&rows=1000&api_key=${key}`;
+ let res;
+ try { res = await fetch(url, { headers: { 'User-Agent': UA } }); }
+ catch { await sleep(5000); continue; }
+ if (res.status === 429) { process.stdout.write('(429→60s) '); await sleep(60000); continue; }
+ if (!res.ok) { console.log(`[si:${unit}] HTTP ${res.status} — stopping unit`); break; }
+ const j = await res.json();
+ total = j.response?.rowCount ?? 0;
+ const recs = j.response?.rows || [];
+ if (!recs.length) break;
+ for (const r of recs) {
+ const entries = []
+ .concat((r.content?.freetext?.name || []).filter((n) => SI_NAME_LABELS.test(n.label || '')).map((n) => n.content))
+ .concat((r.content?.indexedStructured?.name || []).filter((s) => typeof s === 'string'));
+ for (const raw of entries) {
+ const dy = deathYearFrom(raw);
+ const name = cleanName(deinvert(String(raw)
+ .replace(/,?\s*\d{4}\s*[-–—]?\s*(\d{4})?\s*$/, '')
+ .replace(/,?\s*(b\.|d\.|born|died|active|ca\.|fl\.)[^,]*$/i, '')));
+ if (!name || name.length < 3 || /\d/.test(name)) continue;
+ const k = normName(name) + '|' + (dy || '');
+ if (!byKey.has(k)) byKey.set(k, { name, deathYear: dy, qid: null, museum: museumName });
+ }
+ }
+ got += recs.length; start += 1000; page++;
+ if (page % 10 === 0) process.stdout.write(`[si:${unit}] ${got}/${total} … `);
+ await sleep(350);
+ }
+ console.log(start < total
+ ? `\n[si:${unit}] TRUNCATED ${got}/${total} (page cap ${SI_MAX_PAGES_PER_UNIT}; unique-name set saturates early — raise cap to go deeper)`
+ : `\n[si:${unit}] complete: ${got} records`);
+ }
+ const rows = [...byKey.values()];
+ console.log(`[si] ${rows.length} unique names across ${Object.keys(SI_UNIT_NAMES).join('/')}`);
+ await writeCache('roster-si.json', rows);
+ return rows;
+}
+
+// ── P109 universe (for name-matching the QID-less SI rows) ──────────────────
+
+async function fetchUniverse() {
+ if (cachedFile('p109-universe.json')) { console.log('[universe] cached'); return readCache('p109-universe.json'); }
+ const byQid = new Map();
+ for (const occ of OCCUPATIONS) {
+ const q = `SELECT ?person ?personLabel ?sig ?dod ?links
+ (GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs) WHERE {
+ ?person wdt:P31 wd:Q5 ; wdt:P106 wd:${occ} ; wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
+ FILTER(YEAR(?dod) < ${DEATH_CUTOFF})
+ OPTIONAL { ?person wdt:P106 ?o . ?o rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
+ SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+ } GROUP BY ?person ?personLabel ?sig ?dod ?links`;
+ process.stdout.write(`[universe] ${occ} … `);
+ const rows = await runSparql(q);
+ let added = 0;
+ for (const b of rows) {
+ const qid = b.person.value.split('/').pop();
+ if (byQid.has(qid)) continue;
+ byQid.set(qid, {
+ qid,
+ name: b.personLabel?.value || qid,
+ sig: b.sig?.value || null,
+ dod: b.dod?.value || null,
+ links: parseInt(b.links?.value || '0', 10),
+ occs: (b.occs?.value || '').split('|').filter(Boolean),
+ });
+ added++;
+ }
+ console.log(`${rows.length} rows (+${added}, total ${byQid.size})`);
+ await sleep(1500);
+ }
+ const rows = [...byQid.values()];
+ await writeCache('p109-universe.json', rows);
+ return rows;
+}
+
+// ── batched VALUES check for roster QIDs missing from artists-raw ───────────
+
+async function valuesCheck(qids) {
+ const out = [];
+ for (let i = 0; i < qids.length; i += 400) {
+ const batch = qids.slice(i, i + 400);
+ const bf = `values-${i / 400}.json`;
+ let rows;
+ if (cachedFile(bf)) rows = await readCache(bf);
+ else {
+ const q = `SELECT ?person ?personLabel ?sig ?dod ?links
+ (GROUP_CONCAT(DISTINCT ?occLabel; separator="|") AS ?occs) WHERE {
+ VALUES ?person { ${batch.map((q2) => `wd:${q2}`).join(' ')} }
+ ?person wdt:P109 ?sig ; wdt:P570 ?dod ; wikibase:sitelinks ?links .
+ FILTER(YEAR(?dod) < ${DEATH_CUTOFF})
+ OPTIONAL { ?person wdt:P106 ?o . ?o rdfs:label ?occLabel . FILTER(LANG(?occLabel)="en") }
+ SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+ } GROUP BY ?person ?personLabel ?sig ?dod ?links`;
+ rows = await runSparql(q);
+ await writeCache(bf, rows);
+ await sleep(1200);
+ }
+ out.push(...rows);
+ process.stdout.write(`[values] batch ${i / 400 + 1}/${Math.ceil(qids.length / 400)} → ${out.length} P109 hits\r`);
+ }
+ console.log();
+ return out;
+}
+
+// ── main ─────────────────────────────────────────────────────────────────────
+
+async function main() {
+ await mkdir(CACHE, { recursive: true });
+ const rawPath = join(DATA, 'artists-raw.json');
+ const existing = existsSync(rawPath) ? JSON.parse(await readFile(rawPath, 'utf8')) : [];
+ const byQid = new Map(existing.map((a) => [a.qid, a]));
+ console.log(`artists-raw.json: ${existing.length} existing artists (P195 harvest)`);
+
+ // A) harvest museum rosters
+ const rosters = [...await harvestNga(), ...await harvestSi(), ...await harvestMet()];
+
+ // provenance merge for QIDs we already have (adds e.g. "National Gallery of Art" tag)
+ let provenanceAdds = 0;
+ for (const r of rosters) {
+ if (r.qid && byQid.has(r.qid)) {
+ const a = byQid.get(r.qid);
+ if (!a.museums.includes(r.museum)) { a.museums.push(r.museum); provenanceAdds++; }
+ }
+ }
+
+ // B) roster QIDs not yet in artists-raw → VALUES P109 check
+ const newQids = [...new Set(rosters.filter((r) => r.qid && !byQid.has(r.qid)).map((r) => r.qid))];
+ const museumsByQid = new Map();
+ for (const r of rosters) {
+ if (!r.qid) continue;
+ if (!museumsByQid.has(r.qid)) museumsByQid.set(r.qid, new Set());
+ museumsByQid.get(r.qid).add(r.museum);
+ }
+ console.log(`[resolve] ${newQids.length} roster QIDs not in artists-raw → VALUES P109 check`);
+ const hits = newQids.length ? await valuesCheck(newQids) : [];
+ let added = 0;
+ const addArtist = (qid, name, sig, dod, links, occs, museums) => {
+ byQid.set(qid, {
+ qid,
+ wikidata: `http://www.wikidata.org/entity/${qid}`,
+ full_name: name,
+ sitelinks: links,
+ death_date: dod.slice(0, 10),
+ death_year: +dod.slice(0, 4),
+ signature_image_url: sig,
+ museums: [...museums],
+ occupations: occs,
+ });
+ added++;
+ };
+ for (const b of hits) {
+ const qid = b.person.value.split('/').pop();
+ if (byQid.has(qid)) continue;
+ addArtist(qid, b.personLabel?.value || qid, b.sig.value, b.dod.value,
+ parseInt(b.links?.value || '0', 10), (b.occs?.value || '').split('|').filter(Boolean),
+ museumsByQid.get(qid) || []);
+ }
+
+ // C) QID-less rows (mostly SI) → name+deathyear match against the P109 universe
+ const universe = await fetchUniverse();
+ const uniByName = new Map();
+ for (const u of universe) {
+ const k = normName(u.name);
+ if (!uniByName.has(k)) uniByName.set(k, []);
+ uniByName.get(k).push(u);
+ }
+ let nameHits = 0, ambiguous = 0;
+ for (const r of rosters) {
+ if (r.qid) continue;
+ const cands = uniByName.get(normName(r.name)) || [];
+ if (!cands.length) continue;
+ let pick = null;
+ if (r.deathYear) {
+ const dy = cands.filter((c) => c.dod && Math.abs(+c.dod.slice(0, 4) - r.deathYear) <= 1);
+ if (dy.length === 1) pick = dy[0]; else if (dy.length > 1) { ambiguous++; continue; }
+ } else if (cands.length === 1) pick = cands[0]; // unique in the deceased-artist P109 universe
+ else { ambiguous++; continue; }
+ if (!pick) continue;
+ if (byQid.has(pick.qid)) {
+ const a = byQid.get(pick.qid);
+ if (!a.museums.includes(r.museum)) a.museums.push(r.museum);
+ } else { addArtist(pick.qid, pick.name, pick.sig, pick.dod, pick.links, pick.occs, [r.museum]); nameHits++; }
+ }
+
+ // backfill raw-QID names (label-service flakiness)
+ const needLabel = [...byQid.values()].filter((a) => /^Q\d+$/.test(a.full_name)).map((a) => a.qid);
+ if (needLabel.length) {
+ const labels = await fetchLabels(needLabel);
+ for (const a of byQid.values()) if (labels[a.qid]) a.full_name = labels[a.qid];
+ }
+
+ const all = [...byQid.values()].sort((a, b) => b.sitelinks - a.sitelinks);
+ await writeFile(rawPath, JSON.stringify(all, null, 2));
+ console.log(`\n[done] artists-raw.json: ${existing.length} → ${all.length} artists`
+ + ` (+${added} QID-resolved, +${nameHits} name-matched, ${provenanceAdds} provenance tags added, ${ambiguous} ambiguous dropped)`);
+ console.log('next: node scripts/build-artists.mjs (Commons verify + license gate → data/artists.json)');
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });
diff --git a/scripts/fetch-wikidata.mjs b/scripts/fetch-wikidata.mjs
index 45ea477..9695e1b 100644
--- a/scripts/fetch-wikidata.mjs
+++ b/scripts/fetch-wikidata.mjs
@@ -9,15 +9,14 @@
// Output: data/<category>.json + data/celebrity_signatures.{json,csv}
import { writeFile, mkdir } from 'node:fs/promises';
+import {
+ runSparql, fileTitleFromUrl, fetchLicenses, licenseRisk, assess,
+ fetchLabels, csvCell,
+} from './lib/wikidata-common.mjs';
-const UA = 'CelebritySignatures-research/1.0 (steve@designerwallcoverings.com)';
-const SPARQL = 'https://query.wikidata.org/sparql';
-const COMMONS = 'https://commons.wikimedia.org/w/api.php';
const PER_CAT = 320; // fetch headroom (cross-category dedupe is hungry), trimmed to 100
const TARGET = 100;
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
// Authoritative human-curated ranking lists per category (cited as guidance;
// the automated `rank` is the Wikidata sitelink-sorted position).
const RANKING_SOURCES = {
@@ -70,86 +69,6 @@ function buildQuery(body, limit) {
} ORDER BY DESC(?links) LIMIT ${limit}`;
}
-async function runSparql(query, { retries = 20, waitMs = 70000 } = {}) {
- // WDQS periodically throttles to 1 req/min during outages; ride it out.
- for (let attempt = 1; ; attempt++) {
- const res = await fetch(SPARQL, {
- method: 'POST',
- headers: {
- 'User-Agent': UA,
- Accept: 'application/sparql-results+json',
- 'Content-Type': 'application/x-www-form-urlencoded',
- },
- body: 'query=' + encodeURIComponent(query),
- });
- if (res.ok) {
- const txt = await res.text();
- try { return JSON.parse(txt).results.bindings; }
- catch { if (attempt > retries) throw new Error('non-JSON from WDQS'); }
- }
- if (attempt > retries) throw new Error(`SPARQL ${res.status} after ${retries} retries`);
- process.stdout.write(`(WDQS ${res.status}; backoff ${waitMs / 1000}s, try ${attempt}/${retries}) `);
- await sleep(waitMs);
- }
-}
-
-function fileTitleFromUrl(sigUrl) {
- // .../Special:FilePath/<urlencoded filename>
- const m = sigUrl && sigUrl.match(/Special:FilePath\/(.+)$/);
- if (!m) return null;
- return 'File:' + decodeURIComponent(m[1]).replace(/_/g, ' ');
-}
-
-// Pull license/author per Commons file (batches of 50).
-async function fetchLicenses(titles) {
- const out = {};
- for (let i = 0; i < titles.length; i += 50) {
- const batch = titles.slice(i, i + 50);
- const url = `${COMMONS}?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(batch.join('|'))}&origin=*`;
- const res = await fetch(url, { headers: { 'User-Agent': UA } });
- if (!res.ok) { await sleep(500); continue; }
- const j = await res.json();
- const pages = j.query?.pages || {};
- const norm = {};
- (j.query?.normalized || []).forEach((n) => { norm[n.to] = n.from; });
- for (const p of Object.values(pages)) {
- const ii = p.imageinfo?.[0];
- const ext = ii?.extmetadata || {};
- const key = norm[p.title] || p.title;
- out[key] = {
- license: ext.LicenseShortName?.value || 'unknown',
- artist: (ext.Artist?.value || '').replace(/<[^>]+>/g, '').trim(),
- directUrl: ii?.url || null,
- filePage: ii?.descriptionurl || null,
- };
- }
- await sleep(300);
- }
- return out;
-}
-
-function licenseRisk(lic) {
- const l = (lic || '').toLowerCase();
- if (/public domain|pd-|cc0|cc-?0/.test(l)) return 'low';
- if (/cc by/.test(l)) return 'low-medium';
- if (l === 'unknown' || l === '') return 'high';
- return 'medium';
-}
-
-function assess({ deceased, lic, fromArchive }) {
- const lr = fromArchive ? 'low' : licenseRisk(lic);
- const pr = deceased ? 'low' : 'medium';
- const rank = { low: 0, 'low-medium': 1, medium: 2, high: 3 };
- const overall = rank[lr] >= rank[pr] ? lr : pr;
- const risk_level = overall.startsWith('low') ? 'low' : overall;
- let usable;
- const cleanLic = lr === 'low' || lr === 'low-medium';
- if (deceased && cleanLic) usable = 'yes';
- else if (!deceased) usable = 'permission-needed';
- else usable = 'review';
- return { risk_level, usable };
-}
-
async function fetchCategory(catKey) {
const cat = CATEGORIES[catKey];
const rows = await runSparql(buildQuery(cat.body, cat.limit || PER_CAT));
@@ -171,31 +90,6 @@ async function fetchCategory(catKey) {
return people;
}
-function csvCell(v) {
- const s = (v == null ? '' : String(v)).replace(/[\r\n]+/g, ' ').replace(/"/g, '""');
- return `"${s}"`;
-}
-
-// Reliable label resolution via the Wikidata entity API (the SPARQL label SERVICE
-// is flaky during WDQS outages and sometimes leaves names as raw QIDs).
-async function fetchLabels(qids) {
- // Prefer the English Wikipedia sitelink title (robust even when Wikidata's
- // label backend is degraded), fall back to the en label.
- const out = {};
- for (let i = 0; i < qids.length; i += 50) {
- const ids = qids.slice(i, i + 50).join('|');
- const url = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${ids}&props=sitelinks|labels&sitefilter=enwiki&languages=en&format=json`;
- const res = await fetch(url, { headers: { 'User-Agent': UA } });
- if (!res.ok) { await sleep(400); continue; }
- const j = await res.json();
- for (const [qid, e] of Object.entries(j.entities || {})) {
- out[qid] = e.sitelinks?.enwiki?.title || e.labels?.en?.value || qid;
- }
- await sleep(250);
- }
- return out;
-}
-
async function main() {
const arg = (process.argv[2] || 'all').toLowerCase();
const keys = arg === 'all' ? Object.keys(CATEGORIES) : [arg];
diff --git a/scripts/lib/wikidata-common.mjs b/scripts/lib/wikidata-common.mjs
new file mode 100644
index 0000000..e25ae6c
--- /dev/null
+++ b/scripts/lib/wikidata-common.mjs
@@ -0,0 +1,115 @@
+// CelebritySignatures — shared Wikidata/Commons machinery.
+// Extracted verbatim from scripts/fetch-wikidata.mjs so build-artists.mjs and
+// future fetchers reuse one implementation (WDQS backoff, Commons license
+// batching, risk assessment, CSV cells, label backfill).
+
+export const UA = 'CelebritySignatures-research/1.0 (steve@designerwallcoverings.com)';
+export const SPARQL = 'https://query.wikidata.org/sparql';
+export const COMMONS = 'https://commons.wikimedia.org/w/api.php';
+
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+export async function runSparql(query, { retries = 20, waitMs = 70000 } = {}) {
+ // WDQS periodically throttles to 1 req/min during outages; ride it out.
+ for (let attempt = 1; ; attempt++) {
+ const res = await fetch(SPARQL, {
+ method: 'POST',
+ headers: {
+ 'User-Agent': UA,
+ Accept: 'application/sparql-results+json',
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ },
+ body: 'query=' + encodeURIComponent(query),
+ });
+ if (res.ok) {
+ const txt = await res.text();
+ try { return JSON.parse(txt).results.bindings; }
+ catch { if (attempt > retries) throw new Error('non-JSON from WDQS'); }
+ }
+ if (attempt > retries) throw new Error(`SPARQL ${res.status} after ${retries} retries`);
+ process.stdout.write(`(WDQS ${res.status}; backoff ${waitMs / 1000}s, try ${attempt}/${retries}) `);
+ await sleep(waitMs);
+ }
+}
+
+export function fileTitleFromUrl(sigUrl) {
+ // .../Special:FilePath/<urlencoded filename>
+ const m = sigUrl && sigUrl.match(/Special:FilePath\/(.+)$/);
+ if (!m) return null;
+ return 'File:' + decodeURIComponent(m[1]).replace(/_/g, ' ');
+}
+
+// Pull license/author per Commons file (batches of 50).
+export async function fetchLicenses(titles) {
+ const out = {};
+ for (let i = 0; i < titles.length; i += 50) {
+ const batch = titles.slice(i, i + 50);
+ const url = `${COMMONS}?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(batch.join('|'))}&origin=*`;
+ const res = await fetch(url, { headers: { 'User-Agent': UA } });
+ if (!res.ok) { await sleep(500); continue; }
+ const j = await res.json();
+ const pages = j.query?.pages || {};
+ const norm = {};
+ (j.query?.normalized || []).forEach((n) => { norm[n.to] = n.from; });
+ for (const p of Object.values(pages)) {
+ const ii = p.imageinfo?.[0];
+ const ext = ii?.extmetadata || {};
+ const key = norm[p.title] || p.title;
+ out[key] = {
+ license: ext.LicenseShortName?.value || 'unknown',
+ artist: (ext.Artist?.value || '').replace(/<[^>]+>/g, '').trim(),
+ directUrl: ii?.url || null,
+ filePage: ii?.descriptionurl || null,
+ };
+ }
+ await sleep(300);
+ }
+ return out;
+}
+
+export function licenseRisk(lic) {
+ const l = (lic || '').toLowerCase();
+ if (/public domain|pd-|cc0|cc-?0/.test(l)) return 'low';
+ if (/cc by/.test(l)) return 'low-medium';
+ if (l === 'unknown' || l === '') return 'high';
+ return 'medium';
+}
+
+export function assess({ deceased, lic, fromArchive }) {
+ const lr = fromArchive ? 'low' : licenseRisk(lic);
+ const pr = deceased ? 'low' : 'medium';
+ const rank = { low: 0, 'low-medium': 1, medium: 2, high: 3 };
+ const overall = rank[lr] >= rank[pr] ? lr : pr;
+ const risk_level = overall.startsWith('low') ? 'low' : overall;
+ let usable;
+ const cleanLic = lr === 'low' || lr === 'low-medium';
+ if (deceased && cleanLic) usable = 'yes';
+ else if (!deceased) usable = 'permission-needed';
+ else usable = 'review';
+ return { risk_level, usable };
+}
+
+export function csvCell(v) {
+ const s = (v == null ? '' : String(v)).replace(/[\r\n]+/g, ' ').replace(/"/g, '""');
+ return `"${s}"`;
+}
+
+// Reliable label resolution via the Wikidata entity API (the SPARQL label SERVICE
+// is flaky during WDQS outages and sometimes leaves names as raw QIDs).
+export async function fetchLabels(qids) {
+ // Prefer the English Wikipedia sitelink title (robust even when Wikidata's
+ // label backend is degraded), fall back to the en label.
+ const out = {};
+ for (let i = 0; i < qids.length; i += 50) {
+ const ids = qids.slice(i, i + 50).join('|');
+ const url = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${ids}&props=sitelinks|labels&sitefilter=enwiki&languages=en&format=json`;
+ const res = await fetch(url, { headers: { 'User-Agent': UA } });
+ if (!res.ok) { await sleep(400); continue; }
+ const j = await res.json();
+ for (const [qid, e] of Object.entries(j.entities || {})) {
+ out[qid] = e.sitelinks?.enwiki?.title || e.labels?.en?.value || qid;
+ }
+ await sleep(250);
+ }
+ return out;
+}
diff --git a/scripts/merge-artists.mjs b/scripts/merge-artists.mjs
new file mode 100644
index 0000000..6b0928e
--- /dev/null
+++ b/scripts/merge-artists.mjs
@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+// Merge data/artists.json (verified Artists rows from build-artists.mjs) into
+// the live catalog data/celebrity_signatures.{json,csv}. Idempotent: strips any
+// prior Artists rows first. Museums provenance is re-joined from
+// artists-raw.json by QID (build-artists.mjs strips its private _museums field).
+//
+// Validators (fail loudly — never merge bad rows):
+// every Artists row deceased==="yes", real Commons signature URL, no QID
+// already present in another category (cross-category dupes are skipped+logged).
+//
+// Usage: node scripts/merge-artists.mjs
+
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = fileURLToPath(new URL('..', import.meta.url));
+const DATA = join(ROOT, 'data');
+const qidOf = (r) => (r.wikidata || '').split('/').pop();
+
+const artists = JSON.parse(await readFile(join(DATA, 'artists.json'), 'utf8'));
+const raw = JSON.parse(await readFile(join(DATA, 'artists-raw.json'), 'utf8'));
+const existing = JSON.parse(await readFile(join(DATA, 'celebrity_signatures.json'), 'utf8'));
+
+const museumsByQid = new Map(raw.map((a) => [a.qid, a.museums || []]));
+const keep = existing.filter((r) => r.category !== 'Artists');
+const liveQids = new Set(keep.map(qidOf).filter(Boolean));
+
+const merged = [];
+let dupes = 0;
+for (const a of artists) {
+ const qid = qidOf(a);
+ if (liveQids.has(qid)) { dupes++; continue; } // already live in another category
+ if (a.deceased !== 'yes') throw new Error(`non-deceased Artists row: ${a.full_name}`);
+ if (!/upload\.wikimedia\.org|commons\.wikimedia\.org/.test(a.signature_image_url)) {
+ throw new Error(`non-Commons signature URL for ${a.full_name}: ${a.signature_image_url}`);
+ }
+ const museums = museumsByQid.get(qid) || [];
+ merged.push({ ...a, rank: merged.length + 1, museums });
+}
+
+const combined = [...keep, ...merged];
+await writeFile(join(DATA, 'celebrity_signatures.json'), JSON.stringify(combined, null, 2));
+
+const cols = ['category', 'rank', 'full_name', 'wikidata', 'reason_for_ranking', 'ranking_source_urls',
+ 'signature_image_url', 'signature_source_type', 'image_license', 'image_author', 'deceased',
+ 'death_date', 'risk_level', 'usable_in_commercial_collage', 'notes', 'backup_source', 'museums'];
+const cell = (v) => `"${(v == null ? '' : String(v)).replace(/[\r\n]+/g, ' ').replace(/"/g, '""')}"`;
+const csv = [cols.join(',')]
+ .concat(combined.map((r) => cols.map((c) => cell(Array.isArray(r[c]) ? r[c].join('|') : r[c])).join(',')))
+ .join('\n');
+await writeFile(join(DATA, 'celebrity_signatures.csv'), csv);
+
+const byMuseum = {};
+merged.forEach((r) => (r.museums || []).forEach((m) => { byMuseum[m] = (byMuseum[m] || 0) + 1; }));
+console.log(`merged ${merged.length} Artists (${dupes} cross-category dupes skipped) → ${combined.length} total rows`);
+console.log('by museum:', Object.entries(byMuseum).sort((a, b) => b[1] - a[1]).map(([m, n]) => `${m}: ${n}`).join(' | '));
+console.log('usable:', merged.filter((r) => r.usable_in_commercial_collage === 'yes').length, '/', merged.length);
← 31e56bf fix: add package.json (type:module) so the ESM server.js can
·
back to CelebritySignatures
·
Artists category: Wikidata P195 museum-artist harvest + PD-s 3c90e38 →