← back to CelebritySignatures
scripts/fetch-wikidata.mjs
311 lines
#!/usr/bin/env node
// CelebritySignatures — verified-signature fetcher.
// Spine: Wikidata P109 ("signature") links a real person to a real signature
// FILE on Wikimedia Commons. We query the public SPARQL endpoint (no key, free),
// rank by cross-wiki sitelink count, then pull each file's license from the
// Commons API. Nothing is invented — every row traces to a fetchable file.
//
// Usage: node scripts/fetch-wikidata.mjs [category|all]
// Output: data/<category>.json + data/celebrity_signatures.{json,csv}
import { writeFile, mkdir } from 'node:fs/promises';
const UA = 'CelebritySignatures-research/1.0 (steve@designerwallcoverings.com)';
const SPARQL = 'https://query.wikidata.org/sparql';
const COMMONS = 'https://commons.wikimedia.org/w/api.php';
const PER_CAT = 320; // fetch headroom (cross-category dedupe is hungry), trimmed to 100
const TARGET = 100;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// Authoritative human-curated ranking lists per category (cited as guidance;
// the automated `rank` is the Wikidata sitelink-sorted position).
const RANKING_SOURCES = {
politics: ['https://www.gallup.com/poll/most-admired-man-woman.aspx', 'https://time.com/time100/'],
sports: ['https://www.espn.com/', 'https://www.olympics.com/en/athletes', 'https://www.laureus.com/'],
movies: ['https://www.imdb.com/chart/starmeter/', 'https://www.forbes.com/celebrities/'],
tv: ['https://www.emmys.com/', 'https://yougov.com/ratings/'],
hollywood: ['https://www.afi.com/afis-100-years-100-stars/', 'https://walkoffame.com/'],
declaration: ['https://www.archives.gov/founding-docs/declaration', 'https://founders.archives.gov/'],
};
// SPARQL WHERE body per category. Must bind ?person, ?links; ?sig where available.
const CATEGORIES = {
politics: {
label: 'Politics',
body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q82955 ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
},
sports: {
label: 'Sports',
body: `?person wdt:P31 wd:Q5 ; wdt:P641 ?sport ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
},
movies: {
label: 'Movies',
body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links .`,
},
tv: {
label: 'TV',
body: `?person wdt:P31 wd:Q5 ; wdt:P106 ?occ ; wdt:P109 ?sig ; wikibase:sitelinks ?links . VALUES ?occ { wd:Q10798782 wd:Q947873 }`,
},
hollywood: {
// Golden-Age / iconic: film actors who are deceased (cleanest publicity posture).
label: 'Hollywood',
body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links ; wdt:P570 ?deadrequired .`,
},
moviesclassic: {
// Deceased film actors beyond the Hollywood top tier (dedupe gives the next 100) —
// a usable, publicity-safe "Movies" set (living movie stars are permission-needed).
label: 'Movies (Classic)',
body: `?person wdt:P31 wd:Q5 ; wdt:P106 wd:Q10800557 ; wdt:P109 ?sig ; wikibase:sitelinks ?links ; wdt:P570 ?deadrequired .`,
},
// NOTE: the Declaration of Independence set is owned by scripts/fetch-declaration.mjs
// (MediaWiki/Commons APIs, outage-proof). The combiner below merges its output in.
};
function buildQuery(body, limit) {
return `SELECT DISTINCT ?person ?personLabel ?sig ?dod ?links WHERE {
${body}
OPTIONAL { ?person wdt:P570 ?dod . }
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
} ORDER BY DESC(?links) LIMIT ${limit}`;
}
async function runSparql(query, { retries = 20, waitMs = 70000 } = {}) {
// WDQS periodically throttles to 1 req/min during outages; ride it out.
for (let attempt = 1; ; attempt++) {
const res = await fetch(SPARQL, {
method: 'POST',
headers: {
'User-Agent': UA,
Accept: 'application/sparql-results+json',
'Content-Type': 'application/x-www-form-urlencoded',
},
body: 'query=' + encodeURIComponent(query),
});
if (res.ok) {
const txt = await res.text();
try { return JSON.parse(txt).results.bindings; }
catch { if (attempt > retries) throw new Error('non-JSON from WDQS'); }
}
if (attempt > retries) throw new Error(`SPARQL ${res.status} after ${retries} retries`);
process.stdout.write(`(WDQS ${res.status}; backoff ${waitMs / 1000}s, try ${attempt}/${retries}) `);
await sleep(waitMs);
}
}
function fileTitleFromUrl(sigUrl) {
// .../Special:FilePath/<urlencoded filename>
const m = sigUrl && sigUrl.match(/Special:FilePath\/(.+)$/);
if (!m) return null;
return 'File:' + decodeURIComponent(m[1]).replace(/_/g, ' ');
}
// Pull license/author per Commons file (batches of 50).
async function fetchLicenses(titles) {
const out = {};
for (let i = 0; i < titles.length; i += 50) {
const batch = titles.slice(i, i + 50);
const url = `${COMMONS}?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(batch.join('|'))}&origin=*`;
const res = await fetch(url, { headers: { 'User-Agent': UA } });
if (!res.ok) { await sleep(500); continue; }
const j = await res.json();
const pages = j.query?.pages || {};
const norm = {};
(j.query?.normalized || []).forEach((n) => { norm[n.to] = n.from; });
for (const p of Object.values(pages)) {
const ii = p.imageinfo?.[0];
const ext = ii?.extmetadata || {};
const key = norm[p.title] || p.title;
out[key] = {
license: ext.LicenseShortName?.value || 'unknown',
artist: (ext.Artist?.value || '').replace(/<[^>]+>/g, '').trim(),
directUrl: ii?.url || null,
filePage: ii?.descriptionurl || null,
};
}
await sleep(300);
}
return out;
}
function licenseRisk(lic) {
const l = (lic || '').toLowerCase();
if (/public domain|pd-|cc0|cc-?0/.test(l)) return 'low';
if (/cc by/.test(l)) return 'low-medium';
if (l === 'unknown' || l === '') return 'high';
return 'medium';
}
function assess({ deceased, lic, fromArchive }) {
const lr = fromArchive ? 'low' : licenseRisk(lic);
const pr = deceased ? 'low' : 'medium';
const rank = { low: 0, 'low-medium': 1, medium: 2, high: 3 };
const overall = rank[lr] >= rank[pr] ? lr : pr;
const risk_level = overall.startsWith('low') ? 'low' : overall;
let usable;
const cleanLic = lr === 'low' || lr === 'low-medium';
if (deceased && cleanLic) usable = 'yes';
else if (!deceased) usable = 'permission-needed';
else usable = 'review';
return { risk_level, usable };
}
async function fetchCategory(catKey) {
const cat = CATEGORIES[catKey];
const rows = await runSparql(buildQuery(cat.body, cat.limit || PER_CAT));
// dedupe by person QID, keep highest sitelinks (already sorted)
const seen = new Set();
const people = [];
for (const b of rows) {
const qid = b.person.value.split('/').pop();
if (seen.has(qid)) continue;
seen.add(qid);
people.push({
qid,
name: b.personLabel?.value || qid,
sigUrl: b.sig?.value || null,
dod: b.dod?.value || null,
links: parseInt(b.links?.value || '0', 10),
});
}
return people;
}
function csvCell(v) {
const s = (v == null ? '' : String(v)).replace(/[\r\n]+/g, ' ').replace(/"/g, '""');
return `"${s}"`;
}
// Reliable label resolution via the Wikidata entity API (the SPARQL label SERVICE
// is flaky during WDQS outages and sometimes leaves names as raw QIDs).
async function fetchLabels(qids) {
// Prefer the English Wikipedia sitelink title (robust even when Wikidata's
// label backend is degraded), fall back to the en label.
const out = {};
for (let i = 0; i < qids.length; i += 50) {
const ids = qids.slice(i, i + 50).join('|');
const url = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${ids}&props=sitelinks|labels&sitefilter=enwiki&languages=en&format=json`;
const res = await fetch(url, { headers: { 'User-Agent': UA } });
if (!res.ok) { await sleep(400); continue; }
const j = await res.json();
for (const [qid, e] of Object.entries(j.entities || {})) {
out[qid] = e.sitelinks?.enwiki?.title || e.labels?.en?.value || qid;
}
await sleep(250);
}
return out;
}
async function main() {
const arg = (process.argv[2] || 'all').toLowerCase();
const keys = arg === 'all' ? Object.keys(CATEGORIES) : [arg];
await mkdir('data', { recursive: true });
const globalSeen = new Set();
const all = [];
// priority order keeps a deceased icon in Hollywood/declaration over movies/tv
const priority = ['declaration', 'politics', 'sports', 'hollywood', 'moviesclassic', 'movies', 'tv'];
const order = keys.sort((a, b) => priority.indexOf(a) - priority.indexOf(b));
for (const key of order) {
process.stdout.write(`\n[${key}] querying Wikidata… `);
let people;
try {
people = await fetchCategory(key);
} catch (e) {
console.log(`FAILED: ${e.message}`);
continue;
}
console.log(`${people.length} candidates with a signature on file`);
// global dedupe across categories
people = people.filter((p) => !globalSeen.has(p.qid));
// backfill any name that came back as a raw QID (label-service flakiness)
const needLabel = people.filter((p) => /^Q\d+$/.test(p.name)).map((p) => p.qid);
if (needLabel.length) {
const labels = await fetchLabels(needLabel);
for (const p of people) if (labels[p.qid]) p.name = labels[p.qid];
}
// license lookup
const titles = [...new Set(people.map((p) => fileTitleFromUrl(p.sigUrl)).filter(Boolean))];
process.stdout.write(`[${key}] fetching ${titles.length} Commons licenses… `);
const lic = titles.length ? await fetchLicenses(titles) : {};
console.log('done');
const isDecl = key === 'declaration';
const catRows = [];
for (const p of people) {
if (catRows.length >= (isDecl ? 56 : TARGET)) break;
globalSeen.add(p.qid);
const title = fileTitleFromUrl(p.sigUrl);
const meta = title ? lic[title] : null;
const deceased = !!p.dod || isDecl; // all DoI signers deceased
const hasFile = !!p.sigUrl;
const fromArchive = isDecl && !hasFile;
const license = fromArchive
? 'Public domain (US Gov work / pre-1929)'
: (meta?.license || (hasFile ? 'unknown (verify on Commons)' : 'n/a'));
const { risk_level, usable } = assess({ deceased, lic: license, fromArchive });
const sigImg = fromArchive
? 'https://www.archives.gov/founding-docs/declaration-transcript'
: (meta?.directUrl || p.sigUrl);
const sourceType = fromArchive
? 'US National Archives — engrossed Declaration of Independence (PD)'
: 'Wikimedia Commons (Wikidata P109 signature file)';
const filePage = meta?.filePage
|| (title ? `https://commons.wikimedia.org/wiki/${encodeURIComponent(title)}` : null);
catRows.push({
category: CATEGORIES[key].label,
rank: catRows.length + 1,
full_name: p.name,
wikidata: `https://www.wikidata.org/wiki/${p.qid}`,
reason_for_ranking: `Cross-wiki notability: present on ${p.links} language Wikipedias`,
ranking_source_urls: (RANKING_SOURCES[key] || []).join(' | '),
signature_image_url: sigImg,
signature_source_type: sourceType,
image_license: license,
image_author: meta?.artist || (fromArchive ? 'US Government (1776)' : ''),
deceased: deceased ? 'yes' : 'no',
death_date: p.dod ? p.dod.slice(0, 10) : (isDecl ? 'pre-1830' : ''),
risk_level,
usable_in_commercial_collage: usable,
notes: [
deceased ? 'Deceased — low publicity-rights risk' : 'LIVING — right-of-publicity exposure for commercial use',
/share|sa/i.test(license) && /cc by-?sa/i.test(license) ? 'CC BY-SA: derivative collage may need share-alike + attribution' : '',
fromArchive ? 'No standalone Commons signature file; signature appears on the engrossed Declaration (NARA scan, PD)' : '',
].filter(Boolean).join('; '),
backup_source: isDecl
? 'https://founders.archives.gov/'
: (filePage || 'https://catalog.archives.gov/'),
});
}
await writeFile(`data/${key}.json`, JSON.stringify(catRows, null, 2));
console.log(`[${key}] -> data/${key}.json (${catRows.length} rows, ${catRows.filter(r => r.usable_in_commercial_collage === 'yes').length} collage-usable)`);
all.push(...catRows);
}
// merge in the dedicated Declaration of Independence set (if built)
try {
const { readFile } = await import('node:fs/promises');
const decl = JSON.parse(await readFile('data/declaration.json', 'utf8'));
if (decl.length) { all.unshift(...decl); console.log(`merged ${decl.length} Declaration signers`); }
} catch { /* declaration not built yet */ }
// combined outputs
await writeFile('data/celebrity_signatures.json', JSON.stringify(all, null, 2));
const cols = ['category', 'rank', 'full_name', 'wikidata', 'reason_for_ranking', 'ranking_source_urls', 'signature_image_url', 'signature_source_type', 'image_license', 'image_author', 'deceased', 'death_date', 'risk_level', 'usable_in_commercial_collage', 'notes', 'backup_source'];
const csv = [cols.join(',')].concat(all.map((r) => cols.map((c) => csvCell(r[c])).join(','))).join('\n');
await writeFile('data/celebrity_signatures.csv', csv);
console.log(`\n=== TOTAL ${all.length} rows ===`);
const byUsable = all.reduce((a, r) => ((a[r.usable_in_commercial_collage] = (a[r.usable_in_commercial_collage] || 0) + 1), a), {});
console.log('Collage usability:', byUsable);
console.log('Wrote data/celebrity_signatures.{json,csv}');
}
main().catch((e) => { console.error(e); process.exit(1); });