`scripts/lib/wikidata-common.mjs`

116 lines
// CelebritySignatures — shared Wikidata/Commons machinery.
// Extracted verbatim from scripts/fetch-wikidata.mjs so build-artists.mjs and
// future fetchers reuse one implementation (WDQS backoff, Commons license
// batching, risk assessment, CSV cells, label backfill).

export const UA = 'CelebritySignatures-research/1.0 (steve@designerwallcoverings.com)';
export const SPARQL = 'https://query.wikidata.org/sparql';
export const COMMONS = 'https://commons.wikimedia.org/w/api.php';

export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

export async function runSparql(query, { retries = 20, waitMs = 70000 } = {}) {
  // WDQS periodically throttles to 1 req/min during outages; ride it out.
  for (let attempt = 1; ; attempt++) {
    const res = await fetch(SPARQL, {
      method: 'POST',
      headers: {
        'User-Agent': UA,
        Accept: 'application/sparql-results+json',
        'Content-Type': 'application/x-www-form-urlencoded',
      },
      body: 'query=' + encodeURIComponent(query),
    });
    if (res.ok) {
      const txt = await res.text();
      try { return JSON.parse(txt).results.bindings; }
      catch { if (attempt > retries) throw new Error('non-JSON from WDQS'); }
    }
    if (attempt > retries) throw new Error(`SPARQL ${res.status} after ${retries} retries`);
    process.stdout.write(`(WDQS ${res.status}; backoff ${waitMs / 1000}s, try ${attempt}/${retries}) `);
    await sleep(waitMs);
  }
}

export function fileTitleFromUrl(sigUrl) {
  // .../Special:FilePath/<urlencoded filename>
  const m = sigUrl && sigUrl.match(/Special:FilePath\/(.+)$/);
  if (!m) return null;
  return 'File:' + decodeURIComponent(m[1]).replace(/_/g, ' ');
}

// Pull license/author per Commons file (batches of 50).
export async function fetchLicenses(titles) {
  const out = {};
  for (let i = 0; i < titles.length; i += 50) {
    const batch = titles.slice(i, i + 50);
    const url = `${COMMONS}?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(batch.join('|'))}&origin=*`;
    const res = await fetch(url, { headers: { 'User-Agent': UA } });
    if (!res.ok) { await sleep(500); continue; }
    const j = await res.json();
    const pages = j.query?.pages || {};
    const norm = {};
    (j.query?.normalized || []).forEach((n) => { norm[n.to] = n.from; });
    for (const p of Object.values(pages)) {
      const ii = p.imageinfo?.[0];
      const ext = ii?.extmetadata || {};
      const key = norm[p.title] || p.title;
      out[key] = {
        license: ext.LicenseShortName?.value || 'unknown',
        artist: (ext.Artist?.value || '').replace(/<[^>]+>/g, '').trim(),
        directUrl: ii?.url || null,
        filePage: ii?.descriptionurl || null,
      };
    }
    await sleep(300);
  }
  return out;
}

export function licenseRisk(lic) {
  const l = (lic || '').toLowerCase();
  if (/public domain|pd-|cc0|cc-?0/.test(l)) return 'low';
  if (/cc by/.test(l)) return 'low-medium';
  if (l === 'unknown' || l === '') return 'high';
  return 'medium';
}

export function assess({ deceased, lic, fromArchive }) {
  const lr = fromArchive ? 'low' : licenseRisk(lic);
  const pr = deceased ? 'low' : 'medium';
  const rank = { low: 0, 'low-medium': 1, medium: 2, high: 3 };
  const overall = rank[lr] >= rank[pr] ? lr : pr;
  const risk_level = overall.startsWith('low') ? 'low' : overall;
  let usable;
  const cleanLic = lr === 'low' || lr === 'low-medium';
  if (deceased && cleanLic) usable = 'yes';
  else if (!deceased) usable = 'permission-needed';
  else usable = 'review';
  return { risk_level, usable };
}

export function csvCell(v) {
  const s = (v == null ? '' : String(v)).replace(/[\r\n]+/g, ' ').replace(/"/g, '""');
  return `"${s}"`;
}

// Reliable label resolution via the Wikidata entity API (the SPARQL label SERVICE
// is flaky during WDQS outages and sometimes leaves names as raw QIDs).
export async function fetchLabels(qids) {
  // Prefer the English Wikipedia sitelink title (robust even when Wikidata's
  // label backend is degraded), fall back to the en label.
  const out = {};
  for (let i = 0; i < qids.length; i += 50) {
    const ids = qids.slice(i, i + 50).join('|');
    const url = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${ids}&props=sitelinks|labels&sitefilter=enwiki&languages=en&format=json`;
    const res = await fetch(url, { headers: { 'User-Agent': UA } });
    if (!res.ok) { await sleep(400); continue; }
    const j = await res.json();
    for (const [qid, e] of Object.entries(j.entities || {})) {
      out[qid] = e.sitelinks?.enwiki?.title || e.labels?.en?.value || qid;
    }
    await sleep(250);
  }
  return out;
}