`scripts/build-artists.mjs`

161 lines
#!/usr/bin/env node
// Turn data/artists-raw.json into final catalog records matching the
// celebrity_signatures.json schema. Verifies every P109 signature file exists
// AND reads its license via the free Wikimedia Commons imageinfo API (batched
// 50/call). Only files that resolve + are PD/CC0/signature-class are kept.
// Optional --enrich runs a LOCAL Ollama (qwen3:14b) pass to phrase `notes`
// ($0, grounded in real fields, deterministic fallback). No paid APIs.

import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';

const ROOT = fileURLToPath(new URL('..', import.meta.url));
const DATA = join(ROOT, 'data');
const ENRICH = process.argv.includes('--enrich');
const COMMONS = 'https://commons.wikimedia.org/w/api.php';
const UA = 'CelebritySignatures/1.0 (designerwallcoverings.com; PD-signature artist catalog)';

const raw = JSON.parse(await readFile(join(DATA, 'artists-raw.json'), 'utf8'));

// filename out of a Special:FilePath URL → "File:Name.svg"
function fileTitle(sigUrl) {
  const seg = sigUrl.split('/Special:FilePath/').pop();
  return 'File:' + decodeURIComponent(seg).replace(/_/g, ' ');
}

const sleep = ms => new Promise(r => setTimeout(r, ms));
async function commonsBatch(titles) {
  const url = `${COMMONS}?action=query&format=json&prop=imageinfo`
    + `&iiprop=url|extmetadata&iiextmetadatafilter=LicenseShortName|License`
    + `&titles=${encodeURIComponent(titles.join('|'))}&origin=*`;
  // retry with exponential backoff on 429/5xx (shared IP gets rate-limited when
  // the roster harvester is also hitting Commons).
  let res;
  for (let attempt = 0; attempt < 10; attempt++) {
    res = await fetch(url, { headers: { 'User-Agent': UA } });
    if (res.ok) break;
    if (res.status === 429 || res.status >= 500) {
      // Commons 429s outlast short backoffs — honor Retry-After, else wait long.
      const ra = parseInt(res.headers.get('retry-after') || '0', 10);
      const wait = Math.max(ra * 1000, 15000 * (attempt + 1));
      process.stdout.write(`(commons ${res.status}; ${Math.round(wait / 1000)}s backoff) `);
      await sleep(wait);
      continue;
    }
    throw new Error(`commons ${res.status}`);
  }
  if (!res.ok) throw new Error(`commons ${res.status} after retries`);
  const pages = (await res.json())?.query?.pages || {};
  const out = {};
  for (const p of Object.values(pages)) {
    if (p.missing !== undefined || !p.imageinfo) continue;
    const ii = p.imageinfo[0];
    const em = ii.extmetadata || {};
    out[p.title] = {
      url: ii.url,
      license: (em.LicenseShortName?.value || em.License?.value || '').trim(),
    };
  }
  return out;
}

// Is this license PD / signature-class / CC0 → safe for the deceased-only PD catalog?
function isPD(license) {
  const l = (license || '').toLowerCase();
  if (!l) return true; // no license tag on Commons signatures ≈ below-threshold PD
  return /public domain|^pd|pd-|cc0|cc-0|no known copyright|expired/.test(l);
}

// Build title→artist map, batch-verify 50 at a time.
const byTitle = new Map();
for (const a of raw) byTitle.set(fileTitle(a.signature_image_url), a);
const titles = [...byTitle.keys()];

const kept = [];
const dropped = { missing: 0, nonPD: 0 };
for (let i = 0; i < titles.length; i += 50) {
  const chunk = titles.slice(i, i + 50);
  const info = await commonsBatch(chunk);
  await sleep(1500); // polite inter-batch throttle to avoid re-tripping 429
  for (const t of chunk) {
    const a = byTitle.get(t);
    const meta = info[t];
    if (!meta) { dropped.missing++; continue; }
    if (!isPD(meta.license)) { dropped.nonPD++; continue; }
    a._directUrl = meta.url;
    a._license = meta.license || 'Public domain (signature — below threshold of originality)';
    kept.push(a);
  }
  process.stdout.write(`\r  verified ${Math.min(i + 50, titles.length)}/${titles.length} (kept ${kept.length})`);
}
console.log(`\n  dropped: ${dropped.missing} missing, ${dropped.nonPD} non-PD`);

// famous-first, then assign per-category rank
kept.sort((a, b) => b.sitelinks - a.sitelinks);

const artPrimary = ['painter', 'sculptor', 'printmaker', 'draughtsperson', 'illustrator',
  'photographer', 'architect', 'designer', 'engraver', 'artist', 'animator'];
function primaryRole(occs) {
  for (const p of artPrimary) { const hit = occs.find(o => o.toLowerCase() === p); if (hit) return hit; }
  return occs[0] || 'artist';
}

const records = kept.map((a, idx) => {
  const role = primaryRole(a.occupations);
  const museumList = a.museums.join(', ');
  return {
    category: 'Artists',
    rank: idx + 1,
    full_name: a.full_name,
    wikidata: a.wikidata.replace('http://www.wikidata.org/entity/', 'https://www.wikidata.org/wiki/').replace('http://', 'https://'),
    reason_for_ranking: `${role[0].toUpperCase() + role.slice(1)} in the permanent collections of ${museumList}; cross-wiki notability: ${a.sitelinks} language Wikipedias`,
    ranking_source_urls: 'https://www.metmuseum.org/art/collection | https://www.si.edu/collections',
    signature_image_url: a._directUrl,
    signature_source_type: 'Wikimedia Commons (Wikidata P109 signature file)',
    image_license: a._license,
    image_author: '',
    deceased: 'yes',
    death_date: a.death_year.toString(),
    risk_level: 'low',
    usable_in_commercial_collage: 'yes',
    notes: `Public-domain signature; ${role} deceased since ${a.death_year} (>50 yr) — no copyright or right-of-publicity exposure. Works held by ${a.museums.length} major museum${a.museums.length > 1 ? 's' : ''}.`,
    backup_source: a.signature_image_url.replace(/^http:/, 'https:'),
    _museums: a.museums,
    _occupations: a.occupations,
  };
});

// ---- optional LOCAL Ollama phrasing pass ($0, grounded, safe fallback) ----
if (ENRICH) {
  // local Ollama only ($0). qwen3:14b lives on Mac1 — override host/model via env.
  const OLLAMA_URL = process.env.OLLAMA_URL || 'http://localhost:11434';
  const OLLAMA_MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
  console.log(`local-model notes pass (${OLLAMA_MODEL} @ ${OLLAMA_URL})…`);
  const BATCH = 20;
  for (let i = 0; i < records.length; i += BATCH) {
    const chunk = records.slice(i, i + BATCH);
    const facts = chunk.map((r, j) => ({ i: j, name: r.full_name, role: r._occupations[0] || 'artist', died: r.death_date, museums: r._museums }));
    const prompt = `You write one-sentence collector notes for a signature-art catalog. For each artist below, write ONE vivid factual sentence (max 22 words) about why their autograph is desirable, using ONLY the given facts (do not invent works, prizes, or dates). Return ONLY a JSON array of {"i":<index>,"note":"<sentence>"}.\n\n${JSON.stringify(facts)}`;
    try {
      const res = await fetch(`${OLLAMA_URL}/api/generate`, {
        method: 'POST',
        body: JSON.stringify({ model: OLLAMA_MODEL, prompt, stream: false, options: { temperature: 0.4 }, format: 'json' }),
      });
      const out = JSON.parse((await res.json()).response);
      const arr = Array.isArray(out) ? out : (out.notes || out.result || []);
      for (const o of arr) { const r = chunk[o.i]; if (r && o.note && o.note.length > 15) r.notes = String(o.note).trim(); }
    } catch (e) { /* keep deterministic notes on any failure */ }
    process.stdout.write(`\r  enriched ${Math.min(i + BATCH, records.length)}/${records.length}`);
  }
  console.log('');
}

// strip private fields
for (const r of records) { delete r._museums; delete r._occupations; }

await writeFile(join(DATA, 'artists.json'), JSON.stringify(records, null, 2));
console.log(`\nwrote data/artists.json — ${records.length} artists`);
console.log('top 10:');
records.slice(0, 10).forEach(r => console.log(`  ${r.rank}. ${r.full_name} (d.${r.death_date}) — ${r.image_license}`));