← back to Paul Conrad Cartoons Shadowman

scripts/fetch-pd-cartoons.mjs

243 lines

#!/usr/bin/env node
// Inkwell — public-domain cartoon gallery builder.
// Replaces the text-only research-records section: pulls cartoons / caricatures / satirical prints that
// the holding institution itself marks public domain or CC0, downloads a local display copy, and writes
// data/pd-cartoons.json for /api/pd-cartoons.
//
// Sources ($0):
//   - DPLA (libraries, universities, archives)  api.dp.la  rights === rightsstatements.org NoC-US
//     (key: DPLA_API_KEY from env or ~/Projects/secrets-manager/.env — same resolution as paul-conrad-archive)
//   - Art Institute of Chicago  api.artic.edu        rights: is_public_domain === true
//   - The Metropolitan Museum    collectionapi.metmuseum.org  rights: isPublicDomain === true
//   - Cleveland Museum of Art    openaccess-api.clevelandart.org  rights: share_license_status === 'CC0'
// An item is kept ONLY when its source's own rights field says public domain / CC0 — never inferred from date.
//
// Usage: node scripts/fetch-pd-cartoons.mjs [--per-source 30]

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const OUT_JSON = path.join(ROOT, 'data/pd-cartoons.json');
const IMG_DIR = path.join(ROOT, 'public/pd');
const argIdx = process.argv.indexOf('--per-source');
const PER_SOURCE = argIdx > -1 ? Number(process.argv[argIdx + 1]) : 30;
const UA = 'Inkwell/1.0 (private research tool; steve@designerwallcoverings.com)';
// Museum queries deliberately omit bare "cartoon": in museum catalogs it mostly means a full-scale
// preparatory design drawing (tapestry/fresco/textile cartoons), not a satirical cartoon.
const QUERIES = ['political cartoon', 'caricature', 'satire', 'satirical print'];
// Keep works on paper; drop ceramics, sculpture, textiles, etc. that match "caricature" by subject.
const PAPER = /print|drawing|lithograph|etching|engraving|woodcut|wood engraving|graphite|ink|charcoal|watercolor|photomechanical|periodical|book/i;
// Relevance guard: "cartoon" also means a full-scale design drawing (Raphael Cartoons), and "caricature"
// matches textiles, paintings on ivory, album leaves, etc. Drop those.
const NOT_CARTOON = /raphael cartoons|cartoon for|\bcotton\b|\bsilk\b|weave|tempera|ivory|ceramic|porcelain|landscape|recto|verso|mezzotint|drypoint|\bpapers:|films, ltd/i;
// Topical guard for sources whose keyword search is loose (AIC, DPLA): the title or the record's own
// subject/term tags must say cartoon / caricature / satire.
const RELEVANT = /cartoon|caricat|satir|puck\b|punch\b|charivari|lampoon|burlesque|parody|humor|comic/i;
// The naming rule (TK-12230) applies to this gallery too.
const BANNED = /conrad/i;

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

async function getJson(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json' } });
    if (res.ok) return res.json();
    if (attempt === 3 || (res.status < 500 && res.status !== 429)) throw new Error(`HTTP ${res.status} ${url}`);
    await sleep(1000 * 2 ** attempt);
  }
}

async function fromAIC() {
  const out = [];
  const fields = 'id,title,artist_title,date_display,date_start,image_id,is_public_domain,classification_title,medium_display,credit_line,subject_titles,term_titles,category_titles';
  for (const q of QUERIES) {
    const u = new URL('https://api.artic.edu/api/v1/artworks/search');
    u.searchParams.set('q', q);
    u.searchParams.set('query[term][is_public_domain]', 'true');
    u.searchParams.set('limit', '40');
    u.searchParams.set('fields', fields);
    const d = await getJson(u);
    for (const r of d.data || []) {
      if (r.is_public_domain !== true || !r.image_id) continue;
      if (!PAPER.test(`${r.classification_title} ${r.medium_display}`)) continue;
      if (NOT_CARTOON.test(`${r.title} ${r.medium_display}`)) continue;
      if (!RELEVANT.test([r.title, ...(r.subject_titles || []), ...(r.term_titles || []), ...(r.category_titles || [])].join(' '))) continue;
      out.push({
        id: `aic-${r.id}`, source: 'Art Institute of Chicago', source_key: 'aic',
        title: r.title, artist: r.artist_title || 'Unknown artist', date: r.date_display, year: r.date_start || null,
        medium: r.medium_display || r.classification_title || null,
        image_remote: `https://www.artic.edu/iiif/2/${r.image_id}/full/843,/0/default.jpg`,
        source_url: `https://www.artic.edu/artworks/${r.id}`,
        rights: 'Public domain (Art Institute of Chicago open access, CC0)', credit: r.credit_line || null,
      });
    }
    await sleep(300);
  }
  return out;
}

async function fromMet() {
  const out = [];
  const seen = new Set();
  for (const q of QUERIES) {
    const u = `https://collectionapi.metmuseum.org/public/collection/v1/search?hasImages=true&q=${encodeURIComponent(q)}`;
    let d;
    // The Met's edge intermittently 403s bursts; one blocked query must not drop the whole source.
    try { d = await getJson(u); } catch (e) { console.warn('met query skipped:', q, e.message); await sleep(2000); continue; }
    for (const id of (d.objectIDs || []).slice(0, 40)) {
      if (seen.has(id)) continue;
      seen.add(id);
      let r;
      try { r = await getJson(`https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`); } catch { continue; }
      await sleep(120); // Met asks for <= 80 req/s; stay far under it
      if (r.isPublicDomain !== true || !r.primaryImageSmall) continue;
      if (!PAPER.test(`${r.classification} ${r.medium}`)) continue;
      if (NOT_CARTOON.test(`${r.title} ${r.medium}`)) continue;
      out.push({
        id: `met-${r.objectID}`, source: 'The Metropolitan Museum of Art', source_key: 'met',
        title: r.title, artist: r.artistDisplayName || 'Unknown artist', date: r.objectDate, year: r.objectBeginDate || null,
        medium: r.medium || r.classification || null,
        image_remote: r.primaryImageSmall,
        source_url: r.objectURL,
        rights: 'Public domain (The Met Open Access, CC0)', credit: r.creditLine || null,
      });
    }
  }
  return out;
}

async function fromCleveland() {
  const out = [];
  for (const q of QUERIES) {
    const u = `https://openaccess-api.clevelandart.org/api/artworks/?q=${encodeURIComponent(q)}&cc0=1&has_image=1&limit=40`;
    const d = await getJson(u);
    for (const r of d.data || []) {
      const img = r.images && (r.images.web || r.images.print);
      if (r.share_license_status !== 'CC0' || !img) continue;
      if (!/^(Print|Drawing)$/.test(r.type) || NOT_CARTOON.test(`${r.title} ${r.technique}`)) continue;
      out.push({
        id: `cma-${r.id}`, source: 'Cleveland Museum of Art', source_key: 'cma',
        title: r.title, artist: (r.creators && r.creators[0] && r.creators[0].description) || 'Unknown artist',
        date: r.creation_date, year: r.creation_date_earliest || null, medium: r.technique || r.type || null,
        image_remote: img.url,
        source_url: r.url,
        rights: 'Public domain (Cleveland Museum of Art Open Access, CC0)', credit: r.creditline || null,
      });
    }
    await sleep(300);
  }
  return out;
}

function dplaKey() {
  if (process.env.DPLA_API_KEY) return process.env.DPLA_API_KEY;
  try {
    const env = fs.readFileSync(path.join(process.env.HOME, 'Projects/secrets-manager/.env'), 'utf8');
    const m = env.match(/^DPLA_API_KEY=["']?([^"'\n]+)/m);
    return m ? m[1].trim() : null;
  } catch { return null; }
}

const NOC_US = 'http://rightsstatements.org/vocab/NoC-US/1.0/';
const PER_PROVIDER = 8; // keep the gallery from being one archive's run of near-identical scans

async function fromDPLA() {
  const key = dplaKey();
  if (!key) throw new Error('DPLA_API_KEY not set — skipped');
  const out = [];
  const perProvider = new Map();
  for (const q of ['political cartoon', 'editorial cartoon', 'cartoon']) {
    for (let page = 1; page <= 2; page++) {
      const u = new URL('https://api.dp.la/v2/items');
      u.searchParams.set('api_key', key);
      u.searchParams.set('q', q);
      u.searchParams.set('rights', `"${NOC_US}"`);
      u.searchParams.set('page_size', '100');
      u.searchParams.set('page', String(page));
      const d = await getJson(u);
      for (const r of d.docs || []) {
        const sr = r.sourceResource || {};
        if (r.rights !== NOC_US || !r.object) continue; // rights re-checked on the record, not just the query
        const provider = (r.dataProvider && r.dataProvider.name) || String(r.dataProvider || 'Unknown');
        const n = perProvider.get(provider) || 0;
        if (n >= PER_PROVIDER) continue;
        const title = [].concat(sr.title || 'Untitled')[0];
        if (NOT_CARTOON.test(title)) continue;
        const subjects = [].concat(sr.subject || []).map(x => (x && x.name) || '').join(' ');
        if (!RELEVANT.test(`${title} ${subjects}`)) continue;
        const date = [].concat(sr.date || [])[0];
        const display = date && (date.displayDate || date.begin) || null;
        const creator = [].concat(sr.creator || [])[0];
        perProvider.set(provider, n + 1);
        out.push({
          id: `dpla-${r.id}`, source: provider, source_key: 'dpla', via: 'Digital Public Library of America',
          title, artist: creator || 'Unknown artist', date: display,
          year: display && /\d{4}/.test(display) ? Number(display.match(/\d{4}/)[0]) : null,
          medium: [].concat(sr.format || [])[0] || null,
          image_remote: r.object,
          source_url: r.isShownAt || `https://dp.la/item/${r.id}`,
          rights: 'No Copyright – United States (rightsstatements.org NoC-US)', credit: provider,
        });
      }
      await sleep(300);
    }
  }
  return out;
}

async function download(item) {
  const file = `${item.id}.jpg`;
  const dest = path.join(IMG_DIR, file);
  if (!fs.existsSync(dest)) {
    const res = await fetch(item.image_remote, { headers: { 'User-Agent': UA } });
    if (!res.ok) throw new Error(`image HTTP ${res.status}`);
    const buf = Buffer.from(await res.arrayBuffer());
    if (buf.length < 2000) throw new Error('image too small');
    fs.writeFileSync(dest, buf);
  }
  return `/pd/${file}`;
}

// Same work digitized by several libraries (DPLA) arrives under different ids — dedupe on title+date too.
function dedupe(list) {
  const seen = new Set();
  return list.filter(i => {
    const k1 = i.id;
    const k2 = `${String(i.title).toLowerCase().replace(/\W+/g, ' ').trim()}|${i.year || i.date || ''}`;
    if (seen.has(k1) || seen.has(k2)) return false;
    seen.add(k1); seen.add(k2);
    return true;
  });
}

async function main() {
  fs.mkdirSync(IMG_DIR, { recursive: true });
  const sources = { dpla: fromDPLA, aic: fromAIC, met: fromMet, cma: fromCleveland };
  const items = [];
  const report = {};
  for (const [key, fn] of Object.entries(sources)) {
    try {
      const got = dedupe(await fn()).filter(i => !BANNED.test(JSON.stringify(i))).slice(0, PER_SOURCE);
      let ok = 0;
      for (const it of got) {
        try { it.image = await download(it); items.push(it); ok++; } catch (e) { /* skip unreachable image */ }
      }
      report[key] = { candidates: got.length, kept: ok };
    } catch (e) {
      report[key] = { error: e.message };
    }
    console.log(key, JSON.stringify(report[key]));
  }
  if (!items.length) {
    console.error('No public-domain items fetched from any source; leaving existing data untouched.');
    process.exit(1);
  }
  const doc = { generated_at: new Date().toISOString(), sources: report, count: items.length, items };
  fs.writeFileSync(OUT_JSON, JSON.stringify(doc, null, 2));
  console.log(`wrote ${items.length} items -> ${path.relative(ROOT, OUT_JSON)}`);
}

main().catch(e => { console.error(e); process.exit(1); });