auto-save: 2026-07-25T11:35:57 (1 files) — db/ingest-wikimedia-images.mjs

8af535398eba71075d2801ce5dd76ea68e79c49d · 2026-07-25 11:36:04 -0700 · Steve Abrams
Files touched

A db/ingest-wikimedia-images.mjs
Diff

commit 8af535398eba71075d2801ce5dd76ea68e79c49d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Jul 25 11:36:04 2026 -0700

    auto-save: 2026-07-25T11:35:57 (1 files) — db/ingest-wikimedia-images.mjs
---
 db/ingest-wikimedia-images.mjs | 125 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 125 insertions(+)

diff --git a/db/ingest-wikimedia-images.mjs b/db/ingest-wikimedia-images.mjs
new file mode 100644
index 0000000..8e444c9
--- /dev/null
+++ b/db/ingest-wikimedia-images.mjs
@@ -0,0 +1,125 @@
+// Pulls cast headshots from Wikimedia ONLY when the per-file license is free
+// for commercial reuse (PD / CC0 / CC-BY / CC-BY-SA). Rejects NC/ND/fair-use/
+// non-free. Records file_url + license + attribution into the assets rights-ledger
+// and links cast_people.headshot_asset_id. Never assumes a Commons file is free.
+// Run: node db/ingest-wikimedia-images.mjs
+import pg from "pg";
+
+const DATABASE_URL =
+  process.env.DATABASE_URL ?? "postgresql://localhost/nineoh_guide?host=/tmp";
+const pool = new pg.Pool({ connectionString: DATABASE_URL });
+const UA = "nineoh-guide/0.1 (unofficial fan project; contact: set-at-launch)";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+async function fetchJson(url, tries = 4) {
+  for (let i = 0; i < tries; i++) {
+    try {
+      const r = await fetch(url, { headers: { "User-Agent": UA } });
+      const t = await r.text();
+      return JSON.parse(t); // throws on rate-limit HTML → caught + retried
+    } catch {
+      await sleep(800 * (i + 1)); // backoff
+    }
+  }
+  throw new Error("rate-limited after retries");
+}
+
+const FREE = /^(cc0|cc-by-\d|cc-by-sa-\d|pd|public domain)/i;
+const NONFREE = /(nc|nd|non-free|fair use|copyright)/i;
+
+function stripHtml(s) {
+  return (s || "").replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
+}
+
+async function pageImageFile(name) {
+  // Lead/infobox image filename + original URL from English Wikipedia.
+  const u = `https://en.wikipedia.org/w/api.php?action=query&format=json&prop=pageimages&piprop=original|name&titles=${encodeURIComponent(
+    name
+  )}`;
+  const j = await fetchJson(u);
+  const pages = j.query?.pages ?? {};
+  const page = Object.values(pages)[0];
+  if (!page?.pageimage) return null;
+  return { file: page.pageimage, url: page.original?.source ?? null };
+}
+
+async function commonsLicense(file) {
+  const u = `https://commons.wikimedia.org/w/api.php?action=query&format=json&prop=imageinfo&iiprop=extmetadata|url&titles=${encodeURIComponent(
+    "File:" + file
+  )}`;
+  const j = await fetchJson(u);
+  const pages = j.query?.pages ?? {};
+  const page = Object.values(pages)[0];
+  const info = page?.imageinfo?.[0];
+  if (!info) return null;
+  const m = info.extmetadata ?? {};
+  return {
+    license: (m.License?.value || "").toLowerCase(),
+    licenseName: m.LicenseShortName?.value || "",
+    licenseUrl: m.LicenseUrl?.value || "",
+    artist: stripHtml(m.Artist?.value || m.Credit?.value || ""),
+    url: info.url,
+  };
+}
+
+// Only cast that don't already have a headshot — makes re-runs safe (no dup assets).
+const cast = (
+  await pool.query(`select id, name from cast_people where headshot_asset_id is null order by name`)
+).rows;
+let accepted = 0,
+  skipped = 0;
+
+for (const person of cast) {
+  await sleep(700); // be polite to the Wikimedia API
+  try {
+    const img = await pageImageFile(person.name);
+    if (!img) {
+      console.log(`SKIP  ${person.name} — no Wikipedia lead image`);
+      skipped++;
+      continue;
+    }
+    const lic = await commonsLicense(img.file);
+    if (!lic) {
+      console.log(`SKIP  ${person.name} — no license metadata`);
+      skipped++;
+      continue;
+    }
+    const key = lic.license || lic.licenseName;
+    const free = FREE.test(lic.license) || /public domain|cc0|cc by(?!-nc|-nd)/i.test(lic.licenseName);
+    const blocked = NONFREE.test(lic.license) || /nc|nd|fair use/i.test(lic.licenseName);
+    if (!free || blocked) {
+      console.log(`SKIP  ${person.name} — license "${lic.licenseName || key}" not free-for-reuse`);
+      skipped++;
+      continue;
+    }
+    const licType = /cc0|public domain|^pd/i.test(key) ? "public-domain"
+      : /sa/i.test(key) ? "cc-by-sa" : "cc-by";
+    const attribution = `${lic.artist || "Unknown"} / ${lic.licenseName || key} (via Wikimedia Commons)`;
+    const url = lic.url || img.url;
+
+    const asset = await pool.query(
+      `insert into assets (type, file_url, thumbnail_url, license_type, license_url,
+                           attribution_text, usage_scope, source_url, rights_notes, approved_for_marketing)
+       values ('image',$1,$1,$2,$3,$4,'editorial',$5,$6,false)
+       returning id`,
+      [
+        url,
+        licType,
+        lic.licenseUrl || null,
+        attribution,
+        `https://commons.wikimedia.org/wiki/File:${img.file}`,
+        `Wikimedia Commons file, license verified ${new Date().toISOString().slice(0, 10)}`,
+      ]
+    );
+    await pool.query(`update cast_people set headshot_asset_id=$1 where id=$2`, [asset.rows[0].id, person.id]);
+    console.log(`OK    ${person.name} — ${licType} (${lic.licenseName})`);
+    accepted++;
+  } catch (e) {
+    console.log(`SKIP  ${person.name} — error ${e.message}`);
+    skipped++;
+  }
+}
+
+console.log(`\naccepted ${accepted}, skipped ${skipped} of ${cast.length}`);
+console.log("Only free-for-reuse licenses stored; each asset carries attribution + source.");
+await pool.end();
← 8dc29d7 news: index 760 90210 articles (Google News RSS, original+re · back to Nineoh Guide · cast photos: 12/14 headshots from Wikimedia (per-file licens 73371b3 →