← back to AsSeenInMovies
scaffold wikidata-commons-fallback.js — search Commons for headshots when wikidata P18 missing (CC-licenses only)
96cf4db84fe3b28176db97d132e080ffdd7a7e5d · 2026-05-12 15:32:13 -0700 · SteveStudio2
Files touched
A scrapers/wikidata-commons-fallback.js
Diff
commit 96cf4db84fe3b28176db97d132e080ffdd7a7e5d
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue May 12 15:32:13 2026 -0700
scaffold wikidata-commons-fallback.js — search Commons for headshots when wikidata P18 missing (CC-licenses only)
---
scrapers/wikidata-commons-fallback.js | 153 ++++++++++++++++++++++++++++++++++
1 file changed, 153 insertions(+)
diff --git a/scrapers/wikidata-commons-fallback.js b/scrapers/wikidata-commons-fallback.js
new file mode 100644
index 0000000..655408e
--- /dev/null
+++ b/scrapers/wikidata-commons-fallback.js
@@ -0,0 +1,153 @@
+#!/usr/bin/env node
+'use strict';
+// Commons-search fallback for asim_people who have a wikidata_id but no
+// headshot_url (P18 was empty on the wikidata entity, but Commons often
+// still has a "Category:Person Name" with usable photos).
+//
+// Strategy: for each (wd_id, full_name) row missing headshot_url, search
+// Commons for `"<full_name>" filemime:image/jpeg|png` in namespace 6, take
+// the first result, fetch extmetadata, only apply if license is CC-anything
+// (rejects "Fair use" and "Non-free copyright").
+//
+// Polite: batches 1 search/req, 1.1s sleep. UA per Wikimedia policy.
+
+require('dotenv').config();
+const { Pool } = require('pg');
+
+const args = process.argv.slice(2).reduce((m, a) => {
+ const [k, v] = a.replace(/^--/, '').split('=');
+ m[k] = v ?? true; return m;
+}, {});
+
+const LIMIT = parseInt(args.limit || '200', 10);
+const DRY = !!args.dry;
+const SLEEP_MS = parseInt(args.sleep || '1100', 10);
+const UA = 'AsSeenInMovies/0.1 (https://asseeninmovies.com; steve@designerwallcoverings.com)';
+const COMMONS = 'https://commons.wikimedia.org/w/api.php';
+
+const pool = new Pool({
+ connectionString: process.env.DATABASE_URL || 'postgresql:///dw_unified?host=/tmp&user=stevestudio2',
+ max: 4,
+});
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+// CC-* / public-domain only. Reject anything with "fair use", "non-free",
+// "all rights reserved", or no license name at all.
+function isFree(licenseShortName) {
+ if (!licenseShortName) return false;
+ const s = String(licenseShortName).toLowerCase();
+ if (/fair use|non-?free|all rights|copyright/i.test(s)) return false;
+ return /\b(cc|public domain|pd|cc0|cc by|cc-by)\b/i.test(s);
+}
+
+function stripHtml(s) {
+ if (!s) return null;
+ return String(s).replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim() || null;
+}
+
+async function searchOne(name) {
+ // Commons search; namespace 6 = File. Quote name for phrase match.
+ const url = new URL(COMMONS);
+ url.search = new URLSearchParams({
+ action: 'query',
+ list: 'search',
+ srsearch: `"${name}" filemime:image/jpeg|filemime:image/png`,
+ srnamespace: '6',
+ srlimit: '3',
+ format: 'json',
+ origin: '*',
+ }).toString();
+ const r = await fetch(url.toString(), { headers: { 'user-agent': UA } });
+ if (!r.ok) throw new Error(`search ${r.status}`);
+ const j = await r.json();
+ const hits = (j.query && j.query.search) || [];
+ return hits.map(h => h.title); // "File:Foo Bar.jpg"
+}
+
+async function imageInfo(title) {
+ const url = new URL(COMMONS);
+ url.search = new URLSearchParams({
+ action: 'query',
+ titles: title,
+ prop: 'imageinfo',
+ iiprop: 'url|extmetadata',
+ format: 'json',
+ origin: '*',
+ }).toString();
+ const r = await fetch(url.toString(), { headers: { 'user-agent': UA } });
+ if (!r.ok) throw new Error(`info ${r.status}`);
+ const j = await r.json();
+ const pages = (j.query && j.query.pages) || {};
+ const first = Object.values(pages)[0];
+ if (!first || !first.imageinfo || !first.imageinfo[0]) return null;
+ const ii = first.imageinfo[0];
+ const md = ii.extmetadata || {};
+ return {
+ url: ii.url,
+ title,
+ artist: stripHtml((md.Artist && md.Artist.value) || null),
+ license: stripHtml((md.LicenseShortName && md.LicenseShortName.value) || null),
+ licenseUrl: (md.LicenseUrl && md.LicenseUrl.value) || null,
+ descriptionUrl: ii.descriptionurl,
+ };
+}
+
+(async () => {
+ // Order by credit count DESC — most-credited (== most likely to have a
+ // Commons photo) first. Random ordering put set decorators at the top.
+ const sel = await pool.query(`
+ SELECT p.id, p.full_name, p.wikidata_id, COALESCE(cc.c, 0) AS credit_count
+ FROM asim_people p
+ LEFT JOIN (SELECT person_id, count(*) AS c FROM asim_credits GROUP BY person_id) cc
+ ON cc.person_id = p.id
+ WHERE p.wikidata_id IS NOT NULL
+ AND p.headshot_url IS NULL
+ AND p.full_name IS NOT NULL
+ AND length(p.full_name) BETWEEN 4 AND 80
+ ORDER BY cc.c DESC NULLS LAST, p.id
+ LIMIT $1`, [LIMIT]);
+
+ console.log(`[commons-fallback] candidates=${sel.rows.length} dry=${DRY}`);
+ let tried = 0, found = 0, applied = 0, skip = 0;
+
+ for (const row of sel.rows) {
+ tried++;
+ let titles;
+ try { titles = await searchOne(row.full_name); }
+ catch (e) { console.error(` search ${row.full_name}: ${e.message}`); await sleep(SLEEP_MS); continue; }
+
+ if (!titles.length) { await sleep(SLEEP_MS); continue; }
+
+ // Take the first hit only. Future: rank by file name match strength.
+ const title = titles[0];
+ let info;
+ try { info = await imageInfo(title); }
+ catch (e) { console.error(` info ${title}: ${e.message}`); await sleep(SLEEP_MS); continue; }
+ if (!info) { await sleep(SLEEP_MS); continue; }
+
+ found++;
+ if (!isFree(info.license)) {
+ skip++;
+ if (tried % 25 === 0) console.log(` ${tried}/${sel.rows.length} found=${found} applied=${applied} skip-license=${skip}`);
+ await sleep(SLEEP_MS); continue;
+ }
+
+ if (!DRY) {
+ await pool.query(`
+ UPDATE asim_people
+ SET headshot_url = $1,
+ headshot_credit = $2,
+ headshot_license = $3,
+ headshot_source_url = $4
+ WHERE id = $5
+ AND headshot_url IS NULL`,
+ [info.url, info.artist || 'Wikimedia Commons', info.license, info.descriptionUrl, row.id]);
+ applied++;
+ }
+ if (tried % 25 === 0) console.log(` ${tried}/${sel.rows.length} found=${found} applied=${applied} skip-license=${skip}`);
+ await sleep(SLEEP_MS);
+ }
+ console.log(`[commons-fallback] done. tried=${tried} found=${found} applied=${applied} skip-license=${skip}`);
+ await pool.end();
+})().catch(e => { console.error(e); process.exit(1); });
← 40aa91f wikidata-enrich: add --byCredits to rank people by credit co
·
back to AsSeenInMovies
·
asseeninmovies: /movies + /people sort dropdowns + density s 67784fe →