← back to Stayclaim
scripts/fetch-pd-film-images.mjs
498 lines
#!/usr/bin/env node
/**
* Aggregate public-domain film/TV images from multiple sources and upsert
* into pd_film_image. Each adapter returns row objects of shape:
* { source, source_id, title, caption, image_url, thumb_url, width, height,
* year, kind, rights_url, source_url }
*
* Sources (working without API keys):
* wikimedia — Wikimedia Commons (PD-US film categories, broader query)
* archive_org — Internet Archive Feature Films collection
* loc — Library of Congress film/video items
* nara — National Archives Catalog (api.data.gov DEMO_KEY)
* mhdl — Media History Digital Library / Lantern (no key)
* smithsonian — Smithsonian Open Access (DEMO_KEY)
*
* Sources requiring keys (stubbed — skipped when key absent):
* nypl — NYPL Digital Collections (NYPL_API_TOKEN)
* flickr_commons — Flickr Commons (FLICKR_API_KEY)
*
* Run via the existing pastdoor env so DATABASE_URL is set:
* cd /root/public-projects/pastdoor && node scripts/fetch-pd-film-images.mjs --limit=400
*/
import pg from 'pg';
const LIMIT = Number(process.argv.find(a => a.startsWith('--limit='))?.split('=')[1] ?? 400);
const PER_SOURCE = Math.ceil(LIMIT / 8);
const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL, max: 3 });
const UA = 'pastdoor/1.0 (https://wholivedthere.com; steve@designerwallcoverings.com) PD-image-aggregator';
const DATA_GOV_KEY = process.env.DATA_GOV_API_KEY ?? 'DEMO_KEY';
async function fetchJson(url, headers = {}) {
const r = await fetch(url, {
headers: { 'User-Agent': UA, Accept: 'application/json', ...headers },
});
if (!r.ok) throw new Error(`${url} → ${r.status}`);
const ct = r.headers.get('content-type') ?? '';
if (!ct.includes('json')) throw new Error(`${url} → not JSON (${ct})`);
return r.json();
}
// ---------- 1. Wikimedia Commons ----------
// Fixed: use the actual populated PD-film categories. The previous "in_the_public_domain"
// suffixes were sparse/empty. These are populated and reliably PD.
async function fromWikimedia(limit) {
const cats = [
'Public_domain_films',
'Public_domain_film_screenshots',
'Films_in_the_public_domain_in_the_United_States',
'Film_posters_of_the_United_States_in_the_public_domain',
'Lobby_cards', // pre-1928 lobby cards are PD
'Silent_film_screenshots', // mostly pre-1929 → PD
];
const out = [];
const perCat = Math.ceil(limit / cats.length);
for (const cat of cats) {
const url = `https://commons.wikimedia.org/w/api.php?action=query&format=json&generator=categorymembers&gcmtitle=Category:${cat}&gcmtype=file&gcmlimit=${perCat}&prop=imageinfo&iiprop=url|size|extmetadata&iiurlwidth=480&origin=*`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`wikimedia ${cat}: ${e.message}`); continue; }
for (const p of Object.values(data?.query?.pages ?? {})) {
const ii = p.imageinfo?.[0]; if (!ii?.url) continue;
const meta = ii.extmetadata ?? {};
const title = (p.title ?? '').replace(/^File:/, '').replace(/\.[a-z0-9]+$/i, '');
const dateStr = meta.DateTimeOriginal?.value ?? meta.DateTime?.value ?? '';
const yearMatch = String(dateStr).match(/(\d{4})/);
out.push({
source: 'wikimedia',
source_id: String(p.pageid),
title,
caption: (meta.ImageDescription?.value ?? '').replace(/<[^>]+>/g, '').trim().slice(0, 600) || null,
image_url: ii.url,
thumb_url: ii.thumburl ?? null,
width: ii.thumbwidth ?? ii.width ?? null,
height: ii.thumbheight ?? ii.height ?? null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: cat.includes('poster') ? 'poster'
: cat.includes('lobby') ? 'lobby_card'
: cat.includes('screenshot') ? 'screenshot'
: 'still',
rights_url: 'https://commons.wikimedia.org/wiki/Commons:Reusing_content_outside_Wikimedia',
source_url: `https://commons.wikimedia.org/wiki/?curid=${p.pageid}`,
});
}
}
return out.slice(0, limit);
}
// ---------- 2. Internet Archive — feature_films collection ----------
async function fromArchive(limit) {
const url = `https://archive.org/advancedsearch.php?q=collection%3Afeature_films+mediatype%3Amovies&fl%5B%5D=identifier&fl%5B%5D=title&fl%5B%5D=date&fl%5B%5D=description&fl%5B%5D=licenseurl&rows=${limit}&output=json`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`archive: ${e.message}`); return []; }
const out = [];
for (const d of data?.response?.docs ?? []) {
const id = d.identifier; if (!id) continue;
const year = d.date ? Number(String(d.date).slice(0, 4)) : null;
out.push({
source: 'archive_org',
source_id: id,
title: d.title ?? id,
caption: typeof d.description === 'string'
? d.description.replace(/<[^>]+>/g, '').slice(0, 600)
: Array.isArray(d.description)
? d.description.join(' ').replace(/<[^>]+>/g, '').slice(0, 600)
: null,
image_url: `https://archive.org/services/img/${id}`,
thumb_url: `https://archive.org/services/img/${id}`,
width: null, height: null, year,
kind: 'still',
rights_url: d.licenseurl ?? 'https://archive.org/about/terms.php',
source_url: `https://archive.org/details/${id}`,
});
}
return out;
}
// ---------- 3. Library of Congress ----------
async function fromLoc(limit) {
const url = `https://www.loc.gov/film-and-videos/?fa=access-restricted%3Afalse%7Conline-format%3Aimage&fo=json&c=${limit}`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`loc: ${e.message}`); return []; }
const out = [];
for (const r of data?.results ?? []) {
const img = (r.image_url ?? [])[0]; if (!img) continue;
const yearMatch = String(r.dates?.[0] ?? '').match(/(\d{4})/);
out.push({
source: 'loc',
source_id: r.id ?? r.url,
title: (r.title ?? 'Untitled').slice(0, 240),
caption: typeof r.description === 'string' ? r.description.replace(/<[^>]+>/g, '').slice(0, 600) : null,
image_url: img,
thumb_url: img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://www.loc.gov/legal/',
source_url: r.url ?? `https://www.loc.gov${r.id ?? ''}`,
});
}
return out;
}
// ---------- 4. NARA — National Archives Catalog ----------
// Fixed: previous endpoint returned HTML. Use api.data.gov gateway with DEMO_KEY.
// Docs: https://github.com/usnationalarchives/Catalog-API
async function fromNara(limit) {
const url = `https://catalog.archives.gov/api/v2/records/search?q=film+motion+picture&hasDigitalObject=true&recordType=item&limit=${Math.min(limit, 200)}`;
let data;
try {
data = await fetchJson(url, { 'x-api-key': DATA_GOV_KEY });
} catch (e) {
console.warn(`nara: ${e.message}`);
return [];
}
const hits = data?.body?.hits?.hits ?? data?.opaResponse?.results?.result ?? [];
const out = [];
for (const item of hits) {
const f = item._source?.record ?? item.description?.item ?? {};
const naId = f?.naId ?? item._id ?? item.naId; if (!naId) continue;
const obj = (f?.digitalObjects ?? f?.objects?.object ?? [])[0];
const img = obj?.objectUrl ?? obj?.['@url'];
if (!img) continue;
const yearStr = String(
f?.productionDates?.[0]?.startDate ??
f?.productionDates?.[0]?.logicalDate ??
f?.productionDateArray?.proposableQualifiableDate?.[0]?.logicalDate ??
''
);
const yearMatch = yearStr.match(/(\d{4})/);
out.push({
source: 'nara',
source_id: String(naId),
title: (f.title ?? f.title_eng ?? 'Untitled').slice(0, 240),
caption: (f.scopeAndContentNote ?? f.scopeContent ?? '').slice(0, 600) || null,
image_url: img,
thumb_url: obj.thumbnailUrl ?? obj.thumbnail ?? img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://www.archives.gov/research/catalog/lp.html',
source_url: `https://catalog.archives.gov/id/${naId}`,
});
}
return out;
}
// ---------- 5. Media History Digital Library — Lantern ----------
// 3M+ pages of PD film magazines, posters, lobby cards (1894-1972).
// Solr endpoint, no auth. Each item has IIIF imagery at archive.org under the hood.
async function fromMhdl(limit) {
const url = `https://lantern.mediahist.org/catalog.json?q=film+poster&search_field=all_fields&per_page=${Math.min(limit, 100)}`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`mhdl: ${e.message}`); return []; }
const docs = data?.response?.docs ?? data?.data ?? [];
const out = [];
for (const d of docs) {
const id = d.id ?? d.identifier; if (!id) continue;
// MHDL images come from Internet Archive — derive thumbnail from the IA identifier.
// id format: "{archive_identifier}_{page}" — we want the parent IA item.
const archiveId = String(id).replace(/_\d+$/, '');
const img = `https://archive.org/services/img/${archiveId}`;
const yearStr = String(d.date ?? d.year ?? d.dateRange ?? '');
const yearMatch = yearStr.match(/(\d{4})/);
out.push({
source: 'mhdl',
source_id: String(id),
title: (d.title ?? d.title_primary ?? 'Untitled').toString().slice(0, 240),
caption: (d.description ?? d.summary ?? '').toString().slice(0, 600) || null,
image_url: img,
thumb_url: img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'poster',
rights_url: 'https://mediahist.org/about',
source_url: `https://lantern.mediahist.org/catalog/${id}`,
});
}
return out;
}
// ---------- 6. Smithsonian Open Access ----------
// 4.5M+ CC0 items. DEMO_KEY works for low volume.
// Filter: must have image media (online_media_type:Images) and CC0 (cc0:true).
async function fromSmithsonian(limit) {
const q = encodeURIComponent('(cinema OR film OR "motion picture") AND online_media_type:"Images" AND cc0:true');
const url = `https://api.si.edu/openaccess/api/v1.0/search?api_key=${DATA_GOV_KEY}&q=${q}&rows=${Math.min(limit, 100)}`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`smithsonian: ${e.message}`); return []; }
const out = [];
for (const row of data?.response?.rows ?? []) {
const id = row.id; if (!id) continue;
const media = row.content?.descriptiveNonRepeating?.online_media?.media?.[0];
const img = media?.content ?? media?.resources?.find(r => /Screen|Thumbnail/i.test(r.label))?.url;
if (!img) continue;
const yearStr = String(row.content?.indexedStructured?.date?.[0] ?? '');
const yearMatch = yearStr.match(/(\d{4})/);
const recordLink = row.content?.descriptiveNonRepeating?.record_link;
out.push({
source: 'smithsonian',
source_id: String(id),
title: (row.title ?? 'Untitled').slice(0, 240),
caption: row.content?.freetext?.notes?.[0]?.content?.slice(0, 600) ?? null,
image_url: img,
thumb_url: media?.thumbnail ?? img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://www.si.edu/openaccess',
source_url: recordLink ?? `https://www.si.edu/object/${encodeURIComponent(id)}`,
});
}
return out;
}
// ---------- 6b. Openverse — CC0 + Public Domain Mark aggregator ----------
// 800M+ openly-licensed images federated across Flickr Commons, Wikimedia,
// museums (Met, Cleveland, Smithsonian, Rijksmuseum), gov't archives.
// No key required. License filter: cc0,pdm = strict public-domain only.
// Acts as a fallback path for Flickr Commons content (avoids needing a Flickr key).
async function fromOpenverse(limit) {
const queries = ['film poster', 'cinema', 'motion picture', 'movie still', 'silent film'];
const out = [];
const perQuery = Math.ceil(limit / queries.length);
for (const q of queries) {
const url = `https://api.openverse.org/v1/images/?license=cc0,pdm&q=${encodeURIComponent(q)}&page_size=${Math.min(perQuery, 20)}`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`openverse "${q}": ${e.message}`); continue; }
for (const r of data?.results ?? []) {
if (!r.url || !r.id) continue;
const yearMatch = String(r.title ?? '').match(/(?:^|\D)(1[89]\d{2}|20[0-1]\d)(?:\D|$)/);
out.push({
source: 'openverse',
source_id: r.id,
title: (r.title ?? 'Untitled').slice(0, 240),
caption: null,
image_url: r.url,
thumb_url: r.thumbnail ?? r.url,
width: r.width ?? null,
height: r.height ?? null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: q.includes('poster') ? 'poster' : 'still',
rights_url: r.license_url ?? `https://creativecommons.org/${r.license === 'cc0' ? 'publicdomain/zero/1.0/' : 'publicdomain/mark/1.0/'}`,
source_url: r.foreign_landing_url ?? r.url,
});
}
}
// de-dup by source_id (same image can match multiple queries)
const seen = new Set();
return out.filter(r => seen.has(r.source_id) ? false : (seen.add(r.source_id), true)).slice(0, limit);
}
// ---------- 6c. Metropolitan Museum of Art Open Access ----------
// All Met images marked isPublicDomain=true are CC0. No key required.
// Search returns ~175 film-related objects with images; we filter PD-only.
async function fromMet(limit) {
// 1) Get IDs from search
let ids;
try {
const search = await fetchJson('https://collectionapi.metmuseum.org/public/collection/v1/search?q=film&hasImages=true');
ids = (search?.objectIDs ?? []).slice(0, limit);
} catch (e) { console.warn(`met search: ${e.message}`); return []; }
// 2) Hydrate each (Met has no batch endpoint — sequential w/ small delay)
const out = [];
for (const id of ids) {
try {
const o = await fetchJson(`https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`);
if (!o?.isPublicDomain || !o?.primaryImageSmall) continue;
const yearMatch = String(o.objectDate ?? '').match(/(\d{4})/);
out.push({
source: 'met',
source_id: String(o.objectID),
title: (o.title ?? 'Untitled').slice(0, 240),
caption: (o.creditLine ?? o.medium ?? '').slice(0, 600) || null,
image_url: o.primaryImage ?? o.primaryImageSmall,
thumb_url: o.primaryImageSmall ?? o.primaryImage,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://www.metmuseum.org/about-the-met/policies-and-documents/open-access',
source_url: o.objectURL ?? `https://www.metmuseum.org/art/collection/search/${o.objectID}`,
});
// small politeness delay; Met asks for moderate rate
await new Promise(r => setTimeout(r, 80));
} catch (e) { /* skip silently — Met often 404s on weird IDs */ }
}
return out;
}
// ---------- 6d. Digital Public Library of America ----------
// Federates ~50M items from US libraries/museums/archives. Key required (free,
// emailed via POST to api.dp.la/v2/api_key/<your-email>). Skipped without key.
async function fromDpla(limit) {
if (!process.env.DPLA_API_KEY) {
console.warn('dpla: skipped (no DPLA_API_KEY in env)');
return [];
}
const url = `https://api.dp.la/v2/items?api_key=${process.env.DPLA_API_KEY}&q=film+OR+cinema+OR+%22motion+picture%22&sourceResource.type=image&page_size=${Math.min(limit, 100)}&filter=rights:%22No%20Copyright%22+OR+rights:%22Public%20Domain%22`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`dpla: ${e.message}`); return []; }
const out = [];
for (const d of data?.docs ?? []) {
const id = d.id; if (!id) continue;
const sr = d.sourceResource ?? {};
const img = d.object; // direct hot-link to the asset
if (!img) continue;
const yearMatch = String(sr.date?.displayDate ?? sr.date?.begin ?? '').match(/(\d{4})/);
out.push({
source: 'dpla',
source_id: id,
title: (Array.isArray(sr.title) ? sr.title[0] : sr.title ?? 'Untitled').slice(0, 240),
caption: (Array.isArray(sr.description) ? sr.description[0] : sr.description ?? '').slice(0, 600) || null,
image_url: img,
thumb_url: img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: Array.isArray(sr.rights) ? sr.rights[0] : sr.rights ?? null,
source_url: d.isShownAt ?? `https://dp.la/item/${id.split('--')[0]}`,
});
}
return out;
}
// ---------- 7. Flickr Commons (stubbed — needs FLICKR_API_KEY) ----------
async function fromFlickrCommons(limit) {
if (!process.env.FLICKR_API_KEY) {
console.warn('flickr_commons: skipped (no FLICKR_API_KEY in env)');
return [];
}
const key = process.env.FLICKR_API_KEY;
// Group ID 1158177@N20 = Flickr Commons supergroup
const url = `https://www.flickr.com/services/rest/?method=flickr.photos.search&group_id=1158177@N20&text=film+OR+cinema&extras=url_z,url_o,date_taken,owner_name,license&per_page=${Math.min(limit, 100)}&format=json&nojsoncallback=1&api_key=${key}`;
let data;
try { data = await fetchJson(url); } catch (e) { console.warn(`flickr_commons: ${e.message}`); return []; }
const out = [];
for (const p of data?.photos?.photo ?? []) {
const img = p.url_z ?? p.url_o; if (!img) continue;
const yearMatch = String(p.datetaken ?? '').match(/(\d{4})/);
out.push({
source: 'flickr_commons',
source_id: String(p.id),
title: (p.title ?? 'Untitled').slice(0, 240),
caption: null,
image_url: img,
thumb_url: img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://www.flickr.com/commons/usage/',
source_url: `https://www.flickr.com/photos/${p.owner}/${p.id}`,
});
}
return out;
}
// ---------- 8. NYPL Digital Collections (stubbed — needs NYPL_API_TOKEN) ----------
async function fromNypl(limit) {
if (!process.env.NYPL_API_TOKEN) {
console.warn('nypl: skipped (no NYPL_API_TOKEN in env)');
return [];
}
const token = process.env.NYPL_API_TOKEN;
const url = `https://api.repo.nypl.org/api/v2/items/search.json?q=film+poster&publicDomainOnly=true&per_page=${Math.min(limit, 100)}`;
let data;
try { data = await fetchJson(url, { Authorization: `Token token="${token}"` }); }
catch (e) { console.warn(`nypl: ${e.message}`); return []; }
const out = [];
for (const r of data?.nyplAPI?.response?.result ?? []) {
const uuid = r.uuid; if (!uuid) continue;
const img = r.imageLinks?.imageLink?.find?.(l => /w=760/.test(l['@url']))?.['@url'] ??
r.imageLinks?.imageLink?.[0]?.['@url'];
if (!img) continue;
const yearStr = String(r.dateString ?? r.dateDigitized ?? '');
const yearMatch = yearStr.match(/(\d{4})/);
out.push({
source: 'nypl',
source_id: uuid,
title: (r.title ?? 'Untitled').slice(0, 240),
caption: (r.description ?? '').slice(0, 600) || null,
image_url: img,
thumb_url: img,
width: null, height: null,
year: yearMatch ? Number(yearMatch[1]) : null,
kind: 'still',
rights_url: 'https://digitalcollections.nypl.org/about',
source_url: `https://digitalcollections.nypl.org/items/${uuid}`,
});
}
return out;
}
// ---------- upsert ----------
async function upsert(rows) {
if (!rows.length) return 0;
const text = `
INSERT INTO pd_film_image
(source, source_id, title, caption, image_url, thumb_url, width, height, year, kind, rights_url, source_url, fetched_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, now())
ON CONFLICT (source, source_id) DO UPDATE SET
title = EXCLUDED.title,
caption = EXCLUDED.caption,
image_url = EXCLUDED.image_url,
thumb_url = EXCLUDED.thumb_url,
width = EXCLUDED.width,
height = EXCLUDED.height,
year = EXCLUDED.year,
kind = EXCLUDED.kind,
rights_url = EXCLUDED.rights_url,
source_url = EXCLUDED.source_url,
fetched_at = now()
`;
let n = 0;
for (const r of rows) {
try {
await pool.query(text, [
r.source, r.source_id, r.title, r.caption, r.image_url, r.thumb_url,
r.width, r.height, r.year, r.kind, r.rights_url, r.source_url,
]);
n++;
} catch (e) { console.warn(`upsert ${r.source}/${r.source_id}: ${e.message}`); }
}
return n;
}
(async () => {
console.log(`fetching ~${PER_SOURCE} per source (${LIMIT} total target)`);
const settled = await Promise.allSettled([
fromWikimedia(PER_SOURCE),
fromArchive(PER_SOURCE),
fromLoc(PER_SOURCE),
fromNara(PER_SOURCE),
fromMhdl(PER_SOURCE),
fromSmithsonian(PER_SOURCE),
fromOpenverse(PER_SOURCE),
fromMet(PER_SOURCE),
fromDpla(PER_SOURCE),
fromFlickrCommons(PER_SOURCE),
fromNypl(PER_SOURCE),
]);
const labels = ['wikimedia','archive_org','loc','nara','mhdl','smithsonian','openverse','met','dpla','flickr_commons','nypl'];
const all = [];
const counts = {};
settled.forEach((s, i) => {
if (s.status === 'fulfilled') {
counts[labels[i]] = s.value.length;
all.push(...s.value);
} else {
counts[labels[i]] = `error: ${s.reason?.message ?? s.reason}`;
}
});
console.log('fetched:', counts);
const wrote = await upsert(all);
console.log(`upserted ${wrote}/${all.length}`);
await pool.end();
})();