`scripts/fetch-pd-images.js`

216 lines
#!/usr/bin/env node
// Fetch public-domain interior/wallcovering imagery from Wikimedia Commons.
// Strict rule: PD or CC0 only. Anything else gets dropped. Steve's standing
// "no stock images" rule applies — Unsplash/Pexels/Getty NEVER appear here.

import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(__dirname, '..');
const OUT_DIR = path.join(ROOT, 'public', 'img', 'segments');
const MANIFEST = path.join(OUT_DIR, 'manifest.json');

const UA = 'NationalPaperHangers/0.1 (https://nationalpaperhangers.com; ops@nationalpaperhangers.com)';

const SEGMENTS = {
  luxury_residential: [
    'drawing room interior wallpaper',
    'Victorian parlor interior',
    'Edwardian drawing room',
    'historic interior wallpaper',
  ],
  hospitality: [
    'art deco hotel lobby',
    'historic hotel interior',
    'luxury hotel lobby interior',
  ],
  museum: [
    'museum interior decoration',
    'historic dining room interior',
    'period room museum',
  ],
  hand_painted: [
    'hand painted wallpaper',
    'chinoiserie wallpaper',
    'panoramic wallpaper Zuber',
  ],
  grasscloth: [
    'grasscloth wallcovering',
    'natural fiber wall covering',
    'bamboo wall covering interior',
  ],
  silk: [
    'silk damask wallpaper',
    'damask wallpaper interior',
    'historic silk wall hanging',
  ],
  mural: [
    'mural wallpaper interior',
    'scenic wallpaper panorama',
    'fresco interior decoration',
  ],
  generic: [
    'wallpaper sample',
    'wallpaper pattern',
    'wallpaper sidewall',
    'wall covering pattern',
    'historic interior decoration',
    'antique interior decoration',
    'wallpaper roll Cooper Hewitt',
  ],
};

const PER_QUERY_LIMIT = 12;
const TARGET_PER_SEGMENT = 14;
const THUMB_W = 1200;

const PD_HINTS = ['public domain', 'cc0', 'pd-old', 'pd-us', 'pd-art', 'cc-zero'];

function isPublicDomain(meta) {
  if (!meta) return false;
  const license = (meta.LicenseShortName?.value || meta.UsageTerms?.value || '').toLowerCase();
  const tmpl = (meta.LicenseUrl?.value || '').toLowerCase();
  if (PD_HINTS.some((h) => license.includes(h))) return true;
  if (tmpl.includes('publicdomain') || tmpl.includes('zero/1.0')) return true;
  return false;
}

async function searchCommons(query) {
  const params = new URLSearchParams({
    action: 'query',
    format: 'json',
    prop: 'imageinfo',
    generator: 'search',
    gsrsearch: `${query} filetype:bitmap`,
    gsrnamespace: '6',
    gsrlimit: String(PER_QUERY_LIMIT),
    iiprop: 'url|extmetadata|size|mime',
    iiurlwidth: String(THUMB_W),
  });
  const url = `https://commons.wikimedia.org/w/api.php?${params}`;
  const res = await fetch(url, { headers: { 'User-Agent': UA } });
  if (!res.ok) throw new Error(`commons search ${res.status} for ${query}`);
  const j = await res.json();
  return Object.values(j?.query?.pages || {});
}

async function downloadBinary(url, dest) {
  // Be polite to Wikimedia's CDN — bulk hammering triggers 429.
  await new Promise(r => setTimeout(r, 350));
  const res = await fetch(url, { headers: { 'User-Agent': UA } });
  if (res.status === 429) {
    // Back off and retry once.
    await new Promise(r => setTimeout(r, 5000));
    const res2 = await fetch(url, { headers: { 'User-Agent': UA } });
    if (!res2.ok) throw new Error(`download ${res2.status} ${url}`);
    const buf2 = Buffer.from(await res2.arrayBuffer());
    await fs.writeFile(dest, buf2);
    return buf2.length;
  }
  if (!res.ok) throw new Error(`download ${res.status} ${url}`);
  const buf = Buffer.from(await res.arrayBuffer());
  await fs.writeFile(dest, buf);
  return buf.length;
}

function safeName(title, idx) {
  const base = title
    .replace(/^File:/i, '')
    .replace(/\.[a-z0-9]+$/i, '')
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-+|-+$/g, '')
    .slice(0, 60);
  return `${String(idx).padStart(2, '0')}-${base || 'img'}.jpg`;
}

async function ensureDir(p) {
  await fs.mkdir(p, { recursive: true });
}

async function loadManifest() {
  try {
    const txt = await fs.readFile(MANIFEST, 'utf8');
    return JSON.parse(txt);
  } catch {
    return { generated_at: null, items: [] };
  }
}

async function main() {
  await ensureDir(OUT_DIR);
  const manifest = await loadManifest();
  const seen = new Set(manifest.items.map((it) => it.source_title));

  for (const [segment, queries] of Object.entries(SEGMENTS)) {
    const segDir = path.join(OUT_DIR, segment);
    await ensureDir(segDir);
    let kept = (await fs.readdir(segDir)).filter((f) => f.endsWith('.jpg')).length;
    if (kept >= TARGET_PER_SEGMENT) {
      console.log(`[${segment}] already has ${kept}, skipping`);
      continue;
    }

    for (const q of queries) {
      if (kept >= TARGET_PER_SEGMENT) break;
      let pages;
      try {
        pages = await searchCommons(q);
      } catch (e) {
        console.warn(`[${segment}] search failed for "${q}": ${e.message}`);
        continue;
      }
      for (const page of pages) {
        if (kept >= TARGET_PER_SEGMENT) break;
        const ii = page.imageinfo?.[0];
        if (!ii) continue;
        if (!ii.mime || !ii.mime.startsWith('image/')) continue;
        if (seen.has(page.title)) continue;
        if (!isPublicDomain(ii.extmetadata)) continue;
        const thumbUrl = ii.thumburl || ii.url;
        const idx = manifest.items.length + 1;
        const filename = safeName(page.title, idx);
        const dest = path.join(segDir, filename);
        try {
          const bytes = await downloadBinary(thumbUrl, dest);
          if (bytes < 8000) {
            await fs.unlink(dest);
            continue;
          }
          const item = {
            file: `/img/segments/${segment}/${filename}`,
            segment,
            query: q,
            source_title: page.title,
            source_url: ii.descriptionurl || `https://commons.wikimedia.org/wiki/${encodeURIComponent(page.title)}`,
            license: ii.extmetadata?.LicenseShortName?.value || 'Public domain',
            creator: (ii.extmetadata?.Artist?.value || '').replace(/<[^>]+>/g, '').trim() || null,
            credit: (ii.extmetadata?.Credit?.value || '').replace(/<[^>]+>/g, '').trim() || null,
            width: ii.thumbwidth || ii.width,
            height: ii.thumbheight || ii.height,
            bytes,
          };
          manifest.items.push(item);
          seen.add(page.title);
          kept += 1;
          console.log(`[${segment}] +${filename}  (${item.license})`);
        } catch (e) {
          console.warn(`[${segment}] download failed ${page.title}: ${e.message}`);
        }
      }
    }
    console.log(`[${segment}] final count: ${kept}`);
  }

  manifest.generated_at = new Date().toISOString();
  await fs.writeFile(MANIFEST, JSON.stringify(manifest, null, 2));
  console.log(`\nmanifest written: ${manifest.items.length} items at ${MANIFEST}`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});