`scripts/cr-localize-images.js`

68 lines
'use strict';
// costa-rica: localize region + place images. Wikimedia hotlinks 429 on
// burst-loads (grids fire ~50 at once), so download each once (~900px thumb)
// into public/img/{regions,places}/ and repoint the DB, preserving the
// original in image_remote_url. Idempotent: skips rows already local.
// Run ON Kamatera from /root/Projects/costa-rica with DATABASE_URL exported.
const fs = require('fs');
const path = require('path');
const { Pool } = require('pg');

const pool = new Pool({ connectionString: process.env.DATABASE_URL });
const UA = 'CR-Directory/1.0 (https://costarica.agentabrams.com; info@agentabrams.com) image-localizer';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function thumbUrl(remote) {
  const clean = remote.split('?')[0];
  if (clean.includes('upload.wikimedia.org')) {
    // /commons/thumb/<h>/<hh>/<File>/<Npx-File> → real filename is 2nd-to-last
    const segs = clean.split('/');
    const file = decodeURIComponent(clean.includes('/thumb/') ? segs[segs.length - 2] : segs[segs.length - 1]);
    return `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(file)}?width=900`;
  }
  return remote; // non-wikimedia: fetch as-is
}

async function fetchWithRetry(url) {
  for (let a = 1; a <= 4; a++) {
    const res = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow' });
    if (res.status === 429) { console.log(`  429 — backoff 20s (${a})`); await sleep(20000); continue; }
    return res;
  }
  return null;
}

async function localize(table, dir) {
  await pool.query(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS image_remote_url TEXT`);
  const { rows } = await pool.query(
    `SELECT id, slug, image_url FROM ${table} WHERE image_url LIKE 'http%' ORDER BY slug`);
  console.log(`${table}: ${rows.length} remote images`);
  const out = path.join(__dirname, 'public', 'img', dir);
  fs.mkdirSync(out, { recursive: true });
  let ok = 0, fail = 0;
  for (const r of rows) {
    try {
      const res = await fetchWithRetry(thumbUrl(r.image_url));
      if (!res || !res.ok) { console.log(`FAIL ${r.slug}: HTTP ${res && res.status}`); fail++; continue; }
      const ct = res.headers.get('content-type') || '';
      const ext = ct.includes('png') ? 'png' : 'jpg';
      const buf = Buffer.from(await res.arrayBuffer());
      if (buf.length < 4096) { console.log(`FAIL ${r.slug}: tiny (${buf.length}B)`); fail++; continue; }
      fs.writeFileSync(path.join(out, `${r.slug}.${ext}`), buf);
      await pool.query(
        `UPDATE ${table} SET image_remote_url = image_url, image_url = $1 WHERE id = $2`,
        [`/img/${dir}/${r.slug}.${ext}`, r.id]);
      ok++;
      console.log(`ok ${r.slug} (${Math.round(buf.length / 1024)}KB)`);
    } catch (e) { console.log(`FAIL ${r.slug}: ${e.message}`); fail++; }
    await sleep(1100);
  }
  console.log(`${table} DONE ok=${ok} fail=${fail}`);
}

(async () => {
  await localize('regions', 'regions');
  await localize('places', 'places');
  await pool.end();
})();