`db/ingest-beverlyhills-photos.mjs`

91 lines
// Pulls atmospheric photos of BEVERLY HILLS (the city — Rodeo Drive, the sign,
// palm streets), NOT the show. License-verified free-for-reuse only (PD/CC0/CC-BY/
// CC-BY-SA). Stored in assets tagged 'beverly-hills-location' with attribution.
// Run: node db/ingest-beverlyhills-photos.mjs
import pg from "pg";

const DATABASE_URL =
  process.env.DATABASE_URL ?? "postgresql://localhost/nineoh_guide?host=/tmp";
const pool = new pg.Pool({ connectionString: DATABASE_URL });
const UA = "nineoh-guide/0.1 (unofficial fan project; contact: set-at-launch)";
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const stripHtml = (s) => (s || "").replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();

const FREE = /^(cc0|cc-by-\d|cc-by-sa-\d|pd|public domain)/i;
const NONFREE = /(nc|nd|non-free|fair use)/i;

async function fetchJson(url, tries = 4) {
  for (let i = 0; i < tries; i++) {
    try {
      const r = await fetch(url, { headers: { "User-Agent": UA } });
      return JSON.parse(await r.text());
    } catch {
      await sleep(800 * (i + 1));
    }
  }
  throw new Error("rate-limited");
}

const TARGET = 8;
const SEARCHES = [
  "Rodeo Drive Beverly Hills",
  "Beverly Hills sign California",
  "Beverly Hills City Hall",
  "Beverly Hills palm trees street",
  "Beverly Hills California",
];

let stored = 0;
for (const q of SEARCHES) {
  if (stored >= TARGET) break;
  const u =
    `https://commons.wikimedia.org/w/api.php?action=query&format=json&generator=search` +
    `&gsrsearch=${encodeURIComponent(q)}&gsrnamespace=6&gsrlimit=8` +
    `&prop=imageinfo&iiprop=extmetadata|url&iiurlwidth=900`;
  let j;
  try {
    j = await fetchJson(u);
  } catch {
    continue;
  }
  const pages = Object.values(j.query?.pages ?? {});
  for (const page of pages) {
    if (stored >= TARGET) break;
    const info = page.imageinfo?.[0];
    if (!info) continue;
    const m = info.extmetadata ?? {};
    const licName = m.LicenseShortName?.value || "";
    const lic = (m.License?.value || "").toLowerCase();
    const free = FREE.test(lic) || /public domain|cc0|cc by(?!-nc|-nd)/i.test(licName);
    if (!free || NONFREE.test(lic) || /nc|nd|fair use/i.test(licName)) continue;
    // Skip maps/logos/diagrams — want photos.
    const title = (page.title || "").replace(/^File:/, "");
    if (/\.svg$|map|logo|seal|diagram|\.pdf$/i.test(title)) continue;

    const url = info.thumburl || info.url;
    const dup = await pool.query(`select 1 from assets where file_url=$1`, [url]);
    if (dup.rowCount) continue;

    const licType = /cc0|public domain|^pd/i.test(lic) ? "public-domain" : /sa/i.test(lic) ? "cc-by-sa" : "cc-by";
    await pool.query(
      `insert into assets (type, file_url, thumbnail_url, license_type, license_url,
                           attribution_text, usage_scope, source_url, rights_notes, caption, approved_for_marketing)
       values ('image',$1,$1,$2,$3,$4,'editorial',$5,'beverly-hills-location',$6,false)`,
      [
        url,
        licType,
        m.LicenseUrl?.value || null,
        `${stripHtml(m.Artist?.value || m.Credit?.value || "Unknown")} / ${licName} (via Wikimedia Commons)`,
        `https://commons.wikimedia.org/wiki/${encodeURIComponent(page.title)}`,
        title.replace(/\.[a-z]+$/i, "").replace(/_/g, " ").slice(0, 80),
      ]
    );
    stored++;
    console.log(`OK  ${licType.padEnd(13)} ${title.slice(0, 60)}`);
  }
  await sleep(700);
}

console.log(`\nstored ${stored} Beverly Hills location photos (free-licensed).`);
await pool.end();