`scripts/ingest-loc-sanborn.ts`

156 lines
#!/usr/bin/env -S npx tsx
/**
 * Library of Congress Sanborn fire-insurance maps ingest.
 *
 * Queries the LoC Sanborn collection for sheets covering Beverly Hills,
 * stores them as media_asset rows linked to all BH listings (the same
 * sheet covers many parcels — ingest deliberately fans out so each
 * address page can show its own block in context).
 *
 * Public-domain (US gov + expired copyright). Tier B.
 *
 * Usage:
 *   npx tsx scripts/ingest-loc-sanborn.ts                 # all BH listings
 *   npx tsx scripts/ingest-loc-sanborn.ts --city Malibu   # other city
 *   npx tsx scripts/ingest-loc-sanborn.ts --dry
 */
import { Pool } from 'pg';

type LocItem = {
  id: string;
  title?: string;
  url?: string;
  date?: string;
  image_url?: string[];
  resources?: Array<{ image?: string; url?: string }>;
};

async function searchSanborn(city: string, state: string, page: number, perPage = 25): Promise<LocItem[]> {
  // Sanborn collection search; faceted by location.
  const params = new URLSearchParams({
    q: `${city} ${state}`,
    fa: `location:${state.toLowerCase()}`,
    fo: 'json',
    c: String(perPage),
    sp: String(page),
  });
  const url = `https://www.loc.gov/collections/sanborn-maps/?${params}`;
  for (let attempt = 0; attempt < 4; attempt++) {
    const res = await fetch(url, {
      headers: {
        'User-Agent': 'pastdoor/0.1 (steve@pastdoor.com) historical research',
        Accept: 'application/json',
      },
    });
    if (res.status === 429) {
      // Back off exponentially: 3s, 7s, 15s.
      const wait = (attempt + 1) * (attempt + 1) * 1000 + 2000;
      console.warn(`  [LoC 429] backing off ${wait}ms (attempt ${attempt + 1})`);
      await new Promise(r => setTimeout(r, wait));
      continue;
    }
    if (!res.ok) throw new Error(`LoC ${res.status}`);
    const j = (await res.json()) as { results?: LocItem[] };
    return j.results ?? [];
  }
  throw new Error('LoC 429 — exhausted retries');
}

function yearFromTitle(title: string): number | null {
  const m = title.match(/\b(18|19|20)\d{2}\b/);
  return m ? Number(m[0]) : null;
}

function pickThumb(item: LocItem): string | null {
  if (item.image_url && item.image_url.length > 0) return item.image_url[0]!;
  if (item.resources && item.resources.length > 0) {
    const r = item.resources[0]!;
    return r.image ?? r.url ?? null;
  }
  return null;
}

const pool = new Pool({
  host: process.env.PGHOST ?? '/tmp',
  database: process.env.PGDATABASE ?? 'stayclaim',
  user: process.env.PGUSER ?? process.env.USER,
  max: 4,
});

async function main() {
  const args = process.argv.slice(2);
  const cityArg = args.find((a, i) => args[i - 1] === '--city') ?? 'Beverly Hills';
  const stateArg = args.find((a, i) => args[i - 1] === '--state') ?? 'California';
  const dry = args.includes('--dry');

  const items: LocItem[] = [];
  // pull up to 2 pages = 50 results max
  for (let p = 1; p <= 2; p++) {
    try {
      const r = await searchSanborn(cityArg, stateArg, p);
      if (r.length === 0) break;
      items.push(...r);
    } catch (e) {
      console.warn(`page ${p} error:`, (e as Error).message);
      break;
    }
    await new Promise(r => setTimeout(r, 800));
  }
  console.log(`LoC returned ${items.length} candidate items for ${cityArg}, ${stateArg}`);

  // Filter: must look like a Sanborn sheet (title or url contains the city).
  const cityRe = new RegExp(cityArg, 'i');
  const filtered = items.filter(it => cityRe.test(it.title ?? '') || cityRe.test(it.id ?? ''));
  console.log(`Filtered to ${filtered.length} likely-${cityArg} sheets`);

  // Pull listings to attach sheets to.
  const { rows: listings } = await pool.query<{ id: string; slug: string }>(
    'SELECT id, slug FROM listing WHERE city = $1 AND is_public = true',
    [cityArg]
  );
  if (listings.length === 0) {
    console.log(`No public listings for ${cityArg}; nothing to attach.`);
    await pool.end();
    return;
  }
  console.log(`Attaching to ${listings.length} listings.`);

  let inserted = 0;
  let skipped = 0;
  for (const item of filtered) {
    const year = yearFromTitle(item.title ?? '');
    const url = pickThumb(item);
    if (!url) {
      skipped++;
      continue;
    }
    const caption = item.title ? item.title.slice(0, 200) : 'Sanborn sheet';
    for (const l of listings) {
      // Idempotency: skip if we already have this LoC item for this listing.
      const existing = await pool.query(
        `SELECT 1 FROM media_asset WHERE listing_id = $1 AND source_url = $2 LIMIT 1`,
        [l.id, item.url ?? item.id]
      );
      if (existing.rowCount! > 0) {
        skipped++;
        continue;
      }
      if (dry) {
        console.log(`[${l.slug}] DRY would insert ${year ?? '?'} :: ${caption.slice(0, 60)}`);
        continue;
      }
      await pool.query(
        `INSERT INTO media_asset
           (listing_id, kind, year, caption, storage_url, source_tier, source_label, source_url, rights, public_visible)
         VALUES ($1, 'historic_map', $2, $3, $4, 'B', $5, $6, 'Public domain (LoC)', true)`,
        [l.id, year, caption, url, 'Library of Congress · Sanborn Maps Collection', item.url ?? item.id]
      );
      inserted++;
    }
  }
  console.log(JSON.stringify({ found: items.length, sheets: filtered.length, listings: listings.length, inserted, skipped, dry }, null, 2));
  await pool.end();
}

main().catch(e => { console.error(e); process.exit(1); });