`src/enrich/geocode_nominatim.ts`

182 lines
/**
 * Nominatim geocoder — fills lat/lng for firms that have an address but no coordinates.
 *
 * Free, no key. OpenStreetMap's nominatim.openstreetmap.org per their usage policy:
 *   - 1 request/second max
 *   - Meaningful User-Agent required
 *   - Cache results (we already do — geocoded_at gates re-runs)
 *
 * https://operations.osmfoundation.org/policies/nominatim/
 *
 * CLI:
 *   npx tsx src/enrich/geocode_nominatim.ts                # full pass
 *   npx tsx src/enrich/geocode_nominatim.ts --smoke        # first 5
 *   npx tsx src/enrich/geocode_nominatim.ts --limit 100    # cap at N
 */
import 'dotenv/config';
import { pool, query } from '../db/pool.ts';

const SOURCE_NAME = 'OSM Nominatim Geocoder';
const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search';
const RATE_DELAY_MS = Number(process.env.NOMINATIM_DELAY_MS || 1100);
const USER_AGENT = process.env.USER_AGENT
  || 'LawyerDirectoryBuilder/0.1 (research; contact: steveabramsdesigns@gmail.com)';

type Firm = {
  id: number;
  name: string;
  address: string;
  city: string | null;
  state: string | null;
  zip: string | null;
};

type NomHit = {
  lat: string;
  lon: string;
  display_name: string;
  importance?: number;
};

const argv = process.argv.slice(2);
const SMOKE = argv.includes('--smoke');
const LIMIT_IDX = argv.indexOf('--limit');
const LIMIT = LIMIT_IDX >= 0 ? Number(argv[LIMIT_IDX + 1]) : (SMOKE ? 5 : 0);

async function ensureSource(): Promise<number> {
  await query(`
    INSERT INTO sources (source_name, source_type, base_url, terms_notes, allowed_method, rate_limit_rps)
    VALUES ($1, 'api', $2,
      'OSM Nominatim public instance. ODbL. Polite-use ≤1 rps with meaningful UA. Cache results.',
      'api', 1.0)
    ON CONFLICT (source_name) DO NOTHING
  `, [SOURCE_NAME, NOMINATIM_URL]);
  const r = await query<{ id: number }>(`SELECT id FROM sources WHERE source_name = $1`, [SOURCE_NAME]);
  return r.rows[0].id;
}

async function startJob(sourceId: number, label: string): Promise<number> {
  const r = await query<{ id: number }>(`
    INSERT INTO scrape_jobs (source_id, job_label, status, started_at)
    VALUES ($1, $2, 'running', NOW()) RETURNING id
  `, [sourceId, label]);
  return r.rows[0].id;
}

async function finishJob(jobId: number, fields: Record<string, unknown>) {
  const sets: string[] = []; const params: unknown[] = []; let i = 1;
  for (const [k, v] of Object.entries(fields)) { sets.push(`${k} = $${i++}`); params.push(v); }
  sets.push(`finished_at = NOW()`); params.push(jobId);
  await query(`UPDATE scrape_jobs SET ${sets.join(', ')} WHERE id = $${i}`, params);
}

async function loadFirms(): Promise<Firm[]> {
  const lim = LIMIT > 0 ? `LIMIT ${LIMIT}` : '';
  const r = await query<Firm>(`
    SELECT id, name, address, city, state, zip
    FROM organizations
    WHERE type = 'law_firm'
      AND address IS NOT NULL
      AND lat IS NULL
    ORDER BY id
    ${lim}
  `);
  return r.rows;
}

function buildQuery(firm: Firm): string {
  // Nominatim does best with a single comma-separated string. Don't double-include
  // city/state if they're already in the address blob.
  const haystack = firm.address.toLowerCase();
  const parts = [firm.address];
  if (firm.city && !haystack.includes(firm.city.toLowerCase())) parts.push(firm.city);
  if (firm.state && !haystack.includes(`, ${firm.state.toLowerCase()}`)) parts.push(firm.state);
  if (firm.zip && !haystack.includes(firm.zip)) parts.push(firm.zip);
  return parts.join(', ');
}

async function geocode(firm: Firm): Promise<NomHit | null> {
  const { fetch } = await import('undici');
  const params = new URLSearchParams({
    q: buildQuery(firm),
    format: 'json',
    limit: '1',
    countrycodes: 'us',
    addressdetails: '0',
  });
  const r = await fetch(`${NOMINATIM_URL}?${params.toString()}`, {
    headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' },
    signal: AbortSignal.timeout(20000),
  });
  if (!r.ok) {
    if (r.status === 429) throw new Error('NOMINATIM_429');
    return null;
  }
  const j = await r.json() as NomHit[];
  return j && j.length > 0 ? j[0] : null;
}

async function applyHit(firmId: number, hit: NomHit): Promise<void> {
  const lat = parseFloat(hit.lat);
  const lng = parseFloat(hit.lon);
  if (!isFinite(lat) || !isFinite(lng)) return;
  // Guardrail: discard hits outside the LA County bounding box.
  // ~33.7° to 34.85° N, -119.0° to -117.6° W (loose)
  if (lat < 33.5 || lat > 35.0 || lng < -119.5 || lng > -117.5) return;
  await query(
    `UPDATE organizations SET lat = $2, lng = $3, geocoded_at = NOW(), updated_at = NOW() WHERE id = $1 AND lat IS NULL`,
    [firmId, lat, lng]);
}

async function main() {
  const sourceId = await ensureSource();
  const label = SMOKE ? 'nominatim:smoke' : (LIMIT > 0 ? `nominatim:limit-${LIMIT}` : 'nominatim:la-firms');
  const jobId = await startJob(sourceId, label);

  const firms = await loadFirms();
  console.log(`[nom] geocoding ${firms.length} firms…`);

  let seen = 0, hit = 0, miss = 0, errs = 0, outside = 0;
  try {
    for (const firm of firms) {
      seen++;
      try {
        const h = await geocode(firm);
        if (!h) { miss++; }
        else {
          const lat = parseFloat(h.lat); const lng = parseFloat(h.lon);
          if (lat < 33.5 || lat > 35.0 || lng < -119.5 || lng > -117.5) outside++;
          else { await applyHit(firm.id, h); hit++; }
        }
      } catch (e) {
        errs++;
        const msg = (e as Error).message;
        if (msg === 'NOMINATIM_429') {
          console.error('[nom] 429 — backing off 60s');
          await new Promise(r => setTimeout(r, 60000));
        } else {
          console.error(`[nom] firm=${firm.id} err: ${msg}`);
        }
      }
      if (seen % 50 === 0) console.log(`[nom] ${seen}/${firms.length}  hit=${hit} miss=${miss} outside=${outside} errs=${errs}`);
      await new Promise(r => setTimeout(r, RATE_DELAY_MS));
    }
  } finally {
    await finishJob(jobId, {
      status: errs > firms.length / 2 ? 'failed' : 'completed',
      records_found: seen,
      records_updated: hit,
      records_skipped: miss + outside,
      error_message: errs > 0 ? `${errs} per-firm errors, ${outside} outside-LA discarded` : null,
    });
  }
  console.log(`[nom] done. seen=${seen} hit=${hit} miss=${miss} outside=${outside} errs=${errs}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[nom] fatal:', err);
  try { await pool.end(); } catch {}
  process.exit(1);
});