`agents/enrich-agent/geocode.js`

117 lines
#!/usr/bin/env node
/**
 * Free geocoder — Stage 6.
 *
 *   1. US Census Geocoding API   — free, no key, no auth, no rate limit
 *      https://geocoding.geo.census.gov/geocoder/locations/onelineaddress
 *
 *   2. OpenStreetMap Nominatim   — free, polite-use 1 req/s, identifying UA
 *      https://nominatim.openstreetmap.org/search
 *
 * No Google APIs (compliance rule #3 — zero-dollar build).
 *
 * Run:
 *   node agents/enrich-agent/geocode.js          # all rows where lat IS NULL
 *   LIMIT=100 node agents/enrich-agent/geocode.js
 */
const { fetch } = require('undici');
const { pool, query } = require('../shared/db');
const { setHostRateLimit } = require('../shared/compliance');

const CENSUS_URL    = 'https://geocoding.geo.census.gov/geocoder/locations/onelineaddress';
const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search';
const USER_AGENT    = process.env.USER_AGENT
  || 'ProfessionalDirectoryBot/0.1 (research; contact: steveabramsdesigns@gmail.com)';

setHostRateLimit('nominatim.openstreetmap.org', 1.0);   // 1 rps per their ToS
setHostRateLimit('geocoding.geo.census.gov', 5.0);      // generous; no published limit

async function geocodeCensus(addr, city, state, zip) {
  const oneLine = [addr, city, state, zip].filter(Boolean).join(', ');
  const url = `${CENSUS_URL}?address=${encodeURIComponent(oneLine)}&benchmark=Public_AR_Current&format=json`;
  const res = await fetch(url, {
    headers: { 'User-Agent': USER_AGENT },
    signal: AbortSignal.timeout(15000),
  });
  if (!res.ok) return null;
  const j = await res.json();
  const m = j?.result?.addressMatches?.[0];
  if (!m) return null;
  return {
    lat: Number(m.coordinates?.y),
    lng: Number(m.coordinates?.x),
    source: 'us_census',
    matched: m.matchedAddress,
  };
}

async function geocodeNominatim(addr, city, state, zip) {
  const oneLine = [addr, city, state, zip].filter(Boolean).join(', ');
  const url = `${NOMINATIM_URL}?q=${encodeURIComponent(oneLine)}&format=json&limit=1&countrycodes=us`;
  const res = await fetch(url, {
    headers: { 'User-Agent': USER_AGENT, 'Accept-Language': 'en-US' },
    signal: AbortSignal.timeout(15000),
  });
  if (!res.ok) return null;
  const arr = await res.json();
  if (!Array.isArray(arr) || arr.length === 0) return null;
  return {
    lat: Number(arr[0].lat),
    lng: Number(arr[0].lon),
    source: 'nominatim',
    matched: arr[0].display_name,
  };
}

async function main() {
  const limit = process.env.LIMIT ? Number(process.env.LIMIT) : null;
  const sql = `
    SELECT id, address, city, state, zip
      FROM organizations
     WHERE lat IS NULL
       AND address IS NOT NULL
       AND zip IS NOT NULL
     ORDER BY id
     ${limit ? `LIMIT ${limit}` : ''}
  `;
  const rows = (await query(sql, [])).rows;
  console.log(`[geo] candidates=${rows.length}`);

  let ok = 0, fail = 0;
  for (const r of rows) {
    let result;
    try {
      result = await geocodeCensus(r.address, r.city, r.state, r.zip);
      if (!result) {
        // Census misses → polite Nominatim fallback (1 rps).
        await new Promise(res => setTimeout(res, 1100));
        result = await geocodeNominatim(r.address, r.city, r.state, r.zip);
      }
    } catch (e) {
      console.error(`[geo] error org=${r.id}: ${e.message}`);
    }

    if (result && Number.isFinite(result.lat) && Number.isFinite(result.lng)) {
      await query(`
        UPDATE organizations
           SET lat = $2, lng = $3, geocoded_at = NOW(), updated_at = NOW()
         WHERE id = $1
      `, [r.id, result.lat, result.lng]);
      ok++;
    } else {
      fail++;
    }

    if ((ok + fail) % 50 === 0) console.log(`[geo] ok=${ok} fail=${fail}`);
  }

  console.log(`[geo] done. ok=${ok} fail=${fail}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[geo] fatal:', err);
  try { await pool.end(); } catch (_) {}
  process.exit(1);
});