`scripts/geocode-ranked.js`

96 lines
#!/usr/bin/env node
/* geocode-ranked.js — enrich data/ranked.json with real zip + lat/lng via the FREE US Census
 * batch geocoder (no API key, 10k addresses/batch, $0). ADDITIVE + IDEMPOTENT: only fills rows
 * missing `zip`; never changes address/city/price or drops rows. Safe to re-run and safe to run
 * against prod's own ranked.json (it enriches in place, preserving the existing snapshot).
 *
 * WHY: the grid snapshot has address+city but no zip, and LA neighborhoods (Encino, Topanga,
 * Bel-Air…) are folded into city="Los Angeles". Per-city scoping for those needs the zip — the
 * only field that distinguishes an Encino address (91316/91436) from the rest of Los Angeles.
 *
 * Usage:  node scripts/geocode-ranked.js [--limit N] [--dry-run]
 *   --dry-run : geocode + report match rate, DON'T write ranked.json
 *   --limit N : only process the first N un-geocoded rows (for a quick test)
 */
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');
const RANKED = path.join(ROOT, 'data', 'ranked.json');
const BENCHMARK = 'Public_AR_Current';
const CHUNK = 5000;                                   // Census caps at 10k/batch; 5k is comfortable
const args = process.argv.slice(2);
const DRY = args.includes('--dry-run');
const LIMIT = (() => { const i = args.indexOf('--limit'); return i >= 0 ? parseInt(args[i + 1], 10) : Infinity; })();

// Best-effort state inference for the high-volume non-CA cities (improves match rate + disambiguates).
// Everything else defaults to CA — the fleet is all-CA, so CA rows (the only scoping-relevant ones)
// get a correct state; a wrong default on a rare out-of-state city only costs that one row a match.
const STATE = {
  'las vegas': 'NV', 'henderson': 'NV', 'reno': 'NV', 'houston': 'TX', 'dallas': 'TX', 'austin': 'TX',
  'san antonio': 'TX', 'fort worth': 'TX', 'el paso': 'TX', 'miami': 'FL', 'orlando': 'FL', 'tampa': 'FL',
  'jacksonville': 'FL', 'new york': 'NY', 'brooklyn': 'NY', 'bronx': 'NY', 'queens': 'NY', 'phoenix': 'AZ',
  'tucson': 'AZ', 'scottsdale': 'AZ', 'mesa': 'AZ', 'atlanta': 'GA', 'chicago': 'IL', 'seattle': 'WA',
  'portland': 'OR', 'denver': 'CO', 'nashville': 'TN', 'memphis': 'TN', 'charlotte': 'NC', 'columbus': 'OH',
  'philadelphia': 'PA', 'detroit': 'MI', 'boston': 'MA', 'washington': 'DC', 'new orleans': 'LA',
  'oklahoma city': 'OK', 'kansas city': 'MO', 'st louis': 'MO', 'salt lake city': 'UT', 'albuquerque': 'NM',
  'indianapolis': 'IN', 'milwaukee': 'WI', 'louisville': 'KY'
};
const stateFor = c => STATE[String(c || '').toLowerCase().trim()] || 'CA';
const csvSafe = s => String(s == null ? '' : s).replace(/[,\r\n"]/g, ' ').trim();

async function geocodeBatch(rows) {
  // rows: [{idx, address, city}] -> CSV "id,street,city,state,zip"
  const csv = rows.map(r => [r.idx, csvSafe(r.address), csvSafe(r.city), stateFor(r.city), ''].join(',')).join('\n') + '\n';
  const form = new FormData();
  form.append('benchmark', BENCHMARK);
  form.append('addressFile', new Blob([csv], { type: 'text/csv' }), 'addrs.csv');
  const res = await fetch('https://geocoding.geo.census.gov/geocoder/locations/addressbatch', { method: 'POST', body: form });
  if (!res.ok) throw new Error('census HTTP ' + res.status);
  const text = await res.text();
  // Output CSV: "id","input","Match|No_Match|Tie","Exact|Non_Exact","matched addr","lon,lat","tigerid","side"
  const out = {};
  for (const line of text.split('\n')) {
    if (!line.trim()) continue;
    const f = line.match(/"(?:[^"]*)"/g); if (!f) continue;
    const cell = i => (f[i] || '').replace(/^"|"$/g, '');
    const id = cell(0), status = cell(2);
    if (status !== 'Match') continue;
    const matched = cell(4);                          // "304 E 95TH ST, LOS ANGELES, CA, 90003"
    const lonlat = cell(5);                           // "-118.26...,33.95..."
    const zip = (matched.match(/,\s*(\d{5})\s*$/) || [])[1] || '';
    const [lon, lat] = lonlat.split(',');
    if (zip) out[id] = { zip, lat: lat ? +lat : null, lng: lon ? +lon : null };
  }
  return out;
}

(async () => {
  const doc = JSON.parse(fs.readFileSync(RANKED, 'utf8'));
  const arr = doc.ranked || [];
  const todo = [];
  arr.forEach((p, idx) => { if ((!p.zip || !String(p.zip).trim()) && p.address && p.city) todo.push({ idx, address: p.address, city: p.city }); });
  const batchList = todo.slice(0, LIMIT === Infinity ? todo.length : LIMIT);
  console.log(`ranked.json: ${arr.length} rows · ${todo.length} missing zip · geocoding ${batchList.length}${DRY ? ' (DRY RUN)' : ''}`);
  console.log('cost: $0 (US Census batch geocoder — free, no key)');

  let matched = 0;
  for (let i = 0; i < batchList.length; i += CHUNK) {
    const chunk = batchList.slice(i, i + CHUNK);
    process.stdout.write(`  batch ${Math.floor(i / CHUNK) + 1}/${Math.ceil(batchList.length / CHUNK)} (${chunk.length} addrs)… `);
    let res = {};
    try { res = await geocodeBatch(chunk); } catch (e) { console.log('ERR ' + e.message); continue; }
    for (const [id, geo] of Object.entries(res)) {
      const p = arr[+id]; if (!p) continue;
      p.zip = geo.zip; if (geo.lat != null) p.lat = geo.lat; if (geo.lng != null) p.lng = geo.lng;
      matched++;
    }
    console.log(`matched ${Object.keys(res).length}`);
  }
  const rate = batchList.length ? (100 * matched / batchList.length).toFixed(1) : '0';
  console.log(`\nmatched ${matched}/${batchList.length} (${rate}%) · total rows with zip now: ${arr.filter(p => p.zip).length}/${arr.length}`);
  if (DRY) { console.log('DRY RUN — ranked.json not written'); return; }
  fs.copyFileSync(RANKED, RANKED + '.bak-geocode');   // rollback point
  fs.writeFileSync(RANKED, JSON.stringify(doc, null, 0));
  console.log(`wrote ${RANKED} (backup: ranked.json.bak-geocode)`);
})();