`scripts/fetch-bh-properties.js`

190 lines
#!/usr/bin/env node
// fetch-bh-properties.js — Stream Beverly Hills property parcels from LA County Assessor
// (public, FREE, $0 — LA County eGIS ArcGIS feature service)
//
// Source: LA County Assessor Parcel Data (public open data)
//   https://services.arcgis.com/RmCCgQtiZLDCtblq/arcgis/rest/services/Parcel_Data_2021_Table/FeatureServer/0
// Filters: SitusZIP5 IN ('90210','90211','90212') AND RollYear='2025'
// Expected: ~13,658 BH parcels
//
// Usage:
//   node scripts/fetch-bh-properties.js             # full fetch
//   node scripts/fetch-bh-properties.js --limit 100 # smoke test
//
// Output: data/bh-properties.json (array of property objects)
'use strict';
const fs = require('fs');
const path = require('path');
const https = require('https');

const BASE = 'https://services.arcgis.com/RmCCgQtiZLDCtblq/arcgis/rest/services/Parcel_Data_2021_Table/FeatureServer/0';
const PAGE = 2000;
const ROLL_YEAR = '2025';
// Beverly Hills spans 3 zip codes
const BH_ZIPS = ['90210', '90211', '90212'];
const WHERE = `SitusZIP5 IN ('${BH_ZIPS.join("','")}') AND RollYear='${ROLL_YEAR}'`;

const FIELDS = [
  'AIN', 'RollYear', 'PropertyLocation',
  'SitusHouseNo', 'SitusFraction', 'SitusDirection', 'SitusStreet', 'SitusUnit',
  'SitusCity', 'SitusZIP5',
  'UseType', 'UseCode', 'UseCodeDescChar1', 'UseCodeDescChar2',
  'YearBuilt', 'EffectiveYearBuilt', 'SQFTmain', 'Bedrooms', 'Bathrooms', 'Units',
  'RecordingDate',
  'Roll_LandValue', 'Roll_ImpValue', 'Roll_totLandImp', 'Roll_TotalValue',
  'Roll_HomeOwnersExemp', 'netTaxableValue', 'isTaxableParcel',
  'ParcelClassification', 'AdminRegion',
  'CENTER_LAT', 'CENTER_LON'
].join(',');

// Parse --limit N from args
const limitArg = process.argv.indexOf('--limit');
const LIMIT = limitArg >= 0 ? parseInt(process.argv[limitArg + 1], 10) : 0;

const DATA_DIR = path.join(__dirname, '..', 'data');
fs.mkdirSync(DATA_DIR, { recursive: true });
const OUT = path.join(DATA_DIR, 'bh-properties.json');

function getJSON(url) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, { timeout: 60000 }, res => {
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let buf = '';
      res.setEncoding('utf8');
      res.on('data', d => buf += d);
      res.on('end', () => {
        try { resolve(JSON.parse(buf)); }
        catch (e) { reject(new Error('JSON parse error: ' + e.message)); }
      });
    });
    req.on('timeout', () => { req.destroy(new Error('timeout')); });
    req.on('error', reject);
  });
}

function fmtUSD(n) {
  if (!n) return null;
  return new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD', maximumFractionDigits: 0 }).format(n);
}

function normalizeParcel(attrs) {
  // Build a clean address string
  const parts = [attrs.SitusHouseNo, attrs.SitusFraction, attrs.SitusDirection, attrs.SitusStreet, attrs.SitusUnit]
    .filter(Boolean).join(' ').trim();
  const addr = parts || attrs.PropertyLocation || '';

  // Format APN: XXXXXXXXXX → XXXX-XXX-XXX
  const ain = String(attrs.AIN || '');
  const apn = ain.length === 10 ? `${ain.slice(0,4)}-${ain.slice(4,7)}-${ain.slice(7)}` : ain;

  // Convert recording date (milliseconds) to date string
  let lastSaleDate = null;
  if (attrs.RecordingDate) {
    try { lastSaleDate = new Date(attrs.RecordingDate).toISOString().slice(0, 10); } catch (_) {}
  }

  return {
    ain: ain,
    apn: apn,
    address: addr,
    city: attrs.SitusCity || 'BEVERLY HILLS',
    zip: attrs.SitusZIP5 || '',
    property_location: attrs.PropertyLocation || addr,
    use_type: attrs.UseType || '',
    use_code: attrs.UseCode || '',
    use_desc: attrs.UseCodeDescChar1 || '',
    use_desc2: attrs.UseCodeDescChar2 || '',
    year_built: attrs.YearBuilt || null,
    effective_year_built: attrs.EffectiveYearBuilt || null,
    sqft: attrs.SQFTmain || null,
    bedrooms: attrs.Bedrooms || null,
    bathrooms: attrs.Bathrooms || null,
    units: attrs.Units || null,
    last_sale_date: lastSaleDate,
    land_value: attrs.Roll_LandValue || null,
    improvement_value: attrs.Roll_ImpValue || null,
    total_value: attrs.Roll_TotalValue || null,
    net_taxable_value: attrs.netTaxableValue || null,
    homeowner_exemption: attrs.Roll_HomeOwnersExemp || null,
    parcel_classification: attrs.ParcelClassification || '',
    admin_region: attrs.AdminRegion || '',
    lat: attrs.CENTER_LAT || null,
    lon: attrs.CENTER_LON || null,
    roll_year: attrs.RollYear || ROLL_YEAR,
    // formatted display values
    total_value_fmt: fmtUSD(attrs.Roll_TotalValue),
    land_value_fmt: fmtUSD(attrs.Roll_LandValue),
    improvement_value_fmt: fmtUSD(attrs.Roll_ImpValue),
    // county links
    assessor_url: `https://assessor.lacounty.gov/search-by-ain/?ain=${ain}`,
    tax_url: `https://tax.lacounty.gov/`
  };
}

async function main() {
  console.log(`[bh-fetch] LA County Assessor — Beverly Hills Parcel Fetch ($0, free public data)`);
  console.log(`[bh-fetch] Filter: ${WHERE}`);

  // Get count first
  const cntData = await getJSON(
    `${BASE}/query?where=${encodeURIComponent(WHERE)}&returnCountOnly=true&f=json`
  );
  const total = cntData.count || 0;
  const target = LIMIT ? Math.min(LIMIT, total) : total;
  console.log(`[bh-fetch] Total BH parcels available: ${total.toLocaleString()}${LIMIT ? ` (capped to ${target} via --limit)` : ''}`);
  console.log(`[bh-fetch] Output: ${OUT}`);

  const properties = [];
  let offset = 0, pages = 0;

  while (properties.length < target) {
    const url = `${BASE}/query?where=${encodeURIComponent(WHERE)}&outFields=${encodeURIComponent(FIELDS)}` +
      `&returnGeometry=false&orderByFields=AIN&resultOffset=${offset}&resultRecordCount=${PAGE}&f=json`;

    let data;
    for (let attempt = 1; ; attempt++) {
      try { data = await getJSON(url); break; }
      catch (e) {
        if (attempt >= 4) throw new Error(`Page at offset ${offset} failed after ${attempt} tries: ${e.message}`);
        await new Promise(r => setTimeout(r, 1500 * attempt));
      }
    }
    const feats = data.features || [];
    if (!feats.length) break;

    for (const f of feats) {
      if (properties.length >= target) break;
      properties.push(normalizeParcel(f.attributes));
    }

    offset += feats.length;
    pages++;
    const pct = total ? ((properties.length / target) * 100).toFixed(1) : '?';
    process.stdout.write(`[bh-fetch] ${properties.length.toLocaleString()}/${target.toLocaleString()} (${pct}%) · page ${pages}\n`);

    // ArcGIS end-of-results signal
    if (data.exceededTransferLimit === false || feats.length < PAGE) break;
  }

  // Save JSON
  fs.writeFileSync(OUT, JSON.stringify(properties, null, 2));

  // Compute stats
  const valued = properties.filter(p => p.total_value);
  const avgVal = valued.length ? Math.round(valued.reduce((s, p) => s + p.total_value, 0) / valued.length) : 0;
  const maxVal = valued.length ? Math.max(...valued.map(p => p.total_value)) : 0;
  const byUse = {};
  for (const p of properties) {
    const k = p.use_desc || 'Unknown';
    byUse[k] = (byUse[k] || 0) + 1;
  }

  console.log(`\n[bh-fetch] Done: ${properties.length.toLocaleString()} properties saved to ${OUT}`);
  console.log(`[bh-fetch] Avg assessed value: ${fmtUSD(avgVal)}`);
  console.log(`[bh-fetch] Max assessed value: ${fmtUSD(maxVal)}`);
  console.log(`[bh-fetch] By use type:`, byUse);
  console.log(`[bh-fetch] Cost: $0 (LA County open data)`);
}

main().catch(e => { console.error('[bh-fetch] FATAL:', e.message); process.exit(1); });