`scripts/fetch-assessor-keyset.js`

63 lines
#!/usr/bin/env node
// fetch-assessor-keyset.js — COMPLETE the assessor roll via key-set pagination (AIN > cursor).
// The resultOffset pull stalled at ~1.884M (spurious end-signal). Key-set pagination (order by AIN,
// where AIN > lastAIN) can't hit an offset ceiling or false-stop — it walks to the true end.
// Reads the last AIN already in the NDJSON, appends the remainder. Stops only on an empty page.
//   node scripts/fetch-assessor-keyset.js
'use strict';
const fs = require('fs');
const path = require('path');
const https = require('https');
const readline = require('readline');

const BASE = 'https://services.arcgis.com/RmCCgQtiZLDCtblq/arcgis/rest/services/Parcel_Data_2021_Table/FeatureServer/0';
const PAGE = 2000, ROLL_YEAR = '2025';
const FIELDS = ['AIN','RollYear','PropertyLocation','SitusHouseNo','SitusStreet','SitusUnit','SitusCity','SitusZIP5',
  'UseType','UseCode','UseCodeDescChar1','UseCodeDescChar2','YearBuilt','EffectiveYearBuilt','SQFTmain','Bedrooms',
  'Bathrooms','Units','RecordingDate','Roll_LandValue','Roll_ImpValue','Roll_totLandImp','Roll_TotalValue',
  'netTaxableValue','isTaxableParcel','ParcelClassification','AdminRegion','CENTER_LAT','CENTER_LON'].join(',');
const OUT = path.join(__dirname, '..', 'data', 'raw', 'assessor-parcels-2025.ndjson');

function getJSON(url) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, { timeout: 60000 }, res => {
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let buf = ''; res.setEncoding('utf8'); res.on('data', d => buf += d);
      res.on('end', () => { try { resolve(JSON.parse(buf)); } catch (e) { reject(new Error('bad JSON')); } });
    });
    req.on('timeout', () => req.destroy(new Error('timeout'))); req.on('error', reject);
  });
}
function fmtB(n) { return n > 1e9 ? (n / 1e9).toFixed(2) + ' GB' : (n / 1e6).toFixed(1) + ' MB'; }
async function lastAINAndCount() {
  // stream — the NDJSON is >1GB, too big for readFileSync (exceeds V8 max string length).
  return new Promise((resolve) => {
    let last = '', n = 0; const rl = readline.createInterface({ input: fs.createReadStream(OUT) });
    rl.on('line', l => { if (!l) return; n++; const i = l.indexOf('"AIN":"'); if (i >= 0) last = l.slice(i + 7, l.indexOf('"', i + 7)); });
    rl.on('close', () => resolve({ last, count: n }));
  });
}
async function main() {
  const t0 = Date.now();
  const total = (await getJSON(`${BASE}/query?where=${encodeURIComponent(`RollYear='${ROLL_YEAR}'`)}&returnCountOnly=true&f=json`)).count || 0;
  const { last: cursor0, count: have } = await lastAINAndCount();
  let cursor = cursor0;
  console.log(`[keyset] total ${total.toLocaleString()} · have ${have.toLocaleString()} · resuming AIN > ${cursor}`);
  const ws = fs.createWriteStream(OUT, { flags: 'a' });
  let added = 0, pages = 0;
  for (;;) {
    const where = encodeURIComponent(`RollYear='${ROLL_YEAR}' AND AIN>'${cursor}'`);
    const url = `${BASE}/query?where=${where}&outFields=${FIELDS}&returnGeometry=false&orderByFields=AIN&resultRecordCount=${PAGE}&f=json`;
    let data;
    for (let a = 1; ; a++) { try { data = await getJSON(url); break; } catch (e) { if (a >= 5) throw new Error(`page after ${cursor} failed: ${e.message}`); await new Promise(r => setTimeout(r, 1500 * a)); } }
    const feats = data.features || [];
    if (!feats.length) break;                 // true end — no more AINs greater than cursor
    for (const f of feats) { ws.write(JSON.stringify(f.attributes) + '\n'); added++; cursor = f.attributes.AIN; }
    pages++;
    if (pages % 25 === 0) { let sz = 0; try { sz = fs.statSync(OUT).size; } catch (_) {} process.stdout.write(`[keyset] +${added.toLocaleString()} (now ${(have + added).toLocaleString()}/${total.toLocaleString()}) · ${fmtB(sz)} · AIN ${cursor}\n`); }
  }
  await new Promise(r => ws.end(r));
  console.log(`[keyset] DONE — added ${added.toLocaleString()} · file now ${(have + added).toLocaleString()}/${total.toLocaleString()} · ${((Date.now() - t0) / 1000).toFixed(0)}s`);
}
main().catch(e => { console.error('[keyset] FAILED:', e.message); process.exit(1); });