assessor: key-set pagination completion (AIN>cursor) to finish the roll past the resultOffset false-stop at 1.884M

1634e83721d3cc77133d8ebce61419efe306abc8 · 2026-06-30 17:46:04 -0700 · Steve Abrams
Files touched

A scripts/fetch-assessor-keyset.js
Diff

commit 1634e83721d3cc77133d8ebce61419efe306abc8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jun 30 17:46:04 2026 -0700

    assessor: key-set pagination completion (AIN>cursor) to finish the roll past the resultOffset false-stop at 1.884M
---
 scripts/fetch-assessor-keyset.js | 61 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/scripts/fetch-assessor-keyset.js b/scripts/fetch-assessor-keyset.js
new file mode 100644
index 0000000..39f4fc7
--- /dev/null
+++ b/scripts/fetch-assessor-keyset.js
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+// fetch-assessor-keyset.js — COMPLETE the assessor roll via key-set pagination (AIN > cursor).
+// The resultOffset pull stalled at ~1.884M (spurious end-signal). Key-set pagination (order by AIN,
+// where AIN > lastAIN) can't hit an offset ceiling or false-stop — it walks to the true end.
+// Reads the last AIN already in the NDJSON, appends the remainder. Stops only on an empty page.
+//   node scripts/fetch-assessor-keyset.js
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const readline = require('readline');
+
+const BASE = 'https://services.arcgis.com/RmCCgQtiZLDCtblq/arcgis/rest/services/Parcel_Data_2021_Table/FeatureServer/0';
+const PAGE = 2000, ROLL_YEAR = '2025';
+const FIELDS = ['AIN','RollYear','PropertyLocation','SitusHouseNo','SitusStreet','SitusUnit','SitusCity','SitusZIP5',
+  'UseType','UseCode','UseCodeDescChar1','UseCodeDescChar2','YearBuilt','EffectiveYearBuilt','SQFTmain','Bedrooms',
+  'Bathrooms','Units','RecordingDate','Roll_LandValue','Roll_ImpValue','Roll_totLandImp','Roll_TotalValue',
+  'netTaxableValue','isTaxableParcel','ParcelClassification','AdminRegion','CENTER_LAT','CENTER_LON'].join(',');
+const OUT = path.join(__dirname, '..', 'data', 'raw', 'assessor-parcels-2025.ndjson');
+
+function getJSON(url) {
+  return new Promise((resolve, reject) => {
+    const req = https.get(url, { timeout: 60000 }, res => {
+      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+      let buf = ''; res.setEncoding('utf8'); res.on('data', d => buf += d);
+      res.on('end', () => { try { resolve(JSON.parse(buf)); } catch (e) { reject(new Error('bad JSON')); } });
+    });
+    req.on('timeout', () => req.destroy(new Error('timeout'))); req.on('error', reject);
+  });
+}
+function fmtB(n) { return n > 1e9 ? (n / 1e9).toFixed(2) + ' GB' : (n / 1e6).toFixed(1) + ' MB'; }
+async function lastAIN() {
+  return new Promise((resolve) => {
+    let last = ''; const rl = readline.createInterface({ input: fs.createReadStream(OUT) });
+    rl.on('line', l => { const i = l.indexOf('"AIN":"'); if (i >= 0) last = l.slice(i + 7, l.indexOf('"', i + 7)); });
+    rl.on('close', () => resolve(last));
+  });
+}
+async function main() {
+  const t0 = Date.now();
+  const total = (await getJSON(`${BASE}/query?where=${encodeURIComponent(`RollYear='${ROLL_YEAR}'`)}&returnCountOnly=true&f=json`)).count || 0;
+  let cursor = await lastAIN();
+  let have = fs.readFileSync(OUT, 'utf8').split('\n').filter(Boolean).length;
+  console.log(`[keyset] total ${total.toLocaleString()} · have ${have.toLocaleString()} · resuming AIN > ${cursor}`);
+  const ws = fs.createWriteStream(OUT, { flags: 'a' });
+  let added = 0, pages = 0;
+  for (;;) {
+    const where = encodeURIComponent(`RollYear='${ROLL_YEAR}' AND AIN>'${cursor}'`);
+    const url = `${BASE}/query?where=${where}&outFields=${FIELDS}&returnGeometry=false&orderByFields=AIN&resultRecordCount=${PAGE}&f=json`;
+    let data;
+    for (let a = 1; ; a++) { try { data = await getJSON(url); break; } catch (e) { if (a >= 5) throw new Error(`page after ${cursor} failed: ${e.message}`); await new Promise(r => setTimeout(r, 1500 * a)); } }
+    const feats = data.features || [];
+    if (!feats.length) break;                 // true end — no more AINs greater than cursor
+    for (const f of feats) { ws.write(JSON.stringify(f.attributes) + '\n'); added++; cursor = f.attributes.AIN; }
+    pages++;
+    if (pages % 25 === 0) { let sz = 0; try { sz = fs.statSync(OUT).size; } catch (_) {} process.stdout.write(`[keyset] +${added.toLocaleString()} (now ${(have + added).toLocaleString()}/${total.toLocaleString()}) · ${fmtB(sz)} · AIN ${cursor}\n`); }
+  }
+  await new Promise(r => ws.end(r));
+  console.log(`[keyset] DONE — added ${added.toLocaleString()} · file now ${(have + added).toLocaleString()}/${total.toLocaleString()} · ${((Date.now() - t0) / 1000).toFixed(0)}s`);
+}
+main().catch(e => { console.error('[keyset] FAILED:', e.message); process.exit(1); });
← ad0fbf9 Arcstone prior-relationship flag (chip on grid card + list + · back to Commercialrealestate · assessor keyset: stream line-count (1.3GB file exceeds V8 ma 1760f5c →