assessor keyset: stream line-count (1.3GB file exceeds V8 max string; readFileSync failed)

1760f5c9e4d9fea890514dfedfe6645d4c21d764 · 2026-06-30 17:47:28 -0700 · Steve Abrams

Files touched

M scripts/fetch-assessor-keyset.js

Diff

commit 1760f5c9e4d9fea890514dfedfe6645d4c21d764
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jun 30 17:47:28 2026 -0700

    assessor keyset: stream line-count (1.3GB file exceeds V8 max string; readFileSync failed)
---
 scripts/fetch-assessor-keyset.js | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/fetch-assessor-keyset.js b/scripts/fetch-assessor-keyset.js
index 39f4fc7..faa41de 100644
--- a/scripts/fetch-assessor-keyset.js
+++ b/scripts/fetch-assessor-keyset.js
@@ -29,18 +29,19 @@ function getJSON(url) {
   });
 }
 function fmtB(n) { return n > 1e9 ? (n / 1e9).toFixed(2) + ' GB' : (n / 1e6).toFixed(1) + ' MB'; }
-async function lastAIN() {
+async function lastAINAndCount() {
+  // stream — the NDJSON is >1GB, too big for readFileSync (exceeds V8 max string length).
   return new Promise((resolve) => {
-    let last = ''; const rl = readline.createInterface({ input: fs.createReadStream(OUT) });
-    rl.on('line', l => { const i = l.indexOf('"AIN":"'); if (i >= 0) last = l.slice(i + 7, l.indexOf('"', i + 7)); });
-    rl.on('close', () => resolve(last));
+    let last = '', n = 0; const rl = readline.createInterface({ input: fs.createReadStream(OUT) });
+    rl.on('line', l => { if (!l) return; n++; const i = l.indexOf('"AIN":"'); if (i >= 0) last = l.slice(i + 7, l.indexOf('"', i + 7)); });
+    rl.on('close', () => resolve({ last, count: n }));
   });
 }
 async function main() {
   const t0 = Date.now();
   const total = (await getJSON(`${BASE}/query?where=${encodeURIComponent(`RollYear='${ROLL_YEAR}'`)}&returnCountOnly=true&f=json`)).count || 0;
-  let cursor = await lastAIN();
-  let have = fs.readFileSync(OUT, 'utf8').split('\n').filter(Boolean).length;
+  const { last: cursor0, count: have } = await lastAINAndCount();
+  let cursor = cursor0;
   console.log(`[keyset] total ${total.toLocaleString()} · have ${have.toLocaleString()} · resuming AIN > ${cursor}`);
   const ws = fs.createWriteStream(OUT, { flags: 'a' });
   let added = 0, pages = 0;

← 1634e83 assessor: key-set pagination completion (AIN>cursor) to fini · back to Commercialrealestate · Map markers → 100-yr property-history popup (📜 History link ac6be59 →