[object Object]

← back to La Socrata Ingester

Add --year=YYYY for resumable per-roll-year assessor loads

0c2fe1267e2a703d8db10e1e686de459044df127 · 2026-08-11 07:03:03 -0700 · steve

Splits the fragile 12.1M full parcel crawl into idempotent ~2.4M/year chunks so
a mid-run failure only re-costs one year. Accurate run-mode log label too.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 0c2fe1267e2a703d8db10e1e686de459044df127
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Aug 11 07:03:03 2026 -0700

    Add --year=YYYY for resumable per-roll-year assessor loads
    
    Splits the fragile 12.1M full parcel crawl into idempotent ~2.4M/year chunks so
    a mid-run failure only re-costs one year. Accurate run-mode log label too.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/adapters/arcgis.js | 7 +++++--
 src/cli.js             | 1 +
 src/ingest.js          | 7 ++++++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/adapters/arcgis.js b/src/adapters/arcgis.js
index b052a7b..45940d9 100644
--- a/src/adapters/arcgis.js
+++ b/src/adapters/arcgis.js
@@ -7,13 +7,16 @@ import { fetchJson } from './http.js';
 // src fields used: endpoint (…/FeatureServer/0), orderBy, cursorField?, defaultWhere?
 // opts: since (string | null), pageSize (default 2000), maxRows
 export async function* arcgisPages(src, opts = {}) {
-  const { since = null, pageSize = 2000, maxRows = Infinity, fullScan = false } = opts;
+  const { since = null, pageSize = 2000, maxRows = Infinity, fullScan = false, whereOverride = null } = opts;
   const base = `${src.endpoint}/query`;
 
+  // whereOverride (--year) wins outright: pull exactly that slice.
   // fullScan (--full) = every row, ignore the latest-year defaultWhere.
   // Otherwise: incremental cursor if we have one, else the cheap defaultWhere.
   const clauses = [];
-  if (fullScan) {
+  if (whereOverride) {
+    clauses.push(whereOverride);
+  } else if (fullScan) {
     // no filter — all roll years
   } else if (since && src.cursorField) {
     clauses.push(`${src.cursorField} >= '${since}'`);
diff --git a/src/cli.js b/src/cli.js
index ddae529..574c917 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -57,6 +57,7 @@ async function main() {
     full: flags.has('--full'),
     maxRows: kv.max ? Number(kv.max) : undefined,
     pageSize: kv.page ? Number(kv.page) : undefined,
+    year: kv.year || undefined,
   };
 
   const results = [];
diff --git a/src/ingest.js b/src/ingest.js
index 33b2003..ad5ebb9 100644
--- a/src/ingest.js
+++ b/src/ingest.js
@@ -24,9 +24,14 @@ export async function ingestSource(name, opts = {}) {
 
   const pagerOpts = { since, appToken, maxRows: opts.maxRows ?? Infinity, fullScan: !!opts.full };
   if (opts.pageSize) pagerOpts.pageSize = opts.pageSize;
+  // --year=YYYY (ArcGIS only): pull exactly one roll year, resumable/idempotent.
+  if (opts.year && src.platform === 'arcgis' && src.cursorField) {
+    pagerOpts.whereOverride = `${src.cursorField}='${opts.year}'`;
+  }
   const pager = src.platform === 'socrata' ? socrataPages(src, pagerOpts) : arcgisPages(src, pagerOpts);
 
-  log(`▶ ${name} [${src.platform} ${datasetLabel}] ${opts.full ? 'FULL' : since ? `since ${since}` : 'initial'}${opts.maxRows ? ` (max ${opts.maxRows})` : ''}`);
+  const mode = opts.year ? `year=${opts.year}` : opts.full ? 'FULL' : since ? `since ${since}` : 'initial';
+  log(`▶ ${name} [${src.platform} ${datasetLabel}] ${mode}${opts.maxRows ? ` (max ${opts.maxRows})` : ''}`);
 
   let total = 0;
   let maxCursor = prev?.last_cursor || null;

← b045794 assessor --full now scans all roll years (matches docs); def  ·  back to La Socrata Ingester  ·  Add current_permit_leads view + CSV export (2,034 live resid b9cec1a →