← back to La Socrata Ingester

assessor --full now scans all roll years (matches docs); default stays latest year

b04579414913ef6acf16967a158bd4bf476b1434 · 2026-08-11 02:08:13 -0700 · steve

--full was still applying the RollYear=2025 defaultWhere. Now fullScan overrides
to an unfiltered (all 12.1M roll years) crawl as the CLI/README document; the
no-flag default remains the cheap latest-roll-year path.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

• M src/adapters/arcgis.js
• M src/ingest.js

Diff

commit b04579414913ef6acf16967a158bd4bf476b1434
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Aug 11 02:08:13 2026 -0700

    assessor --full now scans all roll years (matches docs); default stays latest year
    
    --full was still applying the RollYear=2025 defaultWhere. Now fullScan overrides
    to an unfiltered (all 12.1M roll years) crawl as the CLI/README document; the
    no-flag default remains the cheap latest-roll-year path.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/adapters/arcgis.js | 13 ++++++++++---
 src/ingest.js          |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/adapters/arcgis.js b/src/adapters/arcgis.js
index 9fba835..b052a7b 100644
--- a/src/adapters/arcgis.js
+++ b/src/adapters/arcgis.js
@@ -7,12 +7,19 @@ import { fetchJson } from './http.js';
 // src fields used: endpoint (…/FeatureServer/0), orderBy, cursorField?, defaultWhere?
 // opts: since (string | null), pageSize (default 2000), maxRows
 export async function* arcgisPages(src, opts = {}) {
-  const { since = null, pageSize = 2000, maxRows = Infinity } = opts;
+  const { since = null, pageSize = 2000, maxRows = Infinity, fullScan = false } = opts;
   const base = `${src.endpoint}/query`;
 
+  // fullScan (--full) = every row, ignore the latest-year defaultWhere.
+  // Otherwise: incremental cursor if we have one, else the cheap defaultWhere.
   const clauses = [];
-  if (since && src.cursorField) clauses.push(`${src.cursorField} >= '${since}'`);
-  else if (src.defaultWhere) clauses.push(src.defaultWhere);
+  if (fullScan) {
+    // no filter — all roll years
+  } else if (since && src.cursorField) {
+    clauses.push(`${src.cursorField} >= '${since}'`);
+  } else if (src.defaultWhere) {
+    clauses.push(src.defaultWhere);
+  }
   const where = clauses.length ? clauses.join(' AND ') : '1=1';
 
   let offset = 0;
diff --git a/src/ingest.js b/src/ingest.js
index 0567d3a..33b2003 100644
--- a/src/ingest.js
+++ b/src/ingest.js
@@ -22,7 +22,7 @@ export async function ingestSource(name, opts = {}) {
   const prev = await getState(name);
   const since = !opts.full && src.cursorField ? prev?.last_cursor || null : null;
 
-  const pagerOpts = { since, appToken, maxRows: opts.maxRows ?? Infinity };
+  const pagerOpts = { since, appToken, maxRows: opts.maxRows ?? Infinity, fullScan: !!opts.full };
   if (opts.pageSize) pagerOpts.pageSize = opts.pageSize;
   const pager = src.platform === 'socrata' ? socrataPages(src, pagerOpts) : arcgisPages(src, pagerOpts);
 

← 4c680aa Fix: dedupe rows by conflict key within upsert batch (last-w  ·  back to La Socrata Ingester  ·  Add --year=YYYY for resumable per-roll-year assessor loads 0c2fe12 →