← back to La Socrata Ingester
Treat audited zoning failure as aggregate degradation
ece3ea5e621dd3a86f9eb0f4b327c6577bada51a · 2026-08-30 08:09:33 -0700 · Steve Abrams
Files touched
M src/cli.jsA src/run-policy.jsM src/sources.jsA test/run-policy.test.mjs
Diff
commit ece3ea5e621dd3a86f9eb0f4b327c6577bada51a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sun Aug 30 08:09:33 2026 -0700
Treat audited zoning failure as aggregate degradation
---
src/cli.js | 16 ++++++++++---
src/run-policy.js | 19 +++++++++++++++
src/sources.js | 13 ++++++++++-
test/run-policy.test.mjs | 61 ++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 105 insertions(+), 4 deletions(-)
diff --git a/src/cli.js b/src/cli.js
index 574c917..cf8031d 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -1,6 +1,7 @@
#!/usr/bin/env node
import { SOURCES, GROUPS } from './sources.js';
import { ingestSource, closePool } from './ingest.js';
+import { classifyRunResults } from './run-policy.js';
const argv = process.argv.slice(2);
const flags = new Set(argv.filter((a) => a.startsWith('--') && !a.includes('=')));
@@ -69,12 +70,21 @@ async function main() {
}
}
+ const policy = classifyRunResults(results, SOURCES, token);
+ const toleratedNames = new Set(policy.tolerated.map((r) => r.name));
+
console.log('\n=== summary ===');
for (const r of results) {
- console.log(r.error ? ` ✖ ${r.name}: ${r.error}` : ` ✔ ${r.name}: ${r.total} rows`);
+ if (!r.error) console.log(` ✔ ${r.name}: ${r.total} rows`);
+ else if (toleratedNames.has(r.name)) {
+ const meta = SOURCES[r.name].allowFailure;
+ console.log(` ⚠ ${r.name}: known degraded (${meta.since}; ${meta.reason}): ${r.error}`);
+ } else console.log(` ✖ ${r.name}: ${r.error}`);
+ }
+ if (policy.tolerated.length) {
+ console.log(` ⚠ tolerated ${policy.tolerated.length} audited failure(s) in aggregate all run`);
}
- const failed = results.filter((r) => r.error).length;
- if (failed) process.exitCode = 1;
+ if (policy.exitCode) process.exitCode = policy.exitCode;
}
main().finally(closePool);
diff --git a/src/run-policy.js b/src/run-policy.js
new file mode 100644
index 0000000..7e9e09d
--- /dev/null
+++ b/src/run-policy.js
@@ -0,0 +1,19 @@
+// Aggregate-run failure policy. Exceptions live on source metadata so they are
+// explicit, reviewable, and automatically disappear from this policy when the
+// metadata is removed. A direct source/group run remains strict.
+export function classifyRunResults(results, sources, targetToken) {
+ const failures = results.filter((result) => result.error);
+ const aggregateAll = targetToken === 'all';
+ const tolerated = aggregateAll
+ ? failures.filter((result) => sources[result.name]?.allowFailure?.scope === 'aggregate-all-only')
+ : [];
+ const toleratedNames = new Set(tolerated.map((result) => result.name));
+ const unexpected = failures.filter((result) => !toleratedNames.has(result.name));
+
+ return {
+ failures,
+ tolerated,
+ unexpected,
+ exitCode: unexpected.length ? 1 : 0,
+ };
+}
diff --git a/src/sources.js b/src/sources.js
index bc477f1..0a683ac 100644
--- a/src/sources.js
+++ b/src/sources.js
@@ -201,7 +201,18 @@ Object.assign(SOURCES, {
},
// Zoning + land use (~50–59k). NavigateLA gateway 502s on sorted geometry -> use
// offset paging with NO orderByFields (noOrder) + small pages.
- gis_zoning: gisSrc('zoning', 71, ['ZONE_CMPLT', 'ZONE_CLASS'], { pageSize: 2000, noOrder: true }),
+ gis_zoning: gisSrc('zoning', 71, ['ZONE_CMPLT', 'ZONE_CLASS'], {
+ pageSize: 2000,
+ noOrder: true,
+ // Audited exception: keep attempting this source so an upstream recovery is
+ // detected automatically, but do not fail the aggregate `all` run for its
+ // documented deep-offset gateway failure. Direct gis_zoning runs stay strict.
+ allowFailure: {
+ since: '2026-08-11',
+ scope: 'aggregate-all-only',
+ reason: 'NavigateLA gateway consistently HTTP 502s on geometry pagination near offset 42000',
+ },
+ }),
// Land use: hosted LADCP FeatureServer (reliable) instead of the flaky NavigateLA
// gateway. Fields are lowercase here (objectid/gplu/gplu_desc).
gis_landuse: {
diff --git a/test/run-policy.test.mjs b/test/run-policy.test.mjs
new file mode 100644
index 0000000..2ab9106
--- /dev/null
+++ b/test/run-policy.test.mjs
@@ -0,0 +1,61 @@
+// TK-10955: zero-network, zero-database proof of aggregate exit semantics.
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { classifyRunResults } from '../src/run-policy.js';
+import { SOURCES } from '../src/sources.js';
+
+const sources = {
+ gis_zoning: {
+ allowFailure: {
+ since: '2026-08-11',
+ scope: 'aggregate-all-only',
+ reason: 'mock known upstream failure',
+ },
+ },
+ healthy_source: {},
+};
+
+const knownFailure = [{ name: 'gis_zoning', error: 'mock HTTP 502 after retries' }];
+const unexpectedFailure = [{ name: 'healthy_source', error: 'mock schema regression' }];
+
+const tolerated = classifyRunResults(knownFailure, sources, 'all');
+assert.equal(tolerated.exitCode, 0, 'known gis_zoning failure must not fail aggregate all');
+assert.deepEqual(tolerated.tolerated.map((r) => r.name), ['gis_zoning']);
+assert.equal(tolerated.unexpected.length, 0);
+
+const unexpected = classifyRunResults(unexpectedFailure, sources, 'all');
+assert.equal(unexpected.exitCode, 1, 'unexpected source failure must fail aggregate all');
+assert.deepEqual(unexpected.unexpected.map((r) => r.name), ['healthy_source']);
+
+const directKnown = classifyRunResults(knownFailure, sources, 'gis_zoning');
+assert.equal(directKnown.exitCode, 1, 'direct source runs must remain strict');
+assert.equal(directKnown.tolerated.length, 0);
+
+const mixed = classifyRunResults([...knownFailure, ...unexpectedFailure], sources, 'all');
+assert.equal(mixed.exitCode, 1, 'a tolerated failure must never mask an unexpected failure');
+assert.deepEqual(mixed.tolerated.map((r) => r.name), ['gis_zoning']);
+assert.deepEqual(mixed.unexpected.map((r) => r.name), ['healthy_source']);
+
+console.log('PASS run policy: known aggregate failure exits0; direct/unexpected/mixed failures exit1');
+
+// Exercise the OS process boundary without importing cli.js (which would open PG).
+// The child receives only mocked results plus real audited source metadata, then exits
+// with the exact code the CLI consumes.
+const policyUrl = new URL('../src/run-policy.js', import.meta.url).href;
+function mockedProcessStatus(results, targetToken) {
+ const child = spawnSync(process.execPath, [
+ '--input-type=module',
+ '--eval',
+ `import { classifyRunResults } from ${JSON.stringify(policyUrl)};
+ const sources = ${JSON.stringify(SOURCES)};
+ const results = ${JSON.stringify(results)};
+ process.exit(classifyRunResults(results, sources, ${JSON.stringify(targetToken)}).exitCode);`,
+ ], { encoding: 'utf8' });
+ assert.equal(child.signal, null, child.stderr);
+ return child.status;
+}
+
+assert.equal(mockedProcessStatus(knownFailure, 'all'), 0, 'known aggregate failure OS exit must be 0');
+assert.equal(mockedProcessStatus(unexpectedFailure, 'all'), 1, 'unexpected aggregate failure OS exit must be 1');
+assert.equal(mockedProcessStatus(knownFailure, 'gis_zoning'), 1, 'direct known-source failure OS exit must be 1');
+console.log('PASS process boundary: mocked no-DB child exits 0/1/1 as required');
← 9460ee0 auto-data-snapshot: 2026-08-27T08:19:36 (1 data files) — dat
·
back to La Socrata Ingester
·
Record LA refresh exit-policy proof 79e59f8 →