← back to Stayclaim
Fail historic monument imports on invalid feature pages
a8d995ede99a1f66eee9c4026370844aafe1f1c4 · 2026-09-10 09:59:40 -0700 · Steve Abrams
Files touched
M scripts/ingest-la-historic-monuments.tsM scripts/tests/ingest-la-historic-monuments.test.cjsM verification/e2e-proof.jsonA verification/hcm-feature-query-tests.txtA verification/hcm-layer-identity-e2e-proof.json
Diff
commit a8d995ede99a1f66eee9c4026370844aafe1f1c4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 09:59:40 2026 -0700
Fail historic monument imports on invalid feature pages
---
scripts/ingest-la-historic-monuments.ts | 13 +++-
.../tests/ingest-la-historic-monuments.test.cjs | 61 ++++++++++++++-
verification/e2e-proof.json | 90 +++++++++++++---------
verification/hcm-feature-query-tests.txt | 37 +++++++++
verification/hcm-layer-identity-e2e-proof.json | 74 ++++++++++++++++++
5 files changed, 231 insertions(+), 44 deletions(-)
diff --git a/scripts/ingest-la-historic-monuments.ts b/scripts/ingest-la-historic-monuments.ts
index 717b8c8..4ff176d 100644
--- a/scripts/ingest-la-historic-monuments.ts
+++ b/scripts/ingest-la-historic-monuments.ts
@@ -46,8 +46,17 @@ async function fetchPage(offset: number) {
u.searchParams.set('orderByFields', 'OBJECTID');
const res = await fetch(u);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
- const j = await res.json() as { features?: { attributes: any; geometry?: { x: number; y: number } }[] };
- return j.features ?? [];
+ const j = await res.json();
+ if (!j || typeof j !== 'object' || Array.isArray(j)) {
+ throw new Error(`Invalid HCM feature page at offset ${offset}: expected an object`);
+ }
+ if ('error' in j) {
+ throw new Error(`HCM feature query returned an ArcGIS error at offset ${offset}`);
+ }
+ if (!Array.isArray(j.features)) {
+ throw new Error(`Invalid HCM feature page at offset ${offset}: expected a features array`);
+ }
+ return j.features as { attributes: any; geometry?: { x: number; y: number } }[];
}
async function ensureSchema() {
diff --git a/scripts/tests/ingest-la-historic-monuments.test.cjs b/scripts/tests/ingest-la-historic-monuments.test.cjs
index 0e91c01..e57e5a4 100644
--- a/scripts/tests/ingest-la-historic-monuments.test.cjs
+++ b/scripts/tests/ingest-la-historic-monuments.test.cjs
@@ -13,7 +13,7 @@ const executable = stripTypeScriptTypes(source.replace("import { Pool } from 'pg
const valid = { name: 'Historic-Cultural Monuments', type: 'Feature Layer' };
async function run({ metadata = valid, metadataStatus = 200, jsonThrows = false, fetchThrows = false, preflight = false, pages = [{ features: [] }], queryThrows = false } = {}) {
- const calls = [], errors = [];
+ const calls = [], errors = [], logs = [];
let exitCode = 0;
class TestPool {
constructor() { calls.push({ boundary: 'pg-create' }); }
@@ -27,7 +27,7 @@ async function run({ metadata = valid, metadataStatus = 200, jsonThrows = false,
const context = {
TestPool, URL, Date,
process: { env: {}, argv: preflight ? ['node', 'importer', '--preflight-only'] : ['node', 'importer'], exit(code) { exitCode = code; } },
- console: { log() {}, error(...args) { errors.push(args.map(String).join(' ')); } },
+ console: { log(...args) { logs.push(args.map(String).join(' ')); }, error(...args) { errors.push(args.map(String).join(' ')); } },
async fetch(url) {
const target = String(url);
calls.push({ boundary: 'fetch', url: target });
@@ -37,11 +37,12 @@ async function run({ metadata = valid, metadataStatus = 200, jsonThrows = false,
}
assert.match(target, /MapServer\/75\/query\?/);
assert.equal(new URL(target).searchParams.get('orderByFields'), 'OBJECTID');
- return { ok: true, async json() { return pages.shift() ?? { features: [] }; } };
+ assert.ok(pages.length, 'unexpected feature page request');
+ return { ok: true, status: 200, async json() { return pages.shift(); } };
},
};
await vm.runInNewContext(executable, context);
- return { calls, errors, exitCode };
+ return { calls, errors, logs, exitCode };
}
for (const [name, options] of [
@@ -81,12 +82,15 @@ test('valid metadata precedes pool, page query, listing/event writes and pool cl
assert.equal(event.values[0], 'test-listing-id');
assert.match(event.values[1], /Test Monument/);
assert.equal(event.values[4], 'hcm:123');
+ assert.ok(result.logs.includes('✓ HCM: 1 monuments tied to listings'));
+ assert.equal(result.errors.length, 0);
});
test('DB failure closes pool and reports failure', async () => {
const result = await run({ queryThrows: true, pages: [{ features: [{ attributes: { NAME: 'Test', LOCATION: '200 Columbia Avenue', OBJECTID: 123 } }] }] });
assert.equal(result.exitCode, 1);
assert.equal(result.calls.at(-1).boundary, 'pg-end');
+ assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
});
test('repeat preflight independently rechecks identity', async () => {
@@ -96,3 +100,52 @@ test('repeat preflight independently rechecks identity', async () => {
assert.equal(second.exitCode, 1);
assert.equal(first.calls.length + second.calls.length, 2);
});
+
+for (const [name, page, message] of [
+ ['ArcGIS error', { error: { code: 499, message: 'Token Required' } }, /ArcGIS error/],
+ ['ArcGIS error with features', { error: { code: 499 }, features: [] }, /ArcGIS error/],
+ ['null container', null, /Invalid HCM feature page/],
+ ['array container', [], /Invalid HCM feature page/],
+ ['string container', 'bad page', /Invalid HCM feature page/],
+ ['missing features', {}, /Invalid HCM feature page/],
+ ['null features', { features: null }, /Invalid HCM feature page/],
+ ['object features', { features: {} }, /Invalid HCM feature page/],
+ ['string features', { features: 'bad features' }, /Invalid HCM feature page/],
+]) {
+ test(`${name}: fails without writes or success and closes pool once`, async () => {
+ const result = await run({ pages: [page] });
+ assert.equal(result.exitCode, 1);
+ assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-end']);
+ assert.equal(result.errors.length, 1);
+ assert.match(result.errors[0], message);
+ assert.match(result.errors[0], /offset 0/);
+ assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
+ });
+}
+
+test('valid empty features succeeds without writes and closes pool once', async () => {
+ const result = await run();
+ assert.equal(result.exitCode, 0);
+ assert.equal(result.errors.length, 0);
+ assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-end']);
+ assert.ok(result.logs.includes('✓ HCM: 0 monuments tied to listings'));
+});
+
+for (const [name, secondPage] of [
+ ['ArcGIS error', { error: { code: 499 } }],
+ ['malformed page', { features: null }],
+]) {
+ test(`later ${name}: preserves earlier processing, fails without final success and closes pool`, async () => {
+ const feature = { attributes: { NAME: 'Test Monument', LOCATION: '200 Columbia Avenue', OBJECTID: 123 } };
+ const result = await run({ pages: [{ features: Array.from({ length: 1000 }, () => feature) }, secondPage] });
+ assert.equal(result.exitCode, 1);
+ const pages = result.calls.filter(x => x.boundary === 'fetch' && x.url.includes('/query?'));
+ assert.deepEqual(pages.map(x => new URL(x.url).searchParams.get('resultOffset')), ['0', '1000']);
+ assert.equal(result.calls.filter(x => x.boundary === 'pg-query').length, 3000);
+ assert.equal(result.calls.filter(x => x.boundary === 'pg-end').length, 1);
+ assert.equal(result.calls.at(-1).boundary, 'pg-end');
+ assert.equal(result.errors.length, 1);
+ assert.match(result.errors[0], /offset 1000/);
+ assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
+ });
+}
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
index 8b9537f..20e3ad8 100644
--- a/verification/e2e-proof.json
+++ b/verification/e2e-proof.json
@@ -1,74 +1,88 @@
{
"schema_version": 1,
- "ticket": "TK-11379-stayclaim-ingest-la-historic-monuments-p",
- "correlation_id": "cycle1620-stayclaim-receipt",
- "timestamp": "2026-09-10T16:27:42.862134+00:00",
- "intent": "Correct dormant HCM importer layer and fail closed before PG or feature queries on metadata mismatch",
- "risk_tier": "R1 isolated code with read-only external metadata boundary",
- "environment": "macstudio3; Node v26.4.0; actual TypeScript source stripped using node:module in VM; PG constructor/query mocked, no real DB driver imported",
+ "ticket": "TK-11399-stayclaim-arcgis-feature-query-errors-si",
+ "correlation_id": "cycle1650-cJ8lZ4-TK11399",
+ "receipt_dm": "M-02741",
+ "timestamp": "2026-09-10T16:59:33.951398+00:00",
+ "intent": "Reject ArcGIS feature-query errors and malformed page/features containers before reporting a successful import; retain valid empty success and pool cleanup.",
+ "risk_tier": "R1 isolated code; external and database boundaries fully substituted",
+ "environment": "macstudio3; actual TypeScript script entrypoint run through Node stripTypeScriptTypes and VM; no real pg driver or network available in test context",
"baseline": {
- "commit": "6449ef37998d48405c3d7821b72788704b4e4779",
- "working_tree": "clean before owned changes",
- "defect": "hardcoded layer74 now Hollywood Walk of Fame"
+ "commit": "3881fa9f9a44020a4f1d86905a9132b9e7986956",
+ "worktree": "clean before owned edits",
+ "defect": "HTTP 200 ArcGIS error treated as empty success",
+ "regression_before_fix": {
+ "passed": 16,
+ "failed": 11,
+ "artifact": "/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260910T1650Z.cJ8lZ4/child-tests-red.tap"
+ }
},
"build_identity": {
- "source_sha256": "ba9932eb85eef8397841b8d0c16f2220590dd303551e756c9a540de7108a53e2"
+ "sha256": {
+ "scripts/ingest-la-historic-monuments.ts": "f3b10c4b7e98fd66fa96c02a4165af0782b9de099b006077248140d7e5ccc153",
+ "scripts/tests/ingest-la-historic-monuments.test.cjs": "68394678d1866d502e2dd292b2aabdd8a90182f111a5c968070b287756c746c4"
+ }
},
"commands": [
"node --test scripts/tests/ingest-la-historic-monuments.test.cjs",
- "node scripts/tests/hcm-live-preflight.cjs",
"git diff --check"
],
"checks": [
{
- "name": "actual importer operational boundary tests",
+ "name": "Actual script entrypoint targeted suite",
"verdict": "PASS",
- "count": 15,
- "artifact": "verification/hcm-operational-tests.txt"
+ "passed": 27,
+ "failed": 0,
+ "artifact": "verification/hcm-feature-query-tests.txt"
},
{
- "name": "real uncached metadata GET using actual importer --preflight-only",
+ "name": "HTTP 200 ArcGIS error with and without features",
"verdict": "PASS",
- "artifact": "verification/hcm-live-preflight.json"
+ "detail": "exit 1, diagnostic with offset, zero SQL, pool.end once, no final success"
},
{
- "name": "correct live field names required by existing importer",
+ "name": "Null/array/string page, absent/null/object/string features",
"verdict": "PASS",
- "detail": "metadata includes NAME LOCATION DATE_ACTIVE OBJECTID MNT_NUM HIST_TYPE NLA_URL"
+ "detail": "fail closed before SQL, exit 1, pool.end once, no final success"
},
{
- "name": "no PG constructor or feature query reached on 11 failure classes or valid preflight",
- "verdict": "PASS"
+ "name": "Valid empty page",
+ "verdict": "PASS",
+ "detail": "exit 0, zero SQL, final zero-row success, pool.end once"
},
{
- "name": "valid metadata permits expected stub listing lookup/insert/event insert and pool close",
- "verdict": "PASS"
+ "name": "Normal row",
+ "verdict": "PASS",
+ "detail": "listing lookup/create and designation insert argument assertions; exit 0; one-row success; pool closed"
},
{
- "name": "simulated DB failure closes pool",
- "verdict": "PASS"
+ "name": "ArcGIS or malformed page after 1000 successful rows",
+ "verdict": "PASS",
+ "detail": "offsets 0 and 1000, 3000 simulated SQL operations before failure, exit 1, pool.end once, no final success"
},
{
- "name": "repeat preflight rechecks changed identity",
- "verdict": "PASS"
+ "name": "Metadata/preflight and DB failure regressions",
+ "verdict": "PASS",
+ "detail": "existing 15 tests preserved; DB failure also suppresses final success"
},
{
- "name": "diff whitespace",
+ "name": "Diff whitespace",
"verdict": "PASS"
},
{
- "name": "real DB ingestion and production persisted state",
+ "name": "Live ingestion / production persisted-state checks",
"verdict": "SKIP",
- "reason": "Outside local increment; explicitly prohibited, needs exact approval and separate canary. No ingestion readiness claim."
- },
- {
- "name": "full project typecheck/build",
- "verdict": "SKIP",
- "reason": "Dependencies absent; no installation or web boundary changes."
+ "reason": "Explicitly outside authorized local fix; no real network/DB calls or writes allowed. No production readiness claim."
}
],
- "cleanup": "No DB or production state created. Evidence retained; rollback local commit if needed, never reset unrelated work.",
- "verdict": "PASS for narrow local layer identity repair; real ingestion unverified and gated",
- "approval_required": "Any real ingestion, production DB writes, deploy/schedule remain separately gated",
- "parent_acceptance": "pending independent verifier/Cody/final DTD"
+ "negative_checks": "ArcGIS error takes precedence even alongside valid empty features; malformed null page no longer hidden by test fixture fallback.",
+ "cleanup": "No real external state created. VM state discarded; logs retained. Prior layer proof preserved in verification/hcm-layer-identity-e2e-proof.json.",
+ "rollback": "Revert the scoped local commit if necessary; do not reset unrelated work.",
+ "limitations": [
+ "Earlier valid pages may remain persisted after a later failure; atomic import/rollback remains out of scope.",
+ "Feature field validation, transaction semantics and full live ingestion remain unchanged."
+ ],
+ "verdict": "PASS for local feature-query failure handling; parent independent acceptance pending",
+ "approval_required": "Any live ingestion, database writes, deployment or schedule installation remains gated.",
+ "parent_acceptance": "pending parent CLI proof, Cody, final DTD"
}
diff --git a/verification/hcm-feature-query-tests.txt b/verification/hcm-feature-query-tests.txt
new file mode 100644
index 0000000..a648760
--- /dev/null
+++ b/verification/hcm-feature-query-tests.txt
@@ -0,0 +1,37 @@
+(node:24732) ExperimentalWarning: stripTypeScriptTypes is an experimental feature and might change at any time
+(Use `node --trace-warnings ...` to show where the warning was created)
+✔ wrong name (Walk of Fame): aborts before pool construction, DB and feature query (6.013334ms)
+✔ same-name group layer: aborts before pool construction, DB and feature query (1.658792ms)
+✔ trailing-space group trap: aborts before pool construction, DB and feature query (0.885875ms)
+✔ trailing-space feature name: aborts before pool construction, DB and feature query (1.359333ms)
+✔ HTTP failure: aborts before pool construction, DB and feature query (0.854584ms)
+✔ ArcGIS error JSON: aborts before pool construction, DB and feature query (1.2895ms)
+✔ malformed JSON: aborts before pool construction, DB and feature query (0.889125ms)
+✔ missing fields: aborts before pool construction, DB and feature query (0.784042ms)
+✔ null metadata: aborts before pool construction, DB and feature query (0.977458ms)
+✔ array metadata: aborts before pool construction, DB and feature query (1.967583ms)
+✔ network failure: aborts before pool construction, DB and feature query (1.598625ms)
+✔ valid preflight exits without pool or feature query (0.873667ms)
+✔ valid metadata precedes pool, page query, listing/event writes and pool close (5.159959ms)
+✔ DB failure closes pool and reports failure (3.216125ms)
+✔ repeat preflight independently rechecks identity (8.338334ms)
+✔ ArcGIS error: fails without writes or success and closes pool once (1.84525ms)
+✔ ArcGIS error with features: fails without writes or success and closes pool once (2.038375ms)
+✔ null container: fails without writes or success and closes pool once (1.8055ms)
+✔ array container: fails without writes or success and closes pool once (1.678166ms)
+✔ string container: fails without writes or success and closes pool once (1.241625ms)
+✔ missing features: fails without writes or success and closes pool once (0.969167ms)
+✔ null features: fails without writes or success and closes pool once (1.748625ms)
+✔ object features: fails without writes or success and closes pool once (1.88975ms)
+✔ string features: fails without writes or success and closes pool once (1.787208ms)
+✔ valid empty features succeeds without writes and closes pool once (3.562833ms)
+✔ later ArcGIS error: preserves earlier processing, fails without final success and closes pool (38.287125ms)
+✔ later malformed page: preserves earlier processing, fails without final success and closes pool (57.138083ms)
+ℹ tests 27
+ℹ suites 0
+ℹ pass 27
+ℹ fail 0
+ℹ cancelled 0
+ℹ skipped 0
+ℹ todo 0
+ℹ duration_ms 450.7035
diff --git a/verification/hcm-layer-identity-e2e-proof.json b/verification/hcm-layer-identity-e2e-proof.json
new file mode 100644
index 0000000..8b9537f
--- /dev/null
+++ b/verification/hcm-layer-identity-e2e-proof.json
@@ -0,0 +1,74 @@
+{
+ "schema_version": 1,
+ "ticket": "TK-11379-stayclaim-ingest-la-historic-monuments-p",
+ "correlation_id": "cycle1620-stayclaim-receipt",
+ "timestamp": "2026-09-10T16:27:42.862134+00:00",
+ "intent": "Correct dormant HCM importer layer and fail closed before PG or feature queries on metadata mismatch",
+ "risk_tier": "R1 isolated code with read-only external metadata boundary",
+ "environment": "macstudio3; Node v26.4.0; actual TypeScript source stripped using node:module in VM; PG constructor/query mocked, no real DB driver imported",
+ "baseline": {
+ "commit": "6449ef37998d48405c3d7821b72788704b4e4779",
+ "working_tree": "clean before owned changes",
+ "defect": "hardcoded layer74 now Hollywood Walk of Fame"
+ },
+ "build_identity": {
+ "source_sha256": "ba9932eb85eef8397841b8d0c16f2220590dd303551e756c9a540de7108a53e2"
+ },
+ "commands": [
+ "node --test scripts/tests/ingest-la-historic-monuments.test.cjs",
+ "node scripts/tests/hcm-live-preflight.cjs",
+ "git diff --check"
+ ],
+ "checks": [
+ {
+ "name": "actual importer operational boundary tests",
+ "verdict": "PASS",
+ "count": 15,
+ "artifact": "verification/hcm-operational-tests.txt"
+ },
+ {
+ "name": "real uncached metadata GET using actual importer --preflight-only",
+ "verdict": "PASS",
+ "artifact": "verification/hcm-live-preflight.json"
+ },
+ {
+ "name": "correct live field names required by existing importer",
+ "verdict": "PASS",
+ "detail": "metadata includes NAME LOCATION DATE_ACTIVE OBJECTID MNT_NUM HIST_TYPE NLA_URL"
+ },
+ {
+ "name": "no PG constructor or feature query reached on 11 failure classes or valid preflight",
+ "verdict": "PASS"
+ },
+ {
+ "name": "valid metadata permits expected stub listing lookup/insert/event insert and pool close",
+ "verdict": "PASS"
+ },
+ {
+ "name": "simulated DB failure closes pool",
+ "verdict": "PASS"
+ },
+ {
+ "name": "repeat preflight rechecks changed identity",
+ "verdict": "PASS"
+ },
+ {
+ "name": "diff whitespace",
+ "verdict": "PASS"
+ },
+ {
+ "name": "real DB ingestion and production persisted state",
+ "verdict": "SKIP",
+ "reason": "Outside local increment; explicitly prohibited, needs exact approval and separate canary. No ingestion readiness claim."
+ },
+ {
+ "name": "full project typecheck/build",
+ "verdict": "SKIP",
+ "reason": "Dependencies absent; no installation or web boundary changes."
+ }
+ ],
+ "cleanup": "No DB or production state created. Evidence retained; rollback local commit if needed, never reset unrelated work.",
+ "verdict": "PASS for narrow local layer identity repair; real ingestion unverified and gated",
+ "approval_required": "Any real ingestion, production DB writes, deploy/schedule remain separately gated",
+ "parent_acceptance": "pending independent verifier/Cody/final DTD"
+}
← 3881fa9 Verify historic monument layer identity before importing
·
back to Stayclaim
·
(newest)