← back to Stayclaim
Verify historic monument layer identity before importing
3881fa9f9a44020a4f1d86905a9132b9e7986956 · 2026-09-10 09:27:44 -0700 · Steve Abrams
Files touched
M scripts/ingest-la-historic-monuments.tsA scripts/tests/hcm-live-preflight.cjsA scripts/tests/ingest-la-historic-monuments.test.cjsA verification/e2e-proof.jsonA verification/hcm-live-preflight.jsonA verification/hcm-operational-tests.txt
Diff
commit 3881fa9f9a44020a4f1d86905a9132b9e7986956
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 09:27:44 2026 -0700
Verify historic monument layer identity before importing
---
scripts/ingest-la-historic-monuments.ts | 84 +++++++++++--------
scripts/tests/hcm-live-preflight.cjs | 30 +++++++
.../tests/ingest-la-historic-monuments.test.cjs | 98 ++++++++++++++++++++++
verification/e2e-proof.json | 74 ++++++++++++++++
verification/hcm-live-preflight.json | 36 ++++++++
verification/hcm-operational-tests.txt | 25 ++++++
6 files changed, 313 insertions(+), 34 deletions(-)
diff --git a/scripts/ingest-la-historic-monuments.ts b/scripts/ingest-la-historic-monuments.ts
index cce8e59..717b8c8 100644
--- a/scripts/ingest-la-historic-monuments.ts
+++ b/scripts/ingest-la-historic-monuments.ts
@@ -1,8 +1,9 @@
/**
* ingest-la-historic-monuments.ts
*
- * LA City Historic-Cultural Monuments via NavigateLA layer 74.
- * Verified 2026-04-30: 7,836 monuments + survey records.
+ * LA City Historic-Cultural Monuments via NavigateLA layer 75.
+ * Layer identity reverified 2026-09-10 after NavigateLA's layer re-index.
+ * Run with --preflight-only to verify metadata without opening PostgreSQL.
*
* Each monument gets:
* - A listing (matched to existing if address matches LADBS data, else stub)
@@ -13,24 +14,28 @@
*/
import { Pool } from 'pg';
-const pool = new Pool({
- host: process.env.PGHOST ?? '/tmp',
- database: process.env.PGDATABASE ?? 'stayclaim',
- user: process.env.PGUSER ?? process.env.USER,
- password: process.env.PGPASSWORD,
- port: parseInt(process.env.PGPORT ?? '5432', 10),
- max: 4,
-});
-
-const URL = 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/74/query';
+let pool: Pool;
+const LAYER_URL = 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75';
+const EXPECTED_LAYER_NAME = 'Historic-Cultural Monuments';
const PAGE = 1000;
+async function assertLayerIdentity() {
+ const res = await fetch(`${LAYER_URL}?f=json`);
+ if (!res.ok) throw new Error(`HCM layer metadata HTTP ${res.status}`);
+ const metadata = await res.json();
+ if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata) || metadata.error ||
+ metadata.name !== EXPECTED_LAYER_NAME || metadata.type !== 'Feature Layer') {
+ throw new Error(`HCM layer identity mismatch at ${LAYER_URL}: expected ${EXPECTED_LAYER_NAME} / Feature Layer`);
+ }
+ console.log(`Verified HCM layer: ${metadata.name} / ${metadata.type} (${LAYER_URL})`);
+}
+
function canonicalize(addr: string): string {
return addr.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '').slice(0, 90);
}
async function fetchPage(offset: number) {
- const u = new URL_(URL);
+ const u = new URL(`${LAYER_URL}/query`);
u.searchParams.set('where', '1=1');
u.searchParams.set('outFields', '*');
u.searchParams.set('returnGeometry', 'true');
@@ -45,9 +50,6 @@ async function fetchPage(offset: number) {
return j.features ?? [];
}
-// Workaround: the URL constant shadows the global URL constructor inside fetchPage above.
-const URL_ = globalThis.URL;
-
async function ensureSchema() {
// Use existing place_event table for the historic designation event
// (kind='historic_designation', source_label='LA City Office of Historic Resources').
@@ -115,26 +117,40 @@ async function upsertOne(f: any) {
}
async function main() {
- await ensureSchema();
- let offset = 0, total = 0, ok = 0, skipped = 0;
- const t0 = Date.now();
- while (true) {
- const features = await fetchPage(offset);
- if (!features.length) break;
- for (const f of features) {
- const r = await upsertOne(f);
- if (r.ok) ok++;
- else skipped++;
+ // Fail closed before creating a pool or issuing any feature query/import work.
+ await assertLayerIdentity();
+ if (process.argv.includes('--preflight-only')) return;
+ pool = new Pool({
+ host: process.env.PGHOST ?? '/tmp',
+ database: process.env.PGDATABASE ?? 'stayclaim',
+ user: process.env.PGUSER ?? process.env.USER,
+ password: process.env.PGPASSWORD,
+ port: parseInt(process.env.PGPORT ?? '5432', 10),
+ max: 4,
+ });
+ try {
+ await ensureSchema();
+ let offset = 0, total = 0, ok = 0, skipped = 0;
+ const t0 = Date.now();
+ while (true) {
+ const features = await fetchPage(offset);
+ if (!features.length) break;
+ for (const f of features) {
+ const r = await upsertOne(f);
+ if (r.ok) ok++;
+ else skipped++;
+ }
+ total += features.length;
+ const dt = (Date.now() - t0) / 1000;
+ console.log(` hcm: ${total} processed (${ok} ok, ${skipped} skipped) — ${(total/dt).toFixed(0)}/s`);
+ if (features.length < PAGE) break;
+ offset += features.length;
+ if (offset > 50000) break;
}
- total += features.length;
- const dt = (Date.now() - t0) / 1000;
- console.log(` hcm: ${total} processed (${ok} ok, ${skipped} skipped) — ${(total/dt).toFixed(0)}/s`);
- if (features.length < PAGE) break;
- offset += features.length;
- if (offset > 50000) break;
+ console.log(`✓ HCM: ${ok} monuments tied to listings`);
+ } finally {
+ await pool.end();
}
- console.log(`✓ HCM: ${ok} monuments tied to listings`);
- await pool.end();
}
main().catch(e => { console.error('FATAL', e); process.exit(1); });
diff --git a/scripts/tests/hcm-live-preflight.cjs b/scripts/tests/hcm-live-preflight.cjs
new file mode 100644
index 0000000..8672529
--- /dev/null
+++ b/scripts/tests/hcm-live-preflight.cjs
@@ -0,0 +1,30 @@
+// Read-only operational proof: execute the actual importer against fresh public
+// metadata. PG is replaced with a constructor that throws if reached.
+const assert = require('node:assert/strict');
+const { readFileSync } = require('node:fs');
+const { stripTypeScriptTypes } = require('node:module');
+const vm = require('node:vm');
+const path = require('node:path');
+const source = readFileSync(path.join(__dirname, '../ingest-la-historic-monuments.ts'), 'utf8');
+assert.equal((source.match(/import \{ Pool \} from 'pg';/g) || []).length, 1);
+const executable = stripTypeScriptTypes(source.replace("import { Pool } from 'pg';", 'const Pool = globalThis.TestPool;'));
+const evidence = { timestamp: new Date().toISOString(), ticket: 'TK-11379', mode: 'actual importer preflight / real public GET / forbidden PG', calls: [], logs: [], errors: [], exit_code: 0 };
+const context = {
+ TestPool: class { constructor() { evidence.calls.push({ boundary: 'pg-create' }); throw new Error('PG forbidden in preflight'); } },
+ URL, Date,
+ process: { env: {}, argv: ['node', 'importer', '--preflight-only'], exit(code) { evidence.exit_code = code; } },
+ console: { log(...args) { evidence.logs.push(args.map(String).join(' ')); }, error(...args) { evidence.errors.push(args.map(String).join(' ')); } },
+ async fetch(url) {
+ assert.equal(String(url), 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75?f=json');
+ const response = await fetch(url, { cache: 'no-store', signal: AbortSignal.timeout(30000) });
+ const metadata = await response.json();
+ evidence.calls.push({ boundary: 'public-metadata-get', url: String(url), status: response.status, name: metadata?.name, type: metadata?.type, fields: metadata?.fields?.map(f => f.name) });
+ return { ok: response.ok, status: response.status, async json() { return metadata; } };
+ },
+};
+(async () => {
+ await vm.runInNewContext(executable, context);
+ evidence.verdict = evidence.exit_code === 0 && evidence.calls.length === 1 && evidence.calls[0].boundary === 'public-metadata-get' ? 'PASS' : 'FAIL';
+ console.log(JSON.stringify(evidence, null, 2));
+ process.exitCode = evidence.verdict === 'PASS' ? 0 : 1;
+})().catch(error => { console.error(error); process.exitCode = 1; });
diff --git a/scripts/tests/ingest-la-historic-monuments.test.cjs b/scripts/tests/ingest-la-historic-monuments.test.cjs
new file mode 100644
index 0000000..0e91c01
--- /dev/null
+++ b/scripts/tests/ingest-la-historic-monuments.test.cjs
@@ -0,0 +1,98 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const { readFileSync } = require('node:fs');
+const { stripTypeScriptTypes } = require('node:module');
+const vm = require('node:vm');
+const path = require('node:path');
+
+// Exercise the actual one-shot entry point. Only external PG/fetch boundaries are
+// replaced: no database driver is imported and no network call escapes the VM.
+const source = readFileSync(path.join(__dirname, '../ingest-la-historic-monuments.ts'), 'utf8');
+assert.equal((source.match(/import \{ Pool \} from 'pg';/g) || []).length, 1);
+const executable = stripTypeScriptTypes(source.replace("import { Pool } from 'pg';", 'const Pool = globalThis.TestPool;'));
+const valid = { name: 'Historic-Cultural Monuments', type: 'Feature Layer' };
+
+async function run({ metadata = valid, metadataStatus = 200, jsonThrows = false, fetchThrows = false, preflight = false, pages = [{ features: [] }], queryThrows = false } = {}) {
+ const calls = [], errors = [];
+ let exitCode = 0;
+ class TestPool {
+ constructor() { calls.push({ boundary: 'pg-create' }); }
+ async query(sql, values) {
+ calls.push({ boundary: 'pg-query', sql, values });
+ if (queryThrows) throw new Error('simulated DB failure');
+ return { rows: sql.startsWith('INSERT INTO listing') ? [{ id: 'test-listing-id' }] : [] };
+ }
+ async end() { calls.push({ boundary: 'pg-end' }); }
+ }
+ const context = {
+ TestPool, URL, Date,
+ process: { env: {}, argv: preflight ? ['node', 'importer', '--preflight-only'] : ['node', 'importer'], exit(code) { exitCode = code; } },
+ console: { log() {}, error(...args) { errors.push(args.map(String).join(' ')); } },
+ async fetch(url) {
+ const target = String(url);
+ calls.push({ boundary: 'fetch', url: target });
+ if (!target.includes('/query?')) {
+ if (fetchThrows) throw new Error('simulated network failure');
+ return { ok: metadataStatus === 200, status: metadataStatus, async json() { if (jsonThrows) throw new SyntaxError('invalid JSON'); return metadata; } };
+ }
+ assert.match(target, /MapServer\/75\/query\?/);
+ assert.equal(new URL(target).searchParams.get('orderByFields'), 'OBJECTID');
+ return { ok: true, async json() { return pages.shift() ?? { features: [] }; } };
+ },
+ };
+ await vm.runInNewContext(executable, context);
+ return { calls, errors, exitCode };
+}
+
+for (const [name, options] of [
+ ['wrong name (Walk of Fame)', { metadata: { ...valid, name: 'Hollywood Walk of Fame' } }],
+ ['same-name group layer', { metadata: { ...valid, type: 'Group Layer' } }],
+ ['trailing-space group trap', { metadata: { name: 'Historic-Cultural Monuments ', type: 'Group Layer' } }],
+ ['trailing-space feature name', { metadata: { ...valid, name: `${valid.name} ` } }],
+ ['HTTP failure', { metadataStatus: 503 }],
+ ['ArcGIS error JSON', { metadata: { ...valid, error: { code: 499 } } }],
+ ['malformed JSON', { jsonThrows: true }],
+ ['missing fields', { metadata: {} }],
+ ['null metadata', { metadata: null }],
+ ['array metadata', { metadata: [] }],
+ ['network failure', { fetchThrows: true }],
+]) {
+ test(`${name}: aborts before pool construction, DB and feature query`, async () => {
+ const result = await run(options);
+ assert.equal(result.exitCode, 1);
+ assert.equal(result.calls.length, 1);
+ assert.equal(result.calls[0].url, 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75?f=json');
+ assert.equal(result.errors.length, 1);
+ });
+}
+
+test('valid preflight exits without pool or feature query', async () => {
+ const result = await run({ preflight: true });
+ assert.equal(result.exitCode, 0);
+ assert.equal(result.calls.length, 1);
+});
+
+test('valid metadata precedes pool, page query, listing/event writes and pool close', async () => {
+ const result = await run({ pages: [{ features: [{ attributes: { NAME: 'Test Monument', LOCATION: '200-240 Columbia Avenue', OBJECTID: 123, MNT_NUM: 10 }, geometry: { x: -118.26, y: 34.06 } }] }] });
+ assert.equal(result.exitCode, 0);
+ assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-query', 'pg-query', 'pg-query', 'pg-end']);
+ const event = result.calls[5];
+ assert.match(event.sql, /INSERT INTO place_event/);
+ assert.equal(event.values[0], 'test-listing-id');
+ assert.match(event.values[1], /Test Monument/);
+ assert.equal(event.values[4], 'hcm:123');
+});
+
+test('DB failure closes pool and reports failure', async () => {
+ const result = await run({ queryThrows: true, pages: [{ features: [{ attributes: { NAME: 'Test', LOCATION: '200 Columbia Avenue', OBJECTID: 123 } }] }] });
+ assert.equal(result.exitCode, 1);
+ assert.equal(result.calls.at(-1).boundary, 'pg-end');
+});
+
+test('repeat preflight independently rechecks identity', async () => {
+ const first = await run({ preflight: true });
+ const second = await run({ preflight: true, metadata: { ...valid, name: 'Hollywood Walk of Fame' } });
+ assert.equal(first.exitCode, 0);
+ assert.equal(second.exitCode, 1);
+ assert.equal(first.calls.length + second.calls.length, 2);
+});
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
new file mode 100644
index 0000000..8b9537f
--- /dev/null
+++ b/verification/e2e-proof.json
@@ -0,0 +1,74 @@
+{
+ "schema_version": 1,
+ "ticket": "TK-11379-stayclaim-ingest-la-historic-monuments-p",
+ "correlation_id": "cycle1620-stayclaim-receipt",
+ "timestamp": "2026-09-10T16:27:42.862134+00:00",
+ "intent": "Correct dormant HCM importer layer and fail closed before PG or feature queries on metadata mismatch",
+ "risk_tier": "R1 isolated code with read-only external metadata boundary",
+ "environment": "macstudio3; Node v26.4.0; actual TypeScript source stripped using node:module in VM; PG constructor/query mocked, no real DB driver imported",
+ "baseline": {
+ "commit": "6449ef37998d48405c3d7821b72788704b4e4779",
+ "working_tree": "clean before owned changes",
+ "defect": "hardcoded layer74 now Hollywood Walk of Fame"
+ },
+ "build_identity": {
+ "source_sha256": "ba9932eb85eef8397841b8d0c16f2220590dd303551e756c9a540de7108a53e2"
+ },
+ "commands": [
+ "node --test scripts/tests/ingest-la-historic-monuments.test.cjs",
+ "node scripts/tests/hcm-live-preflight.cjs",
+ "git diff --check"
+ ],
+ "checks": [
+ {
+ "name": "actual importer operational boundary tests",
+ "verdict": "PASS",
+ "count": 15,
+ "artifact": "verification/hcm-operational-tests.txt"
+ },
+ {
+ "name": "real uncached metadata GET using actual importer --preflight-only",
+ "verdict": "PASS",
+ "artifact": "verification/hcm-live-preflight.json"
+ },
+ {
+ "name": "correct live field names required by existing importer",
+ "verdict": "PASS",
+ "detail": "metadata includes NAME LOCATION DATE_ACTIVE OBJECTID MNT_NUM HIST_TYPE NLA_URL"
+ },
+ {
+ "name": "no PG constructor or feature query reached on 11 failure classes or valid preflight",
+ "verdict": "PASS"
+ },
+ {
+ "name": "valid metadata permits expected stub listing lookup/insert/event insert and pool close",
+ "verdict": "PASS"
+ },
+ {
+ "name": "simulated DB failure closes pool",
+ "verdict": "PASS"
+ },
+ {
+ "name": "repeat preflight rechecks changed identity",
+ "verdict": "PASS"
+ },
+ {
+ "name": "diff whitespace",
+ "verdict": "PASS"
+ },
+ {
+ "name": "real DB ingestion and production persisted state",
+ "verdict": "SKIP",
+ "reason": "Outside local increment; explicitly prohibited, needs exact approval and separate canary. No ingestion readiness claim."
+ },
+ {
+ "name": "full project typecheck/build",
+ "verdict": "SKIP",
+ "reason": "Dependencies absent; no installation or web boundary changes."
+ }
+ ],
+ "cleanup": "No DB or production state created. Evidence retained; rollback local commit if needed, never reset unrelated work.",
+ "verdict": "PASS for narrow local layer identity repair; real ingestion unverified and gated",
+ "approval_required": "Any real ingestion, production DB writes, deploy/schedule remain separately gated",
+ "parent_acceptance": "pending independent verifier/Cody/final DTD"
+}
diff --git a/verification/hcm-live-preflight.json b/verification/hcm-live-preflight.json
new file mode 100644
index 0000000..b485838
--- /dev/null
+++ b/verification/hcm-live-preflight.json
@@ -0,0 +1,36 @@
+{
+ "timestamp": "2026-09-10T16:27:01.033Z",
+ "ticket": "TK-11379",
+ "mode": "actual importer preflight / real public GET / forbidden PG",
+ "calls": [
+ {
+ "boundary": "public-metadata-get",
+ "url": "https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75?f=json",
+ "status": 200,
+ "name": "Historic-Cultural Monuments",
+ "type": "Feature Layer",
+ "fields": [
+ "Shape",
+ "HIST_TYPE",
+ "MNT_TYPE",
+ "MNT_NUM",
+ "NAME",
+ "LOCATION",
+ "DATE_ACTIVE",
+ "CONTRACT_NUM",
+ "CASE_NUM",
+ "HISTORIC_PRSV_ID",
+ "TOOLTIP",
+ "NLA_URL",
+ "OBJECTID",
+ "ESRI_OID"
+ ]
+ }
+ ],
+ "logs": [
+ "Verified HCM layer: Historic-Cultural Monuments / Feature Layer (https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75)"
+ ],
+ "errors": [],
+ "exit_code": 0,
+ "verdict": "PASS"
+}
diff --git a/verification/hcm-operational-tests.txt b/verification/hcm-operational-tests.txt
new file mode 100644
index 0000000..e8caffb
--- /dev/null
+++ b/verification/hcm-operational-tests.txt
@@ -0,0 +1,25 @@
+(node:9522) ExperimentalWarning: stripTypeScriptTypes is an experimental feature and might change at any time
+(Use `node --trace-warnings ...` to show where the warning was created)
+✔ wrong name (Walk of Fame): aborts before pool construction, DB and feature query (3.735875ms)
+✔ same-name group layer: aborts before pool construction, DB and feature query (1.4265ms)
+✔ trailing-space group trap: aborts before pool construction, DB and feature query (32.891709ms)
+✔ trailing-space feature name: aborts before pool construction, DB and feature query (1.037583ms)
+✔ HTTP failure: aborts before pool construction, DB and feature query (1.192416ms)
+✔ ArcGIS error JSON: aborts before pool construction, DB and feature query (1.116333ms)
+✔ malformed JSON: aborts before pool construction, DB and feature query (2.043208ms)
+✔ missing fields: aborts before pool construction, DB and feature query (1.772417ms)
+✔ null metadata: aborts before pool construction, DB and feature query (0.924042ms)
+✔ array metadata: aborts before pool construction, DB and feature query (1.694542ms)
+✔ network failure: aborts before pool construction, DB and feature query (0.746583ms)
+✔ valid preflight exits without pool or feature query (0.885333ms)
+✔ valid metadata precedes pool, page query, listing/event writes and pool close (4.102083ms)
+✔ DB failure closes pool and reports failure (1.77025ms)
+✔ repeat preflight independently rechecks identity (1.692959ms)
+ℹ tests 15
+ℹ suites 0
+ℹ pass 15
+ℹ fail 0
+ℹ cancelled 0
+ℹ skipped 0
+ℹ todo 0
+ℹ duration_ms 316.140667
← 6449ef3 add AdSense-compliant privacy policy (/privacy) + footer lin
·
back to Stayclaim
·
Fail historic monument imports on invalid feature pages a8d995e →