`scripts/tests/ingest-la-historic-monuments.test.cjs`

152 lines
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { readFileSync } = require('node:fs');
const { stripTypeScriptTypes } = require('node:module');
const vm = require('node:vm');
const path = require('node:path');

// Exercise the actual one-shot entry point. Only external PG/fetch boundaries are
// replaced: no database driver is imported and no network call escapes the VM.
const source = readFileSync(path.join(__dirname, '../ingest-la-historic-monuments.ts'), 'utf8');
assert.equal((source.match(/import \{ Pool \} from 'pg';/g) || []).length, 1);
const executable = stripTypeScriptTypes(source.replace("import { Pool } from 'pg';", 'const Pool = globalThis.TestPool;'));
const valid = { name: 'Historic-Cultural Monuments', type: 'Feature Layer' };

async function run({ metadata = valid, metadataStatus = 200, jsonThrows = false, fetchThrows = false, preflight = false, pages = [{ features: [] }], queryThrows = false } = {}) {
  const calls = [], errors = [], logs = [];
  let exitCode = 0;
  class TestPool {
    constructor() { calls.push({ boundary: 'pg-create' }); }
    async query(sql, values) {
      calls.push({ boundary: 'pg-query', sql, values });
      if (queryThrows) throw new Error('simulated DB failure');
      return { rows: sql.startsWith('INSERT INTO listing') ? [{ id: 'test-listing-id' }] : [] };
    }
    async end() { calls.push({ boundary: 'pg-end' }); }
  }
  const context = {
    TestPool, URL, Date,
    process: { env: {}, argv: preflight ? ['node', 'importer', '--preflight-only'] : ['node', 'importer'], exit(code) { exitCode = code; } },
    console: { log(...args) { logs.push(args.map(String).join(' ')); }, error(...args) { errors.push(args.map(String).join(' ')); } },
    async fetch(url) {
      const target = String(url);
      calls.push({ boundary: 'fetch', url: target });
      if (!target.includes('/query?')) {
        if (fetchThrows) throw new Error('simulated network failure');
        return { ok: metadataStatus === 200, status: metadataStatus, async json() { if (jsonThrows) throw new SyntaxError('invalid JSON'); return metadata; } };
      }
      assert.match(target, /MapServer\/75\/query\?/);
      assert.equal(new URL(target).searchParams.get('orderByFields'), 'OBJECTID');
      assert.ok(pages.length, 'unexpected feature page request');
      return { ok: true, status: 200, async json() { return pages.shift(); } };
    },
  };
  await vm.runInNewContext(executable, context);
  return { calls, errors, logs, exitCode };
}

for (const [name, options] of [
  ['wrong name (Walk of Fame)', { metadata: { ...valid, name: 'Hollywood Walk of Fame' } }],
  ['same-name group layer', { metadata: { ...valid, type: 'Group Layer' } }],
  ['trailing-space group trap', { metadata: { name: 'Historic-Cultural Monuments ', type: 'Group Layer' } }],
  ['trailing-space feature name', { metadata: { ...valid, name: `${valid.name} ` } }],
  ['HTTP failure', { metadataStatus: 503 }],
  ['ArcGIS error JSON', { metadata: { ...valid, error: { code: 499 } } }],
  ['malformed JSON', { jsonThrows: true }],
  ['missing fields', { metadata: {} }],
  ['null metadata', { metadata: null }],
  ['array metadata', { metadata: [] }],
  ['network failure', { fetchThrows: true }],
]) {
  test(`${name}: aborts before pool construction, DB and feature query`, async () => {
    const result = await run(options);
    assert.equal(result.exitCode, 1);
    assert.equal(result.calls.length, 1);
    assert.equal(result.calls[0].url, 'https://maps.lacity.org/arcgis/rest/services/Mapping/NavigateLA/MapServer/75?f=json');
    assert.equal(result.errors.length, 1);
  });
}

test('valid preflight exits without pool or feature query', async () => {
  const result = await run({ preflight: true });
  assert.equal(result.exitCode, 0);
  assert.equal(result.calls.length, 1);
});

test('valid metadata precedes pool, page query, listing/event writes and pool close', async () => {
  const result = await run({ pages: [{ features: [{ attributes: { NAME: 'Test Monument', LOCATION: '200-240 Columbia Avenue', OBJECTID: 123, MNT_NUM: 10 }, geometry: { x: -118.26, y: 34.06 } }] }] });
  assert.equal(result.exitCode, 0);
  assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-query', 'pg-query', 'pg-query', 'pg-end']);
  const event = result.calls[5];
  assert.match(event.sql, /INSERT INTO place_event/);
  assert.equal(event.values[0], 'test-listing-id');
  assert.match(event.values[1], /Test Monument/);
  assert.equal(event.values[4], 'hcm:123');
  assert.ok(result.logs.includes('✓ HCM: 1 monuments tied to listings'));
  assert.equal(result.errors.length, 0);
});

test('DB failure closes pool and reports failure', async () => {
  const result = await run({ queryThrows: true, pages: [{ features: [{ attributes: { NAME: 'Test', LOCATION: '200 Columbia Avenue', OBJECTID: 123 } }] }] });
  assert.equal(result.exitCode, 1);
  assert.equal(result.calls.at(-1).boundary, 'pg-end');
  assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
});

test('repeat preflight independently rechecks identity', async () => {
  const first = await run({ preflight: true });
  const second = await run({ preflight: true, metadata: { ...valid, name: 'Hollywood Walk of Fame' } });
  assert.equal(first.exitCode, 0);
  assert.equal(second.exitCode, 1);
  assert.equal(first.calls.length + second.calls.length, 2);
});

for (const [name, page, message] of [
  ['ArcGIS error', { error: { code: 499, message: 'Token Required' } }, /ArcGIS error/],
  ['ArcGIS error with features', { error: { code: 499 }, features: [] }, /ArcGIS error/],
  ['null container', null, /Invalid HCM feature page/],
  ['array container', [], /Invalid HCM feature page/],
  ['string container', 'bad page', /Invalid HCM feature page/],
  ['missing features', {}, /Invalid HCM feature page/],
  ['null features', { features: null }, /Invalid HCM feature page/],
  ['object features', { features: {} }, /Invalid HCM feature page/],
  ['string features', { features: 'bad features' }, /Invalid HCM feature page/],
]) {
  test(`${name}: fails without writes or success and closes pool once`, async () => {
    const result = await run({ pages: [page] });
    assert.equal(result.exitCode, 1);
    assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-end']);
    assert.equal(result.errors.length, 1);
    assert.match(result.errors[0], message);
    assert.match(result.errors[0], /offset 0/);
    assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
  });
}

test('valid empty features succeeds without writes and closes pool once', async () => {
  const result = await run();
  assert.equal(result.exitCode, 0);
  assert.equal(result.errors.length, 0);
  assert.deepEqual(result.calls.map(x => x.boundary), ['fetch', 'pg-create', 'fetch', 'pg-end']);
  assert.ok(result.logs.includes('✓ HCM: 0 monuments tied to listings'));
});

for (const [name, secondPage] of [
  ['ArcGIS error', { error: { code: 499 } }],
  ['malformed page', { features: null }],
]) {
  test(`later ${name}: preserves earlier processing, fails without final success and closes pool`, async () => {
    const feature = { attributes: { NAME: 'Test Monument', LOCATION: '200 Columbia Avenue', OBJECTID: 123 } };
    const result = await run({ pages: [{ features: Array.from({ length: 1000 }, () => feature) }, secondPage] });
    assert.equal(result.exitCode, 1);
    const pages = result.calls.filter(x => x.boundary === 'fetch' && x.url.includes('/query?'));
    assert.deepEqual(pages.map(x => new URL(x.url).searchParams.get('resultOffset')), ['0', '1000']);
    assert.equal(result.calls.filter(x => x.boundary === 'pg-query').length, 3000);
    assert.equal(result.calls.filter(x => x.boundary === 'pg-end').length, 1);
    assert.equal(result.calls.at(-1).boundary, 'pg-end');
    assert.equal(result.errors.length, 1);
    assert.match(result.errors[0], /offset 1000/);
    assert.ok(!result.logs.some(line => line.startsWith('✓ HCM:')));
  });
}