`test/ingest-resilience.test.js`

98 lines
'use strict';
// Ingest-layer resilience (Cody ingest audit, cycle 24):
//   (A) fetchText/fetchJson/fetchBuffer must time out instead of hanging forever
//       on a stalled gov site — behavioral test with a fake, never-resolving fetch.
//   (B) meic-pymes.js + ict-cst.js must isolate each record so one poison row
//       doesn't abort the whole run — structural regression guard on the source.

const { test, after } = require('node:test');
const assert = require('node:assert');
const fs = require('node:fs');
const path = require('node:path');

const lib = require('../scripts/ingest/_lib');
const realFetch = global.fetch;
after(async () => { global.fetch = realFetch; try { await lib.pool.end(); } catch {} });

// (A) ---- fetch timeout behavior -------------------------------------------------

test('fetchText rejects (does not hang) when the server never responds — default-signal timeout', async () => {
  // A fetch that resolves ONLY when its abort signal fires => simulates a hung host.
  global.fetch = (_url, o) => new Promise((_res, rej) => {
    if (o.signal.aborted) return rej(o.signal.reason);
    o.signal.addEventListener('abort', () => rej(o.signal.reason));
  });
  const t0 = Date.now();
  await assert.rejects(
    () => lib.fetchText('http://stalled.example/x', { timeoutMs: 40 }),
    (e) => { assert.equal(e.name, 'TimeoutError', 'a hung fetch must abort with a TimeoutError'); return true; });
  assert.ok(Date.now() - t0 < 2000, 'it must give up in ~40ms, not hang');
});

test('fetchJson honors a CALLER-supplied signal instead of imposing its own timeout', async () => {
  global.fetch = (_url, o) => new Promise((_res, rej) => {
    o.signal.addEventListener('abort', () => rej(o.signal.reason || new Error('aborted')));
  });
  const ctrl = new AbortController();
  const p = lib.fetchJson('http://x.example/j', { signal: ctrl.signal });
  ctrl.abort(new Error('caller-aborted'));
  await assert.rejects(() => p, /caller-aborted/, 'the caller-owned signal must be the one in effect');
});

test('fetchText still returns the body on a fast, healthy response', async () => {
  global.fetch = async () => ({ ok: true, status: 200, text: async () => 'hola' });
  assert.equal(await lib.fetchText('http://ok.example/'), 'hola');
});

// Regression: fetchBuffer(url) with NO opts previously always resolved its timeout
// to a hardcoded 60000 literal via `Number(opts.timeoutMs) || fallbackMs || DEFAULT`
// (undefined -> NaN -> falsy -> the truthy fallbackMs literal always won), so
// SCRAPER_TIMEOUT_MS was silently a no-op for the multi-MB MEIC XLSX download —
// the one fetch that most needs an env-tunable timeout. (Cody gate, cycle 25.)
test('fetchBuffer honors its OWN env knob (SCRAPER_BUFFER_TIMEOUT_MS), independent of SCRAPER_TIMEOUT_MS', async () => {
  process.env.SCRAPER_TIMEOUT_MS = '5000';        // deliberately large/irrelevant
  process.env.SCRAPER_BUFFER_TIMEOUT_MS = '35';    // must be the one that governs
  delete require.cache[require.resolve('../scripts/ingest/_lib')];
  const freshLib = require('../scripts/ingest/_lib');
  global.fetch = (_url, o) => new Promise((_res, rej) => {
    o.signal.addEventListener('abort', () => rej(o.signal.reason));
  });
  const t0 = Date.now();
  await assert.rejects(() => freshLib.fetchBuffer('http://big.example/x.xlsx'), /./);
  const elapsed = Date.now() - t0;
  assert.ok(elapsed < 2000, `fetchBuffer must respect SCRAPER_BUFFER_TIMEOUT_MS=35, not the 60000 default or the unrelated 5000 (took ${elapsed}ms)`);
  delete process.env.SCRAPER_TIMEOUT_MS;
  delete process.env.SCRAPER_BUFFER_TIMEOUT_MS;
  await freshLib.pool.end(); // re-requiring _lib created a SECOND pg Pool; close it too
  delete require.cache[require.resolve('../scripts/ingest/_lib')];
});

// (B) ---- per-record isolation (structural regression guard) ---------------------

function src(rel) { return fs.readFileSync(path.join(__dirname, '..', rel), 'utf8'); }

test('meic-pymes.js isolates each row so one poison record cannot abort the run', () => {
  const s = src('scripts/ingest/meic-pymes.js');
  // The per-row catch's warn string only exists if the row body is wrapped; it +
  // errors++ + the partial status together prove a skipped row is counted, not fatal.
  assert.match(s, /\[meic-pymes\] row skipped/, 'per-row catch (with its skip log) must exist');
  assert.match(s, /errors\+\+/, 'a skipped row must be counted, not fatal');
  assert.match(s, /status: errors \? 'partial' : 'ok'/, 'errors must downgrade to partial, not fail the whole run');
});

test('ict-cst.js isolates each listing so one bad card cannot abort the run', () => {
  const s = src('scripts/ingest/ict-cst.js');
  assert.match(s, /\[ict-cst\] listing skipped/, 'per-listing catch (with its skip log) must exist');
  assert.match(s, /errors\+\+/, 'a skipped listing must be counted, not fatal');
  assert.match(s, /errors \? 'partial' : 'ok'/, 'errors must downgrade to partial');
});

test('cr-osm-match.js isolates each place UPDATE and never leaks its pool', () => {
  const s = src('scripts/cr-osm-match.js');
  assert.match(s, /\[cr-osm-match\] place .* skipped/, 'per-record catch (with its skip log) must exist');
  assert.match(s, /errors\+\+/, 'a skipped place must be counted, not fatal');
  // The whole IIFE must be try/finally so pool.end() runs even on a crash (it was
  // a bare trailing pool.end() before — skipped on any throw, leaking a connection).
  assert.match(s, /finally \{[\s\S]*?pool\.end\(\)/, 'pool.end() must be in a finally, not a bare trailing call');
});