`test/arcgis-adaptive-page.test.mjs`

135 lines
// Regression test for TK-11500: the NavigateLA gateway 502s on a GEOMETRY page whose
// response is too large (the Generalized Zoning OBJECTID 42000-56000 band — ~44MB/2000-row
// page). arcgisPages must HALVE the page and retry the SAME cursor until it fits, then GROW
// the size back once past the heavy band, and stay FAIL-LOUD when a page fails for a
// non-size reason (non-geometry) so a real outage is never masked as a shrink.
// Run: node test/arcgis-adaptive-page.test.mjs   (zero deps; a few s due to real backoff)
// Small floor so the test's small page sizes exercise real shrinking (read at call time).
process.env.LA_ADAPTIVE_MIN_PAGE = '4';
import { arcgisPages } from '../src/adapters/arcgis.js';

let fail = 0;
const feat = (i) => ({ attributes: { OBJECTID: i, ZONE_CMPLT: `Z${i}` }, geometry: { rings: [[[i, i]]] } });

// Build a mock `fetch` over a finite OFFSET-mode dataset of `total` features, where a
// gateway 502 is returned whenever the requested resultRecordCount exceeds `okAtOrBelow`
// AND the current resultOffset is inside [0, bandEnd) — i.e. an oversized-geometry BAND.
// Records every requested page size and each 502 so the test can assert shrink + recovery.
function offsetMock({ total, okAtOrBelow, bandEnd }) {
  const log = { sizes: [], gateway502: 0 };
  globalThis.fetch = async (url) => {
    const u = new URL(url);
    const rc = Number(u.searchParams.get('resultRecordCount'));
    const off = Number(u.searchParams.get('resultOffset'));
    log.sizes.push(rc);
    if (rc > okAtOrBelow && off < bandEnd) {
      log.gateway502++;
      return { ok: false, status: 502, json: async () => ({}), text: async () => 'gateway timeout' };
    }
    const slice = [];
    for (let i = off; i < Math.min(off + rc, total); i++) slice.push(feat(i));
    const body = { features: slice, exceededTransferLimit: off + rc < total };
    return { ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body) };
  };
  return log;
}

const collect = async (src, opts) => {
  const out = [];
  for await (const page of arcgisPages(src, opts)) out.push(...page.rows);
  return out;
};

// ── 1) OFFSET geometry: descends through an oversized band, then GROWS back ──────────────
// total 25, oversized band = offsets [0,20) (502 if page>10 there), floor 5, start page 20.
{
  const log = offsetMock({ total: 25, okAtOrBelow: 10, bandEnd: 20 });
  const src = { endpoint: 'http://fake/MapServer/73', geometry: true, noOrder: true };
  try {
    const rows = await collect(src, { pageSize: 20 });
    const complete = rows.length === 25 && rows.every((r, i) => r.OBJECTID === i && r.__geometry);
    const shrank = log.gateway502 >= 1 && log.sizes.includes(10);           // it halved 20->10 in the band
    const recovered = log.sizes.some((s, k) => s === 20 && k === log.sizes.length - 1); // last page ran full-size past the band
    const ok = complete && shrank && recovered;
    console.log(`OFFSET-BACKOFF: ${ok ? 'PASS (descended past the band, completed 25/25, grew back to full page past it)' :
      `FAIL (rows=${rows.length}, 502s=${log.gateway502}, sizes=${log.sizes.join(',')})`}`);
    if (!ok) fail++;
  } catch (e) {
    console.log(`OFFSET-BACKOFF: FAIL (threw: ${e.message}; sizes=${log.sizes.join(',')})`);
    fail++;
  }
}

// ── 2) NEGATIVE: without the fix this class is unrecoverable — prove the FIX is load-bearing.
// Same oversized band but page size can NEVER be honored at 20; a naive paginator (no
// shrink) would 502 out. With the fix it must still complete. We assert it did NOT simply
// error, AND that it actually used a smaller page (proving the completion came from shrinking,
// not from the band happening to allow 20).
{
  const log = offsetMock({ total: 12, okAtOrBelow: 5, bandEnd: 999 }); // band = whole layer
  const src = { endpoint: 'http://fake/MapServer/73', geometry: true, noOrder: true };
  try {
    const rows = await collect(src, { pageSize: 20 });
    const ok = rows.length === 12 && log.gateway502 >= 1 && log.sizes.some((s) => s <= 5);
    console.log(`ALL-BAND-COMPLETE: ${ok ? 'PASS (a layer that 502s at every full page still completes 12/12 via shrink)' :
      `FAIL (rows=${rows.length}, 502s=${log.gateway502}, minPage=${Math.min(...log.sizes)})`}`);
    if (!ok) fail++;
  } catch (e) {
    console.log(`ALL-BAND-COMPLETE: FAIL (threw instead of shrinking to complete: ${e.message})`);
    fail++;
  }
}

// ── 3) SAFETY: a NON-geometry 502 is a real outage, NOT a size problem — must NOT be masked
// by shrinking; it must throw (fail-loud). geometry:false, 502 on every page.
{
  const log = offsetMock({ total: 10, okAtOrBelow: -1, bandEnd: 999 }); // always 502
  const src = { endpoint: 'http://fake/MapServer/73', geometry: false, noOrder: true };
  const before = log.sizes.length;
  try {
    await collect(src, { pageSize: 20 });
    console.log('NONGEOM-FAILLOUD: FAIL (did not throw on a non-geometry gateway outage)');
    fail++;
  } catch (e) {
    // it must have thrown WITHOUT ever halving the page (no size below the starting 20)
    const neverShrank = log.sizes.every((s) => s === 20);
    const ok = /HTTP 502/.test(e.message) && neverShrank;
    console.log(`NONGEOM-FAILLOUD: ${ok ? 'PASS (non-geometry 502 threw loud, never masked by a page shrink)' :
      `FAIL (msg=${e.message}, sizes=${log.sizes.slice(before).join(',')})`}`);
    if (!ok) fail++;
  }
}

// ── 4) OID mode is wired too (zoning is offset, but the fix is general) ───────────────────
// oid-cursor mock: 502 if page>8; success returns features with OBJECTID > lastOid.
{
  const log = { sizes: [], gateway502: 0 };
  const total = 20;
  globalThis.fetch = async (url) => {
    const u = new URL(url);
    const rc = Number(u.searchParams.get('resultRecordCount'));
    const m = /OBJECTID%20%3E%20(-?\d+)|OBJECTID > (-?\d+)/.exec(decodeURIComponent(u.searchParams.get('where') || ''));
    const lastOid = Number((m && (m[1] ?? m[2])) ?? -1);
    log.sizes.push(rc);
    if (rc > 8) { log.gateway502++; return { ok: false, status: 502, json: async () => ({}), text: async () => 'to' }; }
    const slice = [];
    for (let i = lastOid + 1; i < total && slice.length < rc; i++) slice.push(feat(i));
    const body = { features: slice };
    return { ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body) };
  };
  const src = { endpoint: 'http://fake/FeatureServer/0', paginate: 'oid', geometry: true, oidField: 'OBJECTID' };
  try {
    const rows = await collect(src, { pageSize: 16 });
    const ok = rows.length === total && log.gateway502 >= 1 && log.sizes.includes(8);
    console.log(`OID-BACKOFF: ${ok ? 'PASS (oid-cursor mode also shrinks + completes 20/20)' :
      `FAIL (rows=${rows.length}, 502s=${log.gateway502}, sizes=${log.sizes.join(',')})`}`);
    if (!ok) fail++;
  } catch (e) {
    console.log(`OID-BACKOFF: FAIL (threw: ${e.message})`);
    fail++;
  }
}

console.log(fail ? `\n${fail} FAILED` : '\nALL PASS');
process.exit(fail);