← back to Scarlet Riverboat Masquerade

tests/mock-reliability.cjs

179 lines

// Phase 1 reliability test — archive.org MOCKED via route interception (independent of real archive.org).
// Run: PW=/Users/macstudio3/.npm-global/lib/node_modules/playwright node tests/mock-reliability.cjs
const { chromium } = require(process.env.PW);
const http = require('http');
const fs = require('fs');
const path = require('path');

const ROOT = path.resolve(__dirname, '..');
const MIME = { '.html':'text/html', '.js':'text/javascript', '.css':'text/css', '.json':'application/json', '.png':'image/png', '.jpg':'image/jpeg', '.svg':'image/svg+xml', '.ico':'image/x-icon' };

function serve() {
  return new Promise(res => {
    const srv = http.createServer((req, r) => {
      let p = decodeURIComponent(req.url.split('?')[0]);
      if (p === '/' || p === '') p = '/index.html';
      const f = path.join(ROOT, p);
      if (!f.startsWith(ROOT) || !fs.existsSync(f) || fs.statSync(f).isDirectory()) { r.writeHead(404); return r.end('nf'); }
      r.writeHead(200, { 'Content-Type': MIME[path.extname(f)] || 'application/octet-stream' });
      fs.createReadStream(f).pipe(r);
    });
    srv.listen(0, '127.0.0.1', () => res({ srv, port: srv.address().port }));
  });
}

const CORS = { 'access-control-allow-origin': '*', 'content-type': 'application/json' };
const okBody = (year) => JSON.stringify({ response: { numFound: (year - 1960) * 30, docs: [] } });
const searchBody = JSON.stringify({ response: { numFound: 12345, docs: [{ identifier: 'gd1994', title: 'Grateful Dead', date: '1994-01-01', venue: 'Test', coverage: 'X', downloads: 9 }] } });

function yearOf(url) { const m = /year%3A(\d{4})|year:(\d{4})/.exec(decodeURIComponent(url)); return m ? +(m[1] || m[2]) : null; }
const isCount = (url) => /rows=0/.test(url);

// HOLD each count request open ~120ms so concurrent requests actually pile up — otherwise
// instant fulfillment means nothing ever overlaps and the cap assertion is meaningless.
const HOLD = 120;

// Instrumented route handler factory. mode: 'partial' | 'sustained'
function makeHandler(state, mode) {
  return async (route) => {
    const url = route.request().url();
    if (!/archive\.org\/advancedsearch/.test(url)) return route.continue();
    if (!isCount(url)) { // search query (rows=40) — not part of the throttled burst
      return route.fulfill({ status: 200, headers: CORS, body: searchBody });
    }
    const year = yearOf(url);
    state.totalCount++;
    state.attempts[year] = (state.attempts[year] || 0) + 1;
    (state.times[year] = state.times[year] || []).push(Date.now()); // for backoff-timing assertions
    state.active++;
    if (state.active > state.maxActive) state.maxActive = state.active;
    await new Promise(r => setTimeout(r, HOLD)); // hold the slot so the pool visibly saturates
    const done = (fn) => { state.active--; return fn(); };

    if (mode === 'sustained') { // everything fails → circuit breaker must halt the storm
      return done(() => route.abort('failed'));
    }
    // partial (~40% failure mix): index buckets across the GD 1965-1995 range
    const idx = year - 1965;
    const bucket = ((idx % 5) + 5) % 5;
    const attempt = state.attempts[year];
    if (bucket === 3) { // permanent failure — half timeout, half malformed JSON
      if (year % 2 === 0) return done(() => route.abort('failed'));
      return done(() => route.fulfill({ status: 200, headers: { 'access-control-allow-origin': '*', 'content-type': 'application/json' }, body: 'THIS-IS-NOT-JSON' }));
    }
    if (bucket === 4) { // 429 on first attempt, success after — tests distinct 429 handling + backoff
      state.saw429.add(year);
      if (attempt === 1) return done(() => route.fulfill({ status: 429, headers: CORS, body: '{"error":"rate limited"}' }));
      return done(() => route.fulfill({ status: 200, headers: CORS, body: okBody(year) }));
    }
    return done(() => route.fulfill({ status: 200, headers: CORS, body: okBody(year) }));
  };
}

async function settleTiles(page) {
  // wait until no tile is still "loading" (all resolved to a number or an err mark), OR the map error appears
  await page.waitForFunction(() => {
    if (document.querySelector('.mr-map-err')) return true;
    const tiles = document.querySelectorAll('.mr-yr');
    if (!tiles.length) return false;
    return ![...tiles].some(t => t.classList.contains('loading'));
  }, { timeout: 45000 });
}

(async () => {
  const { srv, port } = await serve();
  const base = `http://127.0.0.1:${port}/index.html`;
  const browser = await chromium.launch({ channel: 'chrome', args: ['--use-gl=swiftshader'] });
  const fails = [];
  const pass = (c, m) => { console.log((c ? 'PASS' : 'FAIL') + ' — ' + m); if (!c) fails.push(m); };

  try {
    // ---------- Scenario A: partial 40% failure mix ----------
    {
      const ctx = await browser.newContext();
      const page = await ctx.newPage();
      const errs = [];
      page.on('pageerror', e => errs.push(String(e.message || e)));
      const state = { active: 0, maxActive: 0, totalCount: 0, attempts: {}, times: {}, saw429: new Set() };
      await page.route('**/archive.org/**', makeHandler(state, 'partial'));
      await page.goto(base, { waitUntil: 'load', timeout: 60000 });
      await page.waitForFunction(() => document.querySelectorAll('.mr-yr').length > 0, { timeout: 30000 });
      await settleTiles(page);
      await page.waitForTimeout(500); // let any trailing paint flush

      // (a) concurrency cap — must saturate the pool (>=4 in flight) yet NEVER exceed 5.
      // The old unbounded Promise.all would drive this to 31; the cap holds it at 5.
      pass(state.maxActive <= 5 && state.maxActive >= 4, `(a) count-query concurrency saturated at ${state.maxActive} (cap 5, never exceeded; unbounded would be 31)`);

      // (b) no tile shows "0"; failed tiles show err/—; failed years never cached
      const ycTexts = await page.$$eval('.mr-yr .yc', els => els.map(e => (e.textContent || '').trim()));
      pass(!ycTexts.includes('0'), `(b) no year tile displays "0" (values: ${[...new Set(ycTexts)].slice(0,8).join(',')}…)`);
      const errYears = await page.$$eval('.mr-yr.err', els => els.map(e => e.dataset.y));
      pass(errYears.length > 0 && (await page.$$eval('.mr-yr.err .yc', els => els.every(e => (e.textContent||'').trim() === '—'))),
        `(b) ${errYears.length} failed tiles render "—" error state, not a false 0`);
      const cache = await page.evaluate(() => JSON.parse(localStorage.getItem('srm.counts.v1') || '{}'));
      const cachedFailed = errYears.filter(y => cache['gd:' + y] !== undefined);
      pass(cachedFailed.length === 0, `(b) failed years never persisted to cache (leaked: ${cachedFailed.join(',') || 'none'})`);
      const cacheVals = Object.values(cache).map(v => v.n);
      pass(cacheVals.length > 0 && cacheVals.every(n => n > 0), `(b) cache holds only positive counts (${cacheVals.length} entries)`);

      // (c) bounded retries per request
      const maxAttempts = Math.max(0, ...Object.values(state.attempts));
      pass(maxAttempts <= 3, `(c) per-request attempts <= 3 (1 + 2 retries); observed max ${maxAttempts}`);

      // (d) 429 handled distinctly — a 429-then-200 year shows its REAL number (not parsed as success, not a 0/—)
      const y429 = [...state.saw429][0];
      const t429 = y429 == null ? null : await page.$eval(`.mr-yr[data-y="${y429}"] .yc`, e => (e.textContent || '').trim());
      const expect429 = y429 == null ? null : ((y429 - 1960) * 30).toLocaleString();
      pass(y429 != null && t429 === expect429 && state.attempts[y429] >= 2,
        `(d) 429 year ${y429}: backed off + retried (attempts ${state.attempts[y429]}) → shows real ${t429} (expected ${expect429}), not parsed as JSON success`);

      // (d) prove the 429 backoff is HARDER than a timeout backoff (not just "it retried").
      // Gap between consecutive handler entries for a year = HOLD + that year's inter-attempt backoff.
      const gap = (y) => { const t = state.times[y]; return t && t.length >= 2 ? t[1] - t[0] : null; };
      const g429 = gap(y429);
      // a timeout-class retry year: bucket 3 AND even (route.abort) — it retries on the network-drop path
      const toYear = Object.keys(state.times).map(Number).find(y => ((y - 1965) % 5 + 5) % 5 === 3 && y % 2 === 0 && (state.times[y] || []).length >= 2);
      const gTO = toYear != null ? gap(toYear) : null;
      pass(g429 != null && gTO != null && g429 > gTO,
        `(d) 429 backoff ${g429}ms > timeout backoff ${gTO}ms (year ${toYear}) — 429 backs off HARDER, distinct code path`);

      pass(errs.length === 0, `(no page errors) ${errs.join(' | ') || 'clean'}`);
      await ctx.close();
    }

    // ---------- Scenario B: sustained failure → circuit breaker halts the storm ----------
    {
      const ctx = await browser.newContext();
      const page = await ctx.newPage();
      const state = { active: 0, maxActive: 0, totalCount: 0, attempts: {}, times: {}, saw429: new Set() };
      await page.route('**/archive.org/**', makeHandler(state, 'sustained'));
      await page.goto(base, { waitUntil: 'load', timeout: 60000 });
      await page.waitForFunction(() => document.querySelector('.mr-map-err') || [...document.querySelectorAll('.mr-yr')].every(t => !t.classList.contains('loading')), { timeout: 45000 });
      await page.waitForTimeout(800); // ensure no late retry storm

      pass(state.maxActive <= 5, `(a) sustained: max concurrent ${state.maxActive} <= 5`);
      // circuit breaker: total requests must be far below unthrottled worst case (31 * 3 = 93) and years short-circuited
      const totalYears = 31;
      const yearsRequested = Object.keys(state.attempts).length;
      pass(state.totalCount <= 30, `(c) circuit-breaker bounds total requests to ${state.totalCount} (<=30; unthrottled worst case ~93)`);
      pass(yearsRequested < totalYears, `(c) circuit-breaker short-circuited ${totalYears - yearsRequested} years (only ${yearsRequested} hit the network)`);
      const maxAttempts = Math.max(0, ...Object.values(state.attempts));
      pass(maxAttempts <= 3, `(c) sustained: per-request attempts <= 3; observed ${maxAttempts}`);
      const hasErr = await page.$('.mr-map-err');
      const zeroTile = await page.$$eval('.mr-yr .yc', els => els.some(e => (e.textContent || '').trim() === '0'));
      pass(!!hasErr && !zeroTile, `(b) sustained failure → graceful map error, no tile shows "0"`);
      await ctx.close();
    }
  } catch (e) {
    fails.push('EXCEPTION: ' + (e && e.stack || e));
    console.log('EXCEPTION', e);
  } finally {
    await browser.close();
    srv.close();
  }

  console.log('\n' + (fails.length ? `RESULT: ${fails.length} FAILURE(S)` : 'RESULT: ALL MOCK GUARDRAILS PASS'));
  process.exit(fails.length ? 1 : 0);
})();