← back to La Socrata Ingester

src/adapters/http.js

24 lines

export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// GET JSON with retry + exponential backoff on 429 / 5xx / network errors.
export async function fetchJson(url, { headers = {}, tries = 5 } = {}) {
  let attempt = 0;
  for (;;) {
    attempt++;
    try {
      const res = await fetch(url, { headers });
      if (res.status === 429 || res.status >= 500) {
        if (attempt >= tries) throw new Error(`HTTP ${res.status} after ${tries} tries: ${url}`);
        const wait = Math.min(30000, 500 * 2 ** attempt);
        await sleep(wait);
        continue;
      }
      if (!res.ok) throw new Error(`HTTP ${res.status}: ${url}\n${(await res.text()).slice(0, 300)}`);
      return await res.json();
    } catch (err) {
      if (attempt >= tries) throw err;
      await sleep(Math.min(30000, 500 * 2 ** attempt));
    }
  }
}