← back to La Socrata Ingester

src/adapters/socrata.js

39 lines

import { fetchJson } from './http.js';

// Socrata SODA paginator. Yields { rows, url } per page.
// Stable pagination via $order=:id (the always-unique system field), so new rows
// arriving mid-crawl never shift the window. Incremental via $where cursor filter.
//
// src fields used: domain, datasetId, cursorField?, staticFilter?
// opts: since (ISO string | null), pageSize (default 50000), appToken, maxRows
export async function* socrataPages(src, opts = {}) {
  const { since = null, pageSize = 50000, appToken, maxRows = Infinity } = opts;
  const base = `https://${src.domain}/resource/${src.datasetId}.json`;
  const headers = appToken ? { 'X-App-Token': appToken } : {};
  let offset = 0;

  for (;;) {
    const limit = Math.min(pageSize, maxRows - offset);
    if (limit <= 0) break;

    const p = new URLSearchParams();
    p.set('$limit', String(limit));
    p.set('$offset', String(offset));
    p.set('$order', ':id');

    const where = [];
    if (src.staticFilter) where.push(src.staticFilter);
    if (since && src.cursorField) where.push(`${src.cursorField} > '${since}'`);
    if (where.length) p.set('$where', where.join(' AND '));

    const url = `${base}?${p}`;
    const rows = await fetchJson(url, { headers });
    if (!rows.length) break;

    yield { rows, url };

    offset += rows.length;
    if (rows.length < limit) break; // last page
  }
}