← back to Nationalrealestate

db/migrations/011_ingest_cursor.sql

11 lines

-- M-PH2: resumable ingest cursor — replaces the count%total offset that wrapped
-- to 0 on full coverage and re-fetched the first pages every hour. Each paging
-- source advances next_offset forward and wraps cleanly, so a covered source
-- does a light rolling refresh instead of re-hammering the endpoint.
CREATE TABLE ingest_cursor (
  source      TEXT PRIMARY KEY,
  next_offset INT NOT NULL DEFAULT 0,
  total       INT,
  updated_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()
);