`scripts/cslb-load.sh`

23 lines
#!/bin/bash
# CSLB full refresh: download the free portal Master CSV, load into cslb_raw, and
# record the load in la_ingest_state so the canary's staleness check reflects REALITY
# (Cody c3: a manually-seeded state row lies; state must be written by the load path).
set -euo pipefail
cd "$(dirname "$0")/.."
DB="${REALESTATE_DB:-realestate}"

echo "1/3 download CSLB Master CSV…"
node src/cslb/download.js

echo "2/3 load into cslb_raw (truncate + copy)…"
psql "$DB" -q -c "TRUNCATE cslb_raw;"
psql "$DB" -q -c "\copy cslb_raw FROM 'tmp/cslb-master.csv' WITH (FORMAT csv, HEADER true)"
N=$(psql "$DB" -tAc "SELECT count(*) FROM cslb_raw")

echo "3/3 record load in la_ingest_state…"
psql "$DB" -q -c "INSERT INTO la_ingest_state (source, dataset_id, last_run, rows_upserted, last_status)
  VALUES ('cslb_master','cslb-portal', now(), $N, 'ok')
  ON CONFLICT (source) DO UPDATE SET last_run=now(), rows_upserted=$N, last_status='ok';"

echo "✔ CSLB loaded: $N contractors; la_ingest_state.cslb_master updated (real load timestamp)."