`scripts/ingest/hacienda-drain.sh`

16 lines
#!/usr/bin/env bash
# Resumable full drain of the Hacienda enrichment over all un-enriched cedulas.
# Idempotent: each chunk skips already-enriched rows. Safe to re-run after a crash.
set -euo pipefail
cd "$(dirname "$0")/../.."
LOG=data/hacienda-drain.log
: > "$LOG"
echo "[drain] start $(date)" | tee -a "$LOG"
while true; do
  REMAIN=$(psql -h /tmp -U stevestudio2 -d costa_rica_directory -tAc \
    "SELECT count(*) FROM places WHERE cedula_juridica IS NOT NULL AND hacienda_enriched_at IS NULL;")
  echo "[drain] remaining=$REMAIN $(date)" | tee -a "$LOG"
  if [ "$REMAIN" -eq 0 ]; then echo "[drain] COMPLETE $(date)" | tee -a "$LOG"; break; fi
  HACIENDA_BATCH=2000 HACIENDA_THROTTLE_MS=1000 node scripts/ingest/hacienda-enricher.js 2>&1 | tee -a "$LOG"
done