`scripts/run-all-night.sh`

66 lines
#!/usr/bin/env bash
# Run-all-night orchestrator.
# Runs every Stage 2-8 importer/enricher serially, in dependency order, and
# tees output to logs/run-all-night.log. Idempotent — safe to re-run.
#
# Pre-reqs: NPI bulk ingest has already completed (~13-15K LA professionals + orgs).
#
# Usage:
#   bash scripts/run-all-night.sh
#   tail -f logs/run-all-night.log

set -uo pipefail
cd "$(dirname "$0")/.."
mkdir -p logs

LOG="logs/run-all-night.log"
exec > >(tee -a "$LOG") 2>&1

step() {
  echo
  echo "═══════════════════════════════════════════════════════════════════"
  echo "▸ $1   ($(date -u +%FT%TZ))"
  echo "═══════════════════════════════════════════════════════════════════"
}

run() {
  local label="$1"; shift
  step "$label"
  if "$@"; then
    echo "✓ $label"
  else
    echo "✗ $label  (continuing — not fatal)"
  fi
}

# ─── Stage 4: facilities (free public bulk files) ─────────────────────────
run "HCAI / CDPH licensed health facilities"          node agents/ingest-agent/importers/hcai.js
run "CMS Hospital General Information"                node agents/ingest-agent/importers/cms-hospitals.js
run "HRSA FQHCs / community health centers"           node agents/ingest-agent/importers/hrsa.js

# ─── Stage 3: CMS Care Compare (DAC) — joins by NPI ───────────────────────
run "CMS Care Compare (Doctors and Clinicians)"       node agents/ingest-agent/importers/cms-care-compare.js

# ─── Stage 2: license enrichment ─────────────────────────────────────────
# DCA is an HTTP scraper — slow but free. Caps at 0.5 rps.
run "DCA license status enrichment"                   node agents/ingest-agent/importers/dca.js

# MBC needs a CSV the user converted from the Access DB; skip if absent.
if [ -f data/mbc/mbc_licenses.csv ]; then
  run "CA Medical Board (MBC) bulk join"              node agents/ingest-agent/importers/mbc.js
else
  echo "› skip MBC (data/mbc/mbc_licenses.csv not present)"
fi

# ─── Stage 6: free geocoding ─────────────────────────────────────────────
run "Geocode (US Census → OSM Nominatim fallback)"    node agents/enrich-agent/geocode.js

# ─── Stats ────────────────────────────────────────────────────────────────
run "Final stats"                                     node scripts/stats.js

echo
echo "═══════════════════════════════════════════════════════════════════"
echo "All-night run complete: $(date -u +%FT%TZ)"
echo "API: node agents/api-agent/server.js  → http://127.0.0.1:9874"
echo "═══════════════════════════════════════════════════════════════════"