`scripts/fetch_all_ca.sh`

32 lines
#!/usr/bin/env bash
# Disk-safe full-California load: fetch each remaining dollar-band ZIP, load it into
# db/ca_unclaimed.sqlite, then DELETE the raw ZIP + CSVs before the next band — so peak
# extra disk stays ~one band (~9GB) instead of ~20GB for the whole unzipped set.
# Band 04 ($500+) is already loaded, so we only fetch 03/02/01.
set -euo pipefail
cd "$(dirname "$0")/.."
SRC="data/ca-source"; BASE="https://claimit.ca.gov/upd-property-records"
mkdir -p "$SRC"

for band in 03_From_100_To_Below_500 02_From_10_To_Below_100 01_From_0_To_Below_10; do
  echo "=== $band ==="
  df -h . | tail -1 | awk '{print "  disk free: "$4}'
  echo "  downloading $band.zip …"
  curl -s -A "Mozilla/5.0" -o "$SRC/$band.zip" "$BASE/$band.zip"
  echo "  unzipping …"
  unzip -o -q "$SRC/$band.zip" -d "$SRC"
  rm -f "$SRC/$band.zip"          # drop the zip immediately (CSVs are what we load)
  echo "  loading into DB …"
  python3 scripts/load_ca.py $(cd "$SRC" && ls ${band#*_}*.csv 2>/dev/null | tr '\n' ' ')
  echo "  cleaning raw CSVs …"
  rm -f "$SRC"/${band#*_}*.csv    # free the ~7GB of CSVs before the next band
  echo "  done $band; DB now:"; ls -lh db/ca_unclaimed.sqlite | awk '{print "    "$5}'
done
echo "=== FULL CALIFORNIA LOAD COMPLETE ==="
python3 - <<'PY'
import sqlite3
c=sqlite3.connect("db/ca_unclaimed.sqlite")
n,d=c.execute("SELECT COUNT(*),SUM(amount) FROM ca_property").fetchone()
print(f"db/ca_unclaimed.sqlite: {n:,} California records · ${d:,.2f} total unclaimed")
PY