`scripts/build-loop.sh`

107 lines
#!/usr/bin/env bash
# ALL-NIGHT autonomous "keep building" loop for HomesOnSpec inventory.
# Layer 1 (this script): deterministic ingest + public-records enrichment across
# the working adapters × top builder states, op-40 each, repeated in freshness
# sweeps until a STOP file appears or MAX_SWEEPS is hit. Facts-only, robots-
# enforced (LiveFetcher), LOCAL DB only — the public Kamatera go-live stays gated.
# Idempotent: re-sweeps of unchanged data are cheap no-ops (contentHash dedup).
# bash-3.2 safe (macOS system bash) — no associative arrays.
# Stop anytime:  touch ~/Projects/homesonspec/tmp/BUILD-LOOP-STOP
set -uo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT/apps/workers" || exit 1
# Load the repo .env (holds the correct DATABASE_URL per host: Kamatera-local on prod, Mac
# socket on dev), then honor any already-injected DATABASE_URL, else default to the Mac socket.
[ -f "$ROOT/.env" ] && { set -a; . "$ROOT/.env"; set +a; }
export DATABASE_URL="${DATABASE_URL:-postgresql://macstudio3@localhost/homesonspec?host=/tmp}"
mkdir -p "$ROOT/tmp"
STOP="$ROOT/tmp/BUILD-LOOP-STOP"
MAX_SWEEPS="${MAX_SWEEPS:-6}"
SWEEP_SLEEP="${SWEEP_SLEEP:-9000}"   # ~2.5h between freshness sweeps (polite)
LOG() { echo "[$(date '+%m-%d %H:%M:%S')] $*"; }
PSQL() { psql "$DATABASE_URL" -tAc "$1" 2>/dev/null; }
CLI() { npx tsx src/cli.ts --adapter="$1" 2>&1 | grep -E '"published"|"pages"|Error|error|inactive|blocked|disallow' | head -3; }

# NATIONWIDE — WEST COAST FIRST (Steve 2026-07-28), then the rest by volume.
# Adapters filter by state; a state a builder doesn't serve is a cheap 0-published no-op.
# CAP = communities/homes per (adapter,state) per sweep — deep enough to actually fill CA.
CAP="${CAP:-120}"
STATES="CA OR WA NV AZ ID HI TX FL NC GA SC TN CO OH PA VA UT IN MD MI MN MO AL OK KY LA NJ NM NY WI AR KS NE IA MS WV CT DE NH ME RI"
fullname() { case "$1" in
  AL) echo "Alabama";; AK) echo "Alaska";; AZ) echo "Arizona";; AR) echo "Arkansas";;
  CA) echo "California";; CO) echo "Colorado";; CT) echo "Connecticut";; DE) echo "Delaware";;
  FL) echo "Florida";; GA) echo "Georgia";; HI) echo "Hawaii";; ID) echo "Idaho";;
  IL) echo "Illinois";; IN) echo "Indiana";; IA) echo "Iowa";; KS) echo "Kansas";;
  KY) echo "Kentucky";; LA) echo "Louisiana";; ME) echo "Maine";; MD) echo "Maryland";;
  MA) echo "Massachusetts";; MI) echo "Michigan";; MN) echo "Minnesota";; MS) echo "Mississippi";;
  MO) echo "Missouri";; MT) echo "Montana";; NE) echo "Nebraska";; NV) echo "Nevada";;
  NH) echo "New Hampshire";; NJ) echo "New Jersey";; NM) echo "New Mexico";; NY) echo "New York";;
  NC) echo "North Carolina";; ND) echo "North Dakota";; OH) echo "Ohio";; OK) echo "Oklahoma";;
  OR) echo "Oregon";; PA) echo "Pennsylvania";; RI) echo "Rhode Island";; SC) echo "South Carolina";;
  SD) echo "South Dakota";; TN) echo "Tennessee";; TX) echo "Texas";; UT) echo "Utah";;
  VT) echo "Vermont";; VA) echo "Virginia";; WA) echo "Washington";; WV) echo "West Virginia";;
  WI) echo "Wisconsin";; WY) echo "Wyoming";;
  *) echo "$1";; esac; }
lc() { echo "$1" | tr '[:upper:]' '[:lower:]'; }
slug() { fullname "$1" | tr '[:upper:] ' '[:lower:]-'; }   # "North Carolina" -> north-carolina

# max-it: one worker per adapter, sweeping ALL states. The 4 run CONCURRENTLY —
# different domains, so each stays polite within its own per-source rate limit while
# 4x the inventory lands per wall-clock second. $0 (no LLM anywhere in ingestion).
sweep_adapter() {
  local kind="$1"
  for S in $STATES; do
    [ -f "$STOP" ] && break
    case "$kind" in
      kb)  KBHOME_STATE="$S"                        CLI kb-home-site    | sed "s/^/  kb  $S: /" ;;
      drh) DRHORTON_STATE="$(slug "$S")"  DRHORTON_PAGE_LIMIT=$CAP  CLI dr-horton-site  | sed "s/^/  drh $S: /" ;;
      plt) PULTE_STATE="$(fullname "$S")" PULTE_PAGE_LIMIT=$CAP     CLI pultegroup-site | sed "s/^/  plt $S: /" ;;
      tri) TRIPOINTE_STATE="$(lc "$S")"   TRIPOINTE_PAGE_LIMIT=$CAP CLI tri-pointe-site | sed "s/^/  tri $S: /" ;;
      len) LENNAR_STATE="$S"              LENNAR_PAGE_LIMIT=$CAP    CLI lennar-site     | sed "s/^/  len $S: /" ;;
    esac
  done
}

# national adapters (single feed, no per-state param) — run ONCE per sweep, not per-state,
# so we don't hammer the same national endpoint 40x. Toll Brothers publishes ~all in one run.
sweep_national() { CLI toll-brothers-site | sed "s/^/  tol NATIONAL: /"; }

# wait until no other adapter run is in flight (avoid same-source races)
while pgrep -f "cli.ts --adapter" 2>/dev/null | grep -qv "^$$\$"; do sleep 20; done

for k in dr-horton-site kb-home-site pultegroup-site tri-pointe-site lennar-site toll-brothers-site; do
  PSQL "update \"SourceRegistry\" set active=true, health='HEALTHY', \"consecutiveFailures\"=0 where key='$k';" >/dev/null
done

sweep=0
while [ ! -f "$STOP" ] && [ "$sweep" -lt "$MAX_SWEEPS" ]; do
  sweep=$((sweep+1))
  s0=$(PSQL "select count(*) from \"InventoryHome\" where status='PUBLISHED';")
  b0=$(PSQL "select count(distinct \"builderId\") from \"InventoryHome\" where status='PUBLISHED';")
  LOG "===== SWEEP $sweep/$MAX_SWEEPS start: $s0 homes / $b0 builders ====="
  LOG "fanning out 5 state adapters + Toll(national) in parallel across ${CAP}-deep states (West Coast first)"
  sweep_adapter kb  &
  sweep_adapter drh &
  sweep_adapter plt &
  sweep_adapter tri &
  sweep_adapter len &
  # Toll = communities-only (0 spec homes until wave-2 QMI); its data changes on the order of
  # weeks, so refresh ~daily (every 8th ~2.5h sweep) not every sweep (DTD 4/5 throttle 2026-07-28).
  if [ $((sweep % 8)) -eq 1 ]; then sweep_national & fi
  wait
  # drain enrichment queue (bigger batches for speed)
  for i in $(seq 1 60); do
    rem=$(PSQL "select count(*) from \"Community\" where lat is not null and lon is not null and amenities is null;")
    [ "${rem:-0}" -eq 0 ] && { LOG "enrich: queue drained"; break; }
    LOG "enrich pass $i: $rem left"
    NEARBY_LIMIT=80 npx tsx scripts/enrich-nearby.ts >/dev/null 2>&1
  done
  s1=$(PSQL "select count(*) from \"InventoryHome\" where status='PUBLISHED';")
  b1=$(PSQL "select count(distinct \"builderId\") from \"InventoryHome\" where status='PUBLISHED';")
  enr=$(PSQL "select count(*) from \"Community\" where amenities is not null;")
  LOG "===== SWEEP $sweep done: $s1 homes / $b1 builders (+$((s1-s0))) · $enr communities enriched ====="
  [ -f "$STOP" ] && break
  [ "$sweep" -lt "$MAX_SWEEPS" ] && { LOG "sleeping ${SWEEP_SLEEP}s"; sleep "$SWEEP_SLEEP"; }
done
LOG "BUILD LOOP EXIT (sweeps=$sweep)"