`scripts/build-agent/watchdog.sh`

76 lines
#!/bin/bash
# Lawyer Build Agent — watchdog
# Run every 5 min via launchd. Ensures Mac1 mockup-render worker is alive.
# Mac2 worker is intentionally NOT auto-started (RAM constrained).
set -euo pipefail

PROJ="/Users/macstudio3/Projects/lawyer-directory-builder"
LOG="$PROJ/logs/build-agent/watchdog.log"
mkdir -p "$(dirname "$LOG")"
ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }

# ── Mac1 worker check ──────────────────────────────────────────────────────
mac1_running() {
  ssh -o ConnectTimeout=4 -o BatchMode=yes mac1ts \
    'pgrep -f "tsx.*src/enrich/render_mockups.ts" >/dev/null 2>&1' \
    >/dev/null 2>&1
}

mac1_tunnel_alive() {
  ssh -o ConnectTimeout=4 -o BatchMode=yes mac1ts \
    'lsof -nP -iTCP:15432 -sTCP:LISTEN >/dev/null 2>&1' \
    >/dev/null 2>&1
}

ensure_mac1_tunnel() {
  if mac1_tunnel_alive; then return 0; fi
  echo "[$(ts)] mac1 tunnel down — reopening" | tee -a "$LOG"
  ssh -o ConnectTimeout=4 mac1ts \
    "ssh -o StrictHostKeyChecking=accept-new -fN -L 15432:/tmp/.s.PGSQL.5432 stevestudio2@100.65.187.120" \
    >>"$LOG" 2>&1 || true
  sleep 2
}

start_mac1_worker() {
  echo "[$(ts)] starting mac1 worker" | tee -a "$LOG"
  local logname="mockup-mac1-unique-$(date +%H%M%S).log"
  ssh -o ConnectTimeout=4 mac1ts \
    "cd ~/Projects/lawyer-directory-builder && \
     nohup env PATH=\"\$HOME/local/bin:\$PATH\" \
       DATABASE_URL='postgres://stevestudio2@127.0.0.1:15432/lawyer_professional_directory' \
       OLLAMA_URL='http://localhost:11434' \
       LLM_MOCKUP_MODEL=qwen3:14b \
       LLM_TIMEOUT_MS=360000 \
       SITE_INTEL_TIMEOUT_MS=12000 \
       \"\$HOME/local/bin/node\" \"\$HOME/Projects/lawyer-directory-builder/node_modules/.bin/tsx\" \
       src/enrich/render_mockups.ts --la-county --id-mod 2 --id-rem 1 \
       > logs/$logname 2>&1 & disown" \
    >>"$LOG" 2>&1
}

# ── ensure DB schema is up to date (migrations sometimes get reverted) ─────
SCHEMA_OK=$(psql -d lawyer_professional_directory -tAc \
  "SELECT 1 FROM information_schema.tables WHERE table_name='site_mockups' LIMIT 1" 2>/dev/null || true)
if [[ "$SCHEMA_OK" != "1" ]]; then
  echo "[$(ts)] schema MISSING site_mockups — running migrations" | tee -a "$LOG"
  ( cd "$PROJ" && npm run migrate >> "$LOG" 2>&1 ) || true
  psql -d lawyer_professional_directory -c \
    "UPDATE app_users SET tier='admin' WHERE role='admin' AND (tier IS NULL OR tier='client')" \
    >> "$LOG" 2>&1 || true
fi

# ── main ───────────────────────────────────────────────────────────────────
ensure_mac1_tunnel
if mac1_running; then
  echo "[$(ts)] mac1 worker OK" >> "$LOG"
else
  echo "[$(ts)] mac1 worker DOWN — restarting" | tee -a "$LOG"
  start_mac1_worker
fi

# ── prune old logs (keep 30 most recent per host) ──────────────────────────
ls -t "$PROJ/logs/"mockup-mac2-unique-*.log 2>/dev/null | tail -n +31 | xargs rm -f 2>/dev/null || true
ssh -o ConnectTimeout=4 mac1ts \
  'ls -t ~/Projects/lawyer-directory-builder/logs/mockup-mac1-unique-*.log 2>/dev/null | tail -n +31 | xargs rm -f 2>/dev/null || true' \
  >/dev/null 2>&1 || true