`scripts/doctor-agent-loop.sh`

142 lines
#!/bin/zsh
# Doctor Agent — overnight build supervisor.
#
# Runs in a tight loop:
#   1. v5 mockup gen on next 25 unclaimed orgs (skips already-generated)
#   2. quality report tail
#   3. pd-* pm2 health check (one auto-restart per process per night)
#   4. sleep N seconds, repeat
#
# Cap: stop at $LOOP_HOURS hours OR when --once flag passed.
# Logs: $PD/logs/doctor-agent-loop.{out,err}.log
#
# Trigger:
#   - manual:   ./scripts/doctor-agent-loop.sh
#   - launchd:  com.steve.doctor-agent-overnight (dusk-to-dawn)

set -u
PD="${PD:-/Users/macstudio3/Projects/professional-directory}"
cd "$PD"

LOOP_HOURS="${LOOP_HOURS:-10}"        # default 10h overnight
BATCH_SIZE="${BATCH_SIZE:-25}"        # orgs per pass
SLEEP_SECONDS="${SLEEP_SECONDS:-30}"  # cool-down between passes
LLM_TIMEOUT_MS="${LLM_TIMEOUT_MS:-540000}"  # 9 min/variant
MODEL_HOST_PRIMARY="${MODEL_HOST_PRIMARY:-http://127.0.0.1:11434}"
MODEL_HOST_FALLBACK="${MODEL_HOST_FALLBACK:-http://192.168.1.133:11434}"
PRIMARY_MODEL="${PRIMARY_MODEL:-gemma3:27b}"
FALLBACK_MODEL="${FALLBACK_MODEL:-qwen3:32b}"

LOG_DIR="$PD/logs"
mkdir -p "$LOG_DIR"
STATE="$LOG_DIR/doctor-agent-loop.state"
HEARTBEAT="$LOG_DIR/doctor-agent-loop.heartbeat"
OUT_LOG="$LOG_DIR/doctor-agent-loop.log"

START_TS=$(date +%s)
DEADLINE=$((START_TS + LOOP_HOURS * 3600))

log()  { echo "[doctor-agent $(date +%H:%M:%S)] $*" | tee -a "$OUT_LOG"; }
beat() { date +%s > "$HEARTBEAT"; }

# Pick the live ollama host. Prefer MS1; fall back to MS2 if MS1 is unreachable.
pick_host() {
  if curl -m 4 -s -f "$MODEL_HOST_PRIMARY/api/tags" >/dev/null 2>&1; then
    echo "$MODEL_HOST_PRIMARY|$PRIMARY_MODEL"
  elif curl -m 4 -s -f "$MODEL_HOST_FALLBACK/api/tags" >/dev/null 2>&1; then
    echo "$MODEL_HOST_FALLBACK|$FALLBACK_MODEL"
  else
    echo "DOWN|"
  fi
}

# Health-check the pd-* PM2 fleet. Auto-restart up to once per process per loop run.
check_pm2() {
  local restarted_marker="$LOG_DIR/doctor-agent-loop.restarted-this-run"
  command -v pm2 >/dev/null || { log "pm2 not found in PATH; skipping health check"; return; }
  local statuses=$(pm2 jlist 2>/dev/null | python3 -c '
import json,sys
try: d=json.load(sys.stdin)
except: sys.exit(0)
for p in d:
  n=p.get("name","")
  if n.startswith("pd-"):
    s=p.get("pm2_env",{}).get("status","?")
    print(f"{n} {s}")
' 2>/dev/null)
  echo "$statuses" | while read -r name pmstatus; do
    [ -z "$name" ] && continue
    if [ "$pmstatus" != "online" ]; then
      if grep -q "^$name$" "$restarted_marker" 2>/dev/null; then
        log "WARN $name is $pmstatus (already auto-restarted this run; not retrying — investigate)"
      else
        log "AUTO-RESTART $name (status=$pmstatus)"
        pm2 restart "$name" >/dev/null 2>&1 || true
        echo "$name" >> "$restarted_marker"
      fi
    fi
  done
}

# Tail-summarize last 25 lines of the v5 quality report.
quality_summary() {
  local qa="$PD/data/mockups/_v5_quality_report.jsonl"
  [ -f "$qa" ] || { log "no quality report yet"; return; }
  python3 - <<'PYEOF' "$qa"
import json,sys
lines=open(sys.argv[1]).read().splitlines()[-50:]
ok=tn=0
issues={}
for ln in lines:
  try: r=json.loads(ln)
  except: continue
  if r.get("ok"): ok+=1
  else:
    tn+=1
    for i in r.get("issues",[]): issues[i]=issues.get(i,0)+1
total=ok+tn
top=sorted(issues.items(),key=lambda x:-x[1])[:3]
print(f"recent: {ok}/{total} ok ({(100*ok/total) if total else 0:.0f}%)  top-issues: {top}")
PYEOF
}

run_one_pass() {
  beat
  local pick=$(pick_host)
  local host="${pick%%|*}"
  local model="${pick##*|}"
  if [ "$host" = "DOWN" ]; then
    log "BOTH OLLAMA HOSTS DOWN — sleeping 5min"
    sleep 300
    return
  fi
  log "pass on $host model=$model batch=$BATCH_SIZE"
  MODEL_HOST="$host" \
  LLM_MOCKUP_MODEL="$model" \
  LIMIT="$BATCH_SIZE" \
  VARIANTS="a,b,c" \
  LLM_TIMEOUT_MS="$LLM_TIMEOUT_MS" \
  STRICT="${STRICT:-0}" \
  MAX_RETRIES="${MAX_RETRIES:-2}" \
    node "$PD/scripts/generate-mockups-v5.js" 2>&1 | tee -a "$OUT_LOG"
  quality_summary | tee -a "$OUT_LOG"
  check_pm2
}

if [ "${1:-}" = "--once" ]; then
  log "RUN-ONCE mode"
  run_one_pass
  exit 0
fi

log "LOOP started — deadline in ${LOOP_HOURS}h"
rm -f "$LOG_DIR/doctor-agent-loop.restarted-this-run"

while [ $(date +%s) -lt $DEADLINE ]; do
  run_one_pass
  log "sleeping ${SLEEP_SECONDS}s"
  sleep "$SLEEP_SECONDS"
done

log "LOOP done — deadline reached"