`supervisor.sh`

191 lines
#!/bin/bash
#
# claude-code-supervisor
# ----------------------
# Persistent, rate-limit-aware supervisor for a single Claude Code process.
#
# Launches Claude Code as a child, watches its combined stdout/stderr for
# rate-limit indicators, and on a hit terminates ONLY that child (and its
# direct descendants), waits, and restarts it in the same working directory.
# Repeated rate-limits back off progressively; a productive run resets the
# delay. Designed to run unattended for days; pair with the launchd plist so
# macOS restarts the supervisor itself after a crash or reboot.
#
# Usage:
#   ./supervisor.sh [-- <claude args...>]
#   CLAUDE_SUP_WORKDIR=/path ./supervisor.sh -- --dangerously-skip-permissions
#
# Everything unrelated to our own child is left untouched. We never pattern-kill
# "claude" — we only signal the exact PID we spawned and `pkill -P` its children.
#
set -uo pipefail

# ---- configuration (override via environment) -------------------------------
WORKDIR="${CLAUDE_SUP_WORKDIR:-$HOME}"
LOG_DIR="${CLAUDE_SUP_LOG_DIR:-$HOME/.claude/supervisor-logs}"
# The command to supervise. Default is bare `claude`; anything after `--` on the
# command line is appended as arguments.
CLAUDE_BIN="${CLAUDE_SUP_BIN:-claude}"

# Backoff ladder (seconds) applied on consecutive problems. After the last entry
# it stays pinned at that value until a productive run resets the index. The
# override exists so the real control flow can be exercised quickly in tests.
IFS=',' read -r -a BACKOFF <<< "${CLAUDE_SUP_BACKOFF:-20,40,60,120,300}"

# A run lasting at least this long (seconds) is treated as "productive" and
# resets the backoff index to 0.
SUCCESS_RESET_SECONDS="${CLAUDE_SUP_SUCCESS_RESET:-180}"

# Delay before restarting after a *clean* exit that followed a productive run.
CLEAN_RESTART_DELAY="${CLAUDE_SUP_CLEAN_DELAY:-5}"

# Poll interval while watching the child (seconds).
POLL="${CLAUDE_SUP_POLL:-2}"

# Case-insensitive regex of rate-limit / usage-cap indicators.
RL_PATTERN="${CLAUDE_SUP_PATTERN:-rate limit|rate-limit|429|too many requests|usage limit|overloaded_error|quota exceeded}"

# ---- setup ------------------------------------------------------------------
mkdir -p "$LOG_DIR"
RUN_LOG="$LOG_DIR/claude-current.log"          # rolling log of the live child
SUP_LOG="$LOG_DIR/supervisor.log"              # the supervisor's own event log

# Parse optional `-- <args...>`
CHILD_ARGS=()
if [ "${1:-}" = "--" ]; then
  shift
  CHILD_ARGS=("$@")
fi

# A bare launchd job has no task to resume. Holding idle is deliberate: it
# prevents a hot loop of `claude` invocations that immediately fail for lack of
# a prompt, while keeping the LaunchAgent healthy and ready for explicit use.
if [ "${#CHILD_ARGS[@]}" -eq 0 ]; then
  log_line="$(date '+%Y-%m-%d %H:%M:%S') [supervisor] inactive: no Claude arguments supplied; use supervisor.sh -- -p '<prompt>'"
  echo "$log_line"
  echo "$log_line" >>"$SUP_LOG"
  while true; do sleep 300; done
fi

log() {
  # timestamped line to both the supervisor log and stdout (captured by launchd)
  local line
  line="$(date '+%Y-%m-%d %H:%M:%S') [supervisor] $*"
  echo "$line"
  echo "$line" >>"$SUP_LOG"
}

CHILD_PID=""
WATCHER_PID=""

# Kill ONLY our child subtree. Never a broad pattern match.
kill_child() {
  [ -n "$CHILD_PID" ] || return 0
  if kill -0 "$CHILD_PID" 2>/dev/null; then
    pkill -TERM -P "$CHILD_PID" 2>/dev/null   # direct descendants first
    kill -TERM "$CHILD_PID" 2>/dev/null
    # grace period, then SIGKILL if still alive
    local n=0
    while kill -0 "$CHILD_PID" 2>/dev/null && [ "$n" -lt 10 ]; do
      sleep 1; n=$((n+1))
    done
    if kill -0 "$CHILD_PID" 2>/dev/null; then
      pkill -KILL -P "$CHILD_PID" 2>/dev/null
      kill -KILL "$CHILD_PID" 2>/dev/null
    fi
  fi
}

cleanup() {
  [ -n "$WATCHER_PID" ] && kill "$WATCHER_PID" 2>/dev/null
  kill_child
  log "supervisor exiting"
}
trap cleanup EXIT
trap 'log "received TERM/INT — shutting down"; exit 0' TERM INT

# Best-effort: if a line says "try again in Ns" / "resets in N seconds", honor it.
parse_reset_seconds() {
  # reads the last ~40 lines of the run log, returns a seconds value or empty
  local secs
  secs="$(tail -n 40 "$RUN_LOG" 2>/dev/null \
    | grep -ioE 'try again in [0-9]+ ?s|resets? in [0-9]+ ?(second|s)' \
    | grep -oE '[0-9]+' | tail -n 1)"
  echo "$secs"
}

log "starting; workdir=$WORKDIR bin=$CLAUDE_BIN args=[${CHILD_ARGS[*]:-}]"
cd "$WORKDIR" || { log "FATAL: cannot cd to $WORKDIR"; exit 1; }

idx=0
while true; do
  : > "$RUN_LOG"                 # fresh log per run (rotated copy kept below)
  rl_flag="$LOG_DIR/.ratelimited.$$"
  rm -f "$rl_flag"

  # --- launch the child, combined output to RUN_LOG ---
  # ${arr[@]+"${arr[@]}"} = the bash-3.2-safe way to expand a possibly-empty
  # array under `set -u` (a bare "${arr[@]}" on an empty array errors here).
  "$CLAUDE_BIN" ${CHILD_ARGS[@]+"${CHILD_ARGS[@]}"} >>"$RUN_LOG" 2>&1 &
  CHILD_PID=$!
  start=$(date +%s)
  log "launched claude pid=$CHILD_PID"

  # --- monitor loop ---
  rate_limited=0
  while kill -0 "$CHILD_PID" 2>/dev/null; do
    if grep -qiE "$RL_PATTERN" "$RUN_LOG" 2>/dev/null; then touch "$rl_flag"; fi
    if [ -f "$rl_flag" ]; then
      rate_limited=1
      log "rate-limit indicator detected — terminating pid=$CHILD_PID"
      kill_child
      break
    fi
    sleep "$POLL"
  done

  # reap exit status (0 if we killed it ourselves)
  wait "$CHILD_PID" 2>/dev/null
  rc=$?
  # Catch a marker printed by a child that exited between monitor polls.
  if grep -qiE "$RL_PATTERN" "$RUN_LOG" 2>/dev/null; then rate_limited=1; fi
  dur=$(( $(date +%s) - start ))

  # keep a timestamped copy of this run's tail for forensics
  cp "$RUN_LOG" "$LOG_DIR/run-$(date '+%Y%m%d-%H%M%S').log" 2>/dev/null

  # productive run resets the backoff ladder
  if [ "$dur" -ge "$SUCCESS_RESET_SECONDS" ]; then
    if [ "$idx" -ne 0 ]; then log "productive run (${dur}s) — backoff reset"; fi
    idx=0
  fi

  # decide reason + delay
  if [ "$rate_limited" -eq 1 ]; then
    reason="rate-limit"
    delay="${BACKOFF[$idx]}"
    resets="$(parse_reset_seconds)"
    if [ -n "$resets" ] && [ "$resets" -gt "$delay" ] 2>/dev/null; then
      log "child reported reset in ${resets}s — honoring it"
      delay="$resets"
    fi
    [ "$idx" -lt $(( ${#BACKOFF[@]} - 1 )) ] && idx=$((idx+1))
  elif [ "$rc" -eq 0 ]; then
    reason="clean-exit(rc=0)"
    if [ "$dur" -ge "$SUCCESS_RESET_SECONDS" ]; then
      delay="$CLEAN_RESTART_DELAY"
    else
      # exited cleanly but very fast — treat as a flap, back off
      delay="${BACKOFF[$idx]}"
      [ "$idx" -lt $(( ${#BACKOFF[@]} - 1 )) ] && idx=$((idx+1))
    fi
  else
    reason="crash(rc=$rc)"
    delay="${BACKOFF[$idx]}"
    [ "$idx" -lt $(( ${#BACKOFF[@]} - 1 )) ] && idx=$((idx+1))
  fi

  log "run ended: ${reason} after ${dur}s — restarting in ${delay}s (backoff idx=$idx)"
  sleep "$delay"
done