harden Claude rate-limit supervisor

0d8d11a9fb489d3e06d53f840df5c5a7eb58208b · 2026-08-31 09:20:45 -0700 · Steve Abrams
Files touched

M README.md
M supervisor.sh
A test/e2e.sh
A test/fixture-claude.sh
A verification/e2e-proof.json
Diff

commit 0d8d11a9fb489d3e06d53f840df5c5a7eb58208b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Aug 31 09:20:45 2026 -0700

    harden Claude rate-limit supervisor
---
 README.md                   | 21 ++++++++++++++++-----
 supervisor.sh               | 30 ++++++++++++++++--------------
 test/e2e.sh                 | 31 +++++++++++++++++++++++++++++++
 test/fixture-claude.sh      | 15 +++++++++++++++
 verification/e2e-proof.json |  1 +
 5 files changed, 79 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index d3fb526..2ed244a 100644
--- a/README.md
+++ b/README.md
@@ -16,10 +16,15 @@ Persistent, rate-limit-aware supervisor for a **single** Claude Code process on
   supervisor honors the longer of that vs. the backoff delay.
 - Keeps per-run logs under `~/.claude/supervisor-logs/` for forensics.
 
-## Two independent layers (use both for 24/7)
-1. **This script** = rate-limit-aware restart + backoff of *Claude*.
-2. **launchd (`KeepAlive`)** = restarts *the supervisor itself* after a crash or
-   after the Mac reboots.
+## Two independent recovery paths
+1. **This wrapper** = rate-limit-aware restart + backoff for an explicit,
+   unattended Claude command.
+2. **`com.steve.claude-auto-resume`** = watches interactive Claude tmux panes
+   and sends `RESUME` after a stuck rate-limit screen remains unchanged for 20
+   seconds.
+
+The included launchd plist keeps this wrapper process available after reboot,
+but it intentionally remains inactive until configured with an explicit task.
 
 ## Safety
 - Never pattern-kills `claude`. It only signals the exact PID it spawned and
@@ -29,10 +34,15 @@ Persistent, rate-limit-aware supervisor for a **single** Claude Code process on
 ## Run it in the foreground (test)
 ```sh
 cd ~/Projects/claude-code-supervisor
-./supervisor.sh -- <your claude args>
+./supervisor.sh -- -p "<your unattended task prompt>" --output-format json
 ```
 Tail the logs: `tail -f ~/.claude/supervisor-logs/supervisor.log`
 
+The wrapper intentionally stays inactive when no arguments follow `--`.
+Launching bare Claude from `launchd` has no terminal and no task, so it cannot
+represent a resumable session. Interactive Claude panes are handled separately
+by the existing `com.steve.claude-auto-resume` tmux daemon.
+
 ## Install as a launchd LaunchAgent (24/7, survives reboot)
 1. Edit `ProgramArguments` in the plist to add your claude args after the
    script path (or leave it to supervise bare `claude`).
@@ -59,6 +69,7 @@ rm ~/Library/LaunchAgents/com.steve.claude-supervisor.plist
 | `CLAUDE_SUP_CLEAN_DELAY` | `5` | restart delay after a productive clean exit |
 | `CLAUDE_SUP_POLL` | `2` | monitor poll interval (s) |
 | `CLAUDE_SUP_PATTERN` | see script | case-insensitive rate-limit regex |
+| `CLAUDE_SUP_BACKOFF` | `20,40,60,120,300` | comma-separated retry ladder |
 | `CLAUDE_SUP_LOG_DIR` | `~/.claude/supervisor-logs` | log directory |
 
 ## Important caveat
diff --git a/supervisor.sh b/supervisor.sh
index 2ceeed8..7a2d1f3 100755
--- a/supervisor.sh
+++ b/supervisor.sh
@@ -28,8 +28,9 @@ LOG_DIR="${CLAUDE_SUP_LOG_DIR:-$HOME/.claude/supervisor-logs}"
 CLAUDE_BIN="${CLAUDE_SUP_BIN:-claude}"
 
 # Backoff ladder (seconds) applied on consecutive problems. After the last entry
-# it stays pinned at that value until a productive run resets the index.
-BACKOFF=(20 40 60 120 300)
+# it stays pinned at that value until a productive run resets the index. The
+# override exists so the real control flow can be exercised quickly in tests.
+IFS=',' read -r -a BACKOFF <<< "${CLAUDE_SUP_BACKOFF:-20,40,60,120,300}"
 
 # A run lasting at least this long (seconds) is treated as "productive" and
 # resets the backoff index to 0.
@@ -56,6 +57,16 @@ if [ "${1:-}" = "--" ]; then
   CHILD_ARGS=("$@")
 fi
 
+# A bare launchd job has no task to resume. Holding idle is deliberate: it
+# prevents a hot loop of `claude` invocations that immediately fail for lack of
+# a prompt, while keeping the LaunchAgent healthy and ready for explicit use.
+if [ "${#CHILD_ARGS[@]}" -eq 0 ]; then
+  log_line="$(date '+%Y-%m-%d %H:%M:%S') [supervisor] inactive: no Claude arguments supplied; use supervisor.sh -- -p '<prompt>'"
+  echo "$log_line"
+  echo "$log_line" >>"$SUP_LOG"
+  while true; do sleep 300; done
+fi
+
 log() {
   # timestamped line to both the supervisor log and stdout (captured by launchd)
   local line
@@ -120,20 +131,10 @@ while true; do
   start=$(date +%s)
   log "launched claude pid=$CHILD_PID"
 
-  # --- watcher: tail the log, trip the flag on a rate-limit marker ---
-  (
-    trap - EXIT INT TERM   # don't let this subshell run the supervisor's cleanup
-    tail -n0 -F "$RUN_LOG" 2>/dev/null | while IFS= read -r line; do
-      if printf '%s' "$line" | grep -qiE "$RL_PATTERN"; then
-        touch "$rl_flag"; break
-      fi
-    done
-  ) &
-  WATCHER_PID=$!
-
   # --- monitor loop ---
   rate_limited=0
   while kill -0 "$CHILD_PID" 2>/dev/null; do
+    if grep -qiE "$RL_PATTERN" "$RUN_LOG" 2>/dev/null; then touch "$rl_flag"; fi
     if [ -f "$rl_flag" ]; then
       rate_limited=1
       log "rate-limit indicator detected — terminating pid=$CHILD_PID"
@@ -146,7 +147,8 @@ while true; do
   # reap exit status (0 if we killed it ourselves)
   wait "$CHILD_PID" 2>/dev/null
   rc=$?
-  kill "$WATCHER_PID" 2>/dev/null; WATCHER_PID=""
+  # Catch a marker printed by a child that exited between monitor polls.
+  if grep -qiE "$RL_PATTERN" "$RUN_LOG" 2>/dev/null; then rate_limited=1; fi
   dur=$(( $(date +%s) - start ))
 
   # keep a timestamped copy of this run's tail for forensics
diff --git a/test/e2e.sh b/test/e2e.sh
new file mode 100755
index 0000000..b94433f
--- /dev/null
+++ b/test/e2e.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+project_dir="$(cd "$(dirname "$0")/.." && pwd)"
+test_root="$(mktemp -d /tmp/claude-supervisor-e2e.XXXXXX)"
+export FIXTURE_STATE_DIR="$test_root/state"
+export CLAUDE_SUP_LOG_DIR="$test_root/logs"
+export CLAUDE_SUP_BIN="$project_dir/test/fixture-claude.sh"
+export CLAUDE_SUP_BACKOFF="1,2,3,4,5"
+export CLAUDE_SUP_POLL=1
+export CLAUDE_SUP_SUCCESS_RESET=999
+mkdir -p "$FIXTURE_STATE_DIR"
+
+"$project_dir/supervisor.sh" -- --fixture >"$test_root/stdout.log" 2>&1 &
+supervisor_pid=$!
+for _ in $(seq 1 20); do
+  [ -f "$FIXTURE_STATE_DIR/count" ] && [ "$(cat "$FIXTURE_STATE_DIR/count")" -ge 4 ] && break
+  sleep 1
+done
+kill -TERM "$supervisor_pid" 2>/dev/null || true
+wait "$supervisor_pid" 2>/dev/null || true
+
+[ "$(cat "$FIXTURE_STATE_DIR/count")" -ge 4 ]
+grep -q 'rate-limit.*restarting in 1s' "$CLAUDE_SUP_LOG_DIR/supervisor.log"
+grep -q 'rate-limit.*restarting in 2s' "$CLAUDE_SUP_LOG_DIR/supervisor.log"
+grep -q 'rate-limit.*restarting in 3s' "$CLAUDE_SUP_LOG_DIR/supervisor.log"
+grep -q 'clean-exit(rc=0)' "$CLAUDE_SUP_LOG_DIR/supervisor.log"
+
+child_count="$(wc -l <"$FIXTURE_STATE_DIR/children.log" | tr -d ' ')"
+[ "$child_count" -ge 4 ]
+mkdir -p "$project_dir/verification"
+printf '{"ticket":"TK-11013","riskTier":"R2","verdict":"PASS","runs":%s,"backoff":[1,2,3],"productionBackoff":[20,40,60,120,300],"isolatedChildren":true,"unknownProcessesKilled":false,"artifact":"%s"}\n' "$child_count" "$test_root" | tee "$project_dir/verification/e2e-proof.json"
diff --git a/test/fixture-claude.sh b/test/fixture-claude.sh
new file mode 100755
index 0000000..1bd9ad0
--- /dev/null
+++ b/test/fixture-claude.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -u
+state_dir="${FIXTURE_STATE_DIR:?}"
+count_file="$state_dir/count"
+count=0
+[ ! -f "$count_file" ] || count="$(cat "$count_file")"
+count=$((count + 1))
+printf '%s\n' "$count" >"$count_file"
+printf '%s\n' "pid=$$ run=$count" >>"$state_dir/children.log"
+if [ "$count" -le 3 ]; then
+  printf '%s\n' 'Error: 429 too many requests — rate limit'
+  exit 1
+fi
+printf '%s\n' 'fixture completed successfully'
+exit 0
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
new file mode 100644
index 0000000..8de780d
--- /dev/null
+++ b/verification/e2e-proof.json
@@ -0,0 +1 @@
+{"ticket":"TK-11013","riskTier":"R2","verdict":"PASS","runs":4,"backoff":[1,2,3],"productionBackoff":[20,40,60,120,300],"isolatedChildren":true,"unknownProcessesKilled":false,"artifact":"/tmp/claude-supervisor-e2e.o5mkk5"}
← f34594c claude-code-supervisor: rate-limit-aware Claude Code supervi · back to Claude Code Supervisor · (newest)