`claude-resume-daemon.sh`

85 lines
#!/usr/bin/env bash
# claude-resume-daemon.sh — always-on supervisor (run via launchd) that auto-
# recovers ANY claude tmux pane from the transient "temporarily limiting
# requests" API error (the server-side limiter, NOT your usage cap).
#
# Every POLL seconds it scans all tmux panes; for any pane whose last few lines
# show the limiter signature, it waits DELAY then types RESUME + Enter into that
# pane, with a per-pane COOLDOWN so it never spams while a retry is in flight.
# Stays alive across tmux-server restarts; never exits on transient errors.
#
# Tunables via env (set in the plist): RESUME_DELAY RESUME_COOLDOWN RESUME_POLL
# RESUME_TAIL RESUME_MSG. Disable wholesale: touch ~/.claude-resume-disabled
set -uo pipefail   # NOT -e: a transient tmux hiccup must not kill the daemon

DELAY="${RESUME_DELAY:-3}"          # wait before sending RESUME
COOLDOWN="${RESUME_COOLDOWN:-30}"   # min seconds between RESUMEs to the same pane
POLL="${RESUME_POLL:-4}"            # scan cadence
TAIL="${RESUME_TAIL:-25}"           # bottom-most N screen lines (was 4 — far too few:
                                    # the error sits ABOVE the input-box chrome, and
                                    # the TUI wraps it across several lines)
MSG="${RESUME_MSG:-RESUME}"
DISABLE_FLAG="$HOME/.claude-resume-disabled"
STATE_DIR="${TMPDIR:-/tmp}/claude-resume"; mkdir -p "$STATE_DIR" 2>/dev/null || true

# Limiter signature — matched against a WHITESPACE-STRIPPED, lowercased capture
# (see below). The TUI hard-wraps the error mid-word ("temporar\nily limiting"),
# so phrases must be written with their spaces removed to survive wrapping.
SIG='temporarilylimitingrequests|serveristemporarily|apierror.*rate.?limit'

log()  { printf '%s %s\n' "$(date '+%F %T')" "$*"; }           # -> stdout (ledger-quiet)
err()  { printf '%s %s\n' "$(date '+%F %T')" "$*" >&2; }       # -> stderr (real failures)
now()  { date +%s; }

while :; do
  # Master kill-switch — keep the process alive but idle so launchd doesn't
  # crash-loop and you can re-enable without a reload.
  if [ -e "$DISABLE_FLAG" ]; then sleep "$POLL"; continue; fi

  # No tmux binary / no server yet → idle and retry, never exit.
  if ! command -v tmux >/dev/null 2>&1; then sleep 15; continue; fi

  panes="$(tmux list-panes -a -F '#{pane_id}' 2>/dev/null || true)"
  if [ -n "$panes" ]; then
    while IFS= read -r pid; do
      [ -n "$pid" ] || continue
      cap="$(tmux capture-pane -p -t "$pid" 2>/dev/null | tail -n "$TAIL" || true)"
      [ -n "$cap" ] || continue
      # Normalize: strip ALL whitespace + lowercase, so the terminal's hard-wrap
      # (which splits the error phrase mid-word) can't defeat the signature.
      capn="$(printf '%s' "$cap" | tr -d '[:space:]' | tr 'A-Z' 'a-z')"
      printf '%s' "$capn" | grep -qE "$SIG" || continue

      key="$STATE_DIR/$(printf '%s' "$pid" | tr -dc 'A-Za-z0-9')"
      lf="$(cat "$key" 2>/dev/null || echo 0)"
      [ $(( $(now) - lf )) -ge "$COOLDOWN" ] || continue

      # STUCK-PANE confirmation (2026-05-30): a REAL limiter FREEZES the pane;
      # a session merely QUOTING the phrase (e.g. pasting the error, or a claude
      # discussing it) keeps producing output. Snapshot now, wait DELAY, snapshot
      # again — only RESUME if the pane is UNCHANGED (genuinely stuck) AND still
      # shows the signature. This excludes phrase-quoting ACTIVE sessions
      # fleet-wide, with no per-pane identification, so the daemon can run for all
      # panes without self-triggering on sessions that just talk about the error.
      log "[resume] $pid limiter seen — confirming stuck for ${DELAY}s"
      sleep "$DELAY"
      [ -e "$DISABLE_FLAG" ] && continue
      cap2="$(tmux capture-pane -p -t "$pid" 2>/dev/null | tail -n "$TAIL" || true)"
      capn2="$(printf '%s' "$cap2" | tr -d '[:space:]' | tr 'A-Z' 'a-z')"
      if [ "$capn2" != "$capn" ] || ! printf '%s' "$capn2" | grep -qE "$SIG"; then
        log "[resume] $pid pane changed/cleared in ${DELAY}s — active or recovered, skip (false positive)"
        continue
      fi
      if tmux send-keys -t "$pid" "$MSG" Enter 2>/dev/null; then
        log "[resume] $pid stuck-confirmed — sent '$MSG'"
        printf '%s' "$(now)" > "$key"
      else
        err "[resume] send-keys failed for $pid"
      fi
    done <<EOF
$panes
EOF
  fi
  sleep "$POLL"
done