`scripts/cncp-watchdog.sh`

86 lines
#!/usr/bin/env bash
# cncp-watchdog.sh — ping CNCP every run; kickstart-restart if hung.
#
# Owned by ~/Library/LaunchAgents/com.steve.cncp-watchdog.plist (runs every 300s).
# Logs to ~/Projects/cncp-mcp/logs/watchdog.log (rotated by size at 1 MB).
#
# Probe : GET http://localhost:3333/api/stats, expect HTTP 200 within 10s.
# Two-strike : a single timeout doesn't trigger restart — only two in a row, 10s apart.
# Cooldown : after a kickstart, don't kickstart again for 600s (CNCP cold-start is ~8s
#            but we want headroom; back-to-back restarts cascade-fail).
# Boot-grace : after a kickstart, wait 20s before re-probing.

set -u

URL="http://localhost:3333/api/stats"
PROBE_TIMEOUT=10            # CNCP cold-start measured ~8s
RETRY_TIMEOUT=10
BOOT_GRACE=20
KICKSTART_COOLDOWN=600      # don't kickstart more than once per 10 min

LOG_DIR="$HOME/Projects/cncp-mcp/logs"
LOG="$LOG_DIR/watchdog.log"
STATE="$LOG_DIR/.watchdog-state"
SERVICE="gui/$(id -u)/com.user.cncp"
MAX_LOG_BYTES=1048576

mkdir -p "$LOG_DIR"

# Rotate log if too big.
if [[ -f "$LOG" ]]; then
  size=$(stat -f%z "$LOG" 2>/dev/null || stat -c%s "$LOG" 2>/dev/null || echo 0)
  [[ "$size" -gt "$MAX_LOG_BYTES" ]] && mv "$LOG" "$LOG.1"
fi

ts() { /bin/date -u +"%Y-%m-%dT%H:%M:%SZ"; }
now_epoch() { /bin/date +%s; }

probe() {
  local timeout="$1"
  /usr/bin/curl -sS -o /dev/null -w "%{http_code}" --max-time "$timeout" "$URL" 2>/dev/null || echo "000"
}

last_kickstart_at() {
  [[ -f "$STATE" ]] && /bin/cat "$STATE" 2>/dev/null || echo 0
}

# ── strike 1
code1=$(probe "$PROBE_TIMEOUT")
if [[ "$code1" == "200" ]]; then
  echo "$(ts) ok cncp $code1" >> "$LOG"
  exit 0
fi

# ── strike 2 (10s later) — single transient hiccups shouldn't trigger a restart
sleep "$RETRY_TIMEOUT"
code2=$(probe "$PROBE_TIMEOUT")
if [[ "$code2" == "200" ]]; then
  echo "$(ts) transient cncp recovered ($code1 → $code2)" >> "$LOG"
  exit 0
fi

# ── cooldown check
NOW=$(now_epoch)
LAST=$(last_kickstart_at)
ELAPSED=$((NOW - LAST))
if [[ "$ELAPSED" -lt "$KICKSTART_COOLDOWN" ]]; then
  echo "$(ts) DOWN cncp ($code1 then $code2) — in cooldown, ${ELAPSED}s since last kickstart, skipping" >> "$LOG"
  exit 0
fi

# ── kickstart
echo "$(ts) DOWN cncp ($code1 then $code2) → kickstart $SERVICE" >> "$LOG"
/bin/launchctl kickstart -k "$SERVICE" >> "$LOG" 2>&1
echo "$NOW" > "$STATE"

# ── boot grace
sleep "$BOOT_GRACE"
code3=$(probe "$PROBE_TIMEOUT")
if [[ "$code3" == "200" ]]; then
  echo "$(ts) recovered cncp $code3 after kickstart (+${BOOT_GRACE}s boot grace)" >> "$LOG"
  exit 0
else
  echo "$(ts) STILL DOWN cncp http=$code3 after kickstart+grace — will retry after ${KICKSTART_COOLDOWN}s cooldown" >> "$LOG"
  exit 1
fi