add cncp watchdog: launchd 5-min health probe + auto-kickstart with cooldown

8a40632daa2338b110611a661c454069676b7896 · 2026-05-10 14:24:47 -0700 · Steve Abrams

Files touched

A logs/watchdog.stderr
A logs/watchdog.stdout
A scripts/cncp-watchdog.sh

Diff

commit 8a40632daa2338b110611a661c454069676b7896
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun May 10 14:24:47 2026 -0700

    add cncp watchdog: launchd 5-min health probe + auto-kickstart with cooldown
---
 logs/watchdog.stderr     |  0
 logs/watchdog.stdout     |  0
 scripts/cncp-watchdog.sh | 85 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 85 insertions(+)

diff --git a/logs/watchdog.stderr b/logs/watchdog.stderr
new file mode 100644
index 0000000..e69de29
diff --git a/logs/watchdog.stdout b/logs/watchdog.stdout
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/cncp-watchdog.sh b/scripts/cncp-watchdog.sh
new file mode 100755
index 0000000..b3f0d17
--- /dev/null
+++ b/scripts/cncp-watchdog.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# cncp-watchdog.sh — ping CNCP every run; kickstart-restart if hung.
+#
+# Owned by ~/Library/LaunchAgents/com.steve.cncp-watchdog.plist (runs every 300s).
+# Logs to ~/Projects/cncp-mcp/logs/watchdog.log (rotated by size at 1 MB).
+#
+# Probe : GET http://localhost:3333/api/stats, expect HTTP 200 within 10s.
+# Two-strike : a single timeout doesn't trigger restart — only two in a row, 10s apart.
+# Cooldown : after a kickstart, don't kickstart again for 600s (CNCP cold-start is ~8s
+#            but we want headroom; back-to-back restarts cascade-fail).
+# Boot-grace : after a kickstart, wait 20s before re-probing.
+
+set -u
+
+URL="http://localhost:3333/api/stats"
+PROBE_TIMEOUT=10            # CNCP cold-start measured ~8s
+RETRY_TIMEOUT=10
+BOOT_GRACE=20
+KICKSTART_COOLDOWN=600      # don't kickstart more than once per 10 min
+
+LOG_DIR="$HOME/Projects/cncp-mcp/logs"
+LOG="$LOG_DIR/watchdog.log"
+STATE="$LOG_DIR/.watchdog-state"
+SERVICE="gui/$(id -u)/com.user.cncp"
+MAX_LOG_BYTES=1048576
+
+mkdir -p "$LOG_DIR"
+
+# Rotate log if too big.
+if [[ -f "$LOG" ]]; then
+  size=$(stat -f%z "$LOG" 2>/dev/null || stat -c%s "$LOG" 2>/dev/null || echo 0)
+  [[ "$size" -gt "$MAX_LOG_BYTES" ]] && mv "$LOG" "$LOG.1"
+fi
+
+ts() { /bin/date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+now_epoch() { /bin/date +%s; }
+
+probe() {
+  local timeout="$1"
+  /usr/bin/curl -sS -o /dev/null -w "%{http_code}" --max-time "$timeout" "$URL" 2>/dev/null || echo "000"
+}
+
+last_kickstart_at() {
+  [[ -f "$STATE" ]] && /bin/cat "$STATE" 2>/dev/null || echo 0
+}
+
+# ── strike 1
+code1=$(probe "$PROBE_TIMEOUT")
+if [[ "$code1" == "200" ]]; then
+  echo "$(ts) ok cncp $code1" >> "$LOG"
+  exit 0
+fi
+
+# ── strike 2 (10s later) — single transient hiccups shouldn't trigger a restart
+sleep "$RETRY_TIMEOUT"
+code2=$(probe "$PROBE_TIMEOUT")
+if [[ "$code2" == "200" ]]; then
+  echo "$(ts) transient cncp recovered ($code1 → $code2)" >> "$LOG"
+  exit 0
+fi
+
+# ── cooldown check
+NOW=$(now_epoch)
+LAST=$(last_kickstart_at)
+ELAPSED=$((NOW - LAST))
+if [[ "$ELAPSED" -lt "$KICKSTART_COOLDOWN" ]]; then
+  echo "$(ts) DOWN cncp ($code1 then $code2) — in cooldown, ${ELAPSED}s since last kickstart, skipping" >> "$LOG"
+  exit 0
+fi
+
+# ── kickstart
+echo "$(ts) DOWN cncp ($code1 then $code2) → kickstart $SERVICE" >> "$LOG"
+/bin/launchctl kickstart -k "$SERVICE" >> "$LOG" 2>&1
+echo "$NOW" > "$STATE"
+
+# ── boot grace
+sleep "$BOOT_GRACE"
+code3=$(probe "$PROBE_TIMEOUT")
+if [[ "$code3" == "200" ]]; then
+  echo "$(ts) recovered cncp $code3 after kickstart (+${BOOT_GRACE}s boot grace)" >> "$LOG"
+  exit 0
+else
+  echo "$(ts) STILL DOWN cncp http=$code3 after kickstart+grace — will retry after ${KICKSTART_COOLDOWN}s cooldown" >> "$LOG"
+  exit 1
+fi

← d8161f8 initial scaffold: cncp-mcp v0.1.0 — 12 tools (read+write) ov · back to Cncp Mcp · gitignore: drop logs/ from tracking (watchdog stdout/stderr/ 51e1232 →