dw-cert-renewal-canary: read-only watchdog that keeps the Kamatera LE fleet always-clean (alerts CNCP+George on worsening expired/expiring/renew-failure counts)

7fcabc5a8c915697bcbb5ec9f16f09a0ad31eaf3 · 2026-07-15 11:11:32 -0700 · Steve
Files touched

A .gitignore
A data/latest.json
A run.sh
Diff

commit 7fcabc5a8c915697bcbb5ec9f16f09a0ad31eaf3
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jul 15 11:11:32 2026 -0700

    dw-cert-renewal-canary: read-only watchdog that keeps the Kamatera LE fleet always-clean (alerts CNCP+George on worsening expired/expiring/renew-failure counts)
---
 .gitignore       |  5 ++++
 data/latest.json |  1 +
 run.sh           | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..266a7e9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+run.log
+launchd.log
+data/state.json
+node_modules/
+.DS_Store
diff --git a/data/latest.json b/data/latest.json
new file mode 100644
index 0000000..df55322
--- /dev/null
+++ b/data/latest.json
@@ -0,0 +1 @@
+{"ts":"2026-07-15 11:11:30","verdict":"FAIL","total":226,"expired":1,"expiring":0,"failures":99,"reachable":true}
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..8703987
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,72 @@
+#!/bin/zsh
+# dw-cert-renewal-canary — READ-ONLY watchdog that keeps the Kamatera Let's Encrypt
+# fleet "always clean". SSHes read-only to 45.61.58.125, reads `certbot certificates`
+# + the last `certbot renew` failure summary, and classifies:
+#   EXPIRED   = any cert past expiry (already down)
+#   EXPIRING  = any cert < EXPIRING_DAYS to expiry (about to go down)
+#   FAILURES  = the last renewal run's "N renew failure(s)" count
+# Alerts (CNCP parking-lot card + George email) ONLY on a WORSENING transition
+# (expired↑, expiring↑, or failures↑) — never on steady-state, recovery, or the
+# first baseline run. Touches nothing on prod (pure reads). Writes data/latest.json
+# so the meta-watchdog sees it ran. Born 2026-07-15 after the 99/225 renew-failure find.
+set -u
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+HERE="${0:A:h}"; DATA="$HERE/data"; mkdir -p "$DATA"
+STATE="$DATA/state.json"; LATEST="$DATA/latest.json"; LOG="$HERE/run.log"
+HOST="${CERT_HOST:-root@45.61.58.125}"
+EXPIRING_DAYS="${EXPIRING_DAYS:-14}"
+TS="$(date '+%Y-%m-%d %H:%M:%S')"
+log(){ echo "[$TS] $*" >>"$LOG"; }
+
+# --- read-only pull from Kamatera --------------------------------------------
+RAW="$(ssh -o ConnectTimeout=20 -o BatchMode=yes "$HOST" '
+  echo "FAILURES=$(grep -oE "[0-9]+ renew failure" /var/log/letsencrypt/letsencrypt.log 2>/dev/null | tail -1 | grep -oE "^[0-9]+")"
+  echo "TOTAL=$(ls /etc/letsencrypt/renewal/*.conf 2>/dev/null | wc -l | tr -d " ")"
+  certbot certificates 2>/dev/null | grep -E "Certificate Name:|Expiry Date:"
+' 2>/dev/null)"
+if [ -z "$RAW" ]; then
+  log "UNREACHABLE — no data (ssh failed); writing UNKNOWN, no alert"
+  echo "{\"ts\":\"$TS\",\"verdict\":\"UNKNOWN\",\"reachable\":false}" > "$LATEST"
+  exit 0
+fi
+
+FAILURES=$(echo "$RAW" | sed -n 's/^FAILURES=//p'); FAILURES=${FAILURES:-0}
+TOTAL=$(echo "$RAW" | sed -n 's/^TOTAL=//p'); TOTAL=${TOTAL:-0}
+# count expired (INVALID) + expiring (VALID: n days, n<=EXPIRING_DAYS)
+EXPIRED=$(echo "$RAW" | grep -c "INVALID")
+EXPIRING=$(echo "$RAW" | grep -oE "VALID: [0-9]+ day" | grep -oE "[0-9]+" | awk -v t="$EXPIRING_DAYS" '$1<=t{c++} END{print c+0}')
+# names of the expired/expiring for the alert body (cert name precedes its expiry line)
+SOON=$(echo "$RAW" | awk -v t="$EXPIRING_DAYS" '
+  /Certificate Name:/{n=$3}
+  /Expiry Date:/{ if($0 ~ /INVALID/){print n" (EXPIRED)"} else { match($0,/VALID: [0-9]+/); d=substr($0,RSTART+7,RLENGTH-7)+0; if(d<=t) print n" ("d"d)" } }' | head -25)
+
+if [ "$EXPIRED" -gt 0 ] || [ "$FAILURES" -gt 0 ] || [ "$EXPIRING" -gt 0 ]; then
+  if [ "$EXPIRED" -gt 0 ]; then VERDICT="FAIL"; else VERDICT="WARN"; fi
+else VERDICT="OK"; fi
+log "certs total=$TOTAL expired=$EXPIRED expiring<=${EXPIRING_DAYS}d=$EXPIRING renew_failures=$FAILURES → $VERDICT"
+
+# --- prior state (for worsening-only alerting) -------------------------------
+P_EXP=$(jq -r '.expired // 0' "$STATE" 2>/dev/null); P_ING=$(jq -r '.expiring // 0' "$STATE" 2>/dev/null)
+P_FAIL=$(jq -r '.failures // 0' "$STATE" 2>/dev/null); FIRST=1; [ -f "$STATE" ] && FIRST=0
+echo "{\"ts\":\"$TS\",\"verdict\":\"$VERDICT\",\"total\":$TOTAL,\"expired\":$EXPIRED,\"expiring\":$EXPIRING,\"failures\":$FAILURES,\"reachable\":true}" | tee "$LATEST" > "$STATE"
+
+# worsening = any of the three counts went UP (and not the very first baseline run)
+WORSE=0
+[ "$FIRST" -eq 0 ] && { [ "$EXPIRED" -gt "$P_EXP" ] && WORSE=1; [ "$EXPIRING" -gt "$P_ING" ] && WORSE=1; [ "$FAILURES" -gt "$P_FAIL" ] && WORSE=1; }
+if [ "$FIRST" -eq 1 ]; then log "baseline run — recorded, no alert"; exit 0; fi
+if [ "$WORSE" -eq 0 ]; then log "✓ no worsening (expired $P_EXP→$EXPIRED, expiring $P_ING→$EXPIRING, fail $P_FAIL→$FAILURES)"; exit 0; fi
+
+# --- ALERT (CNCP + George) on worsening only ---------------------------------
+CNCP="${CNCP_URL:-http://localhost:3333}"
+NOTE="[CERT-RENEWAL $(date +%H:%M) $VERDICT] Kamatera LE fleet worsened — expired=$EXPIRED (was $P_EXP), expiring<=${EXPIRING_DAYS}d=$EXPIRING (was $P_ING), last-run renew_failures=$FAILURES (was $P_FAIL) of $TOTAL certs. Soon: $(echo "$SOON"|tr '\n' ',')"
+curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
+  -d "$(jq -n --arg u "certrenew://kamatera" --arg note "$NOTE" '{url:$u,note:$note}')" >/dev/null 2>&1 \
+  && log "CNCP posted" || log "CNCP fail"
+if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
+  . "$HOME/.claude/skills/_shared/george-send.sh"
+  ROWS=$(echo "$SOON" | sed 's/&/\&amp;/g;s/</\&lt;/g' | awk 'NF{print "<div>• "$0"</div>"}')
+  BODY="<div style=\"font-family:-apple-system,sans-serif\"><h3 style=\"color:#b23b3b\">⚠ Kamatera SSL renewal worsening</h3><div><b>$EXPIRED</b> expired (was $P_EXP) · <b>$EXPIRING</b> expiring &le;${EXPIRING_DAYS}d (was $P_ING) · last renew run had <b>$FAILURES</b> failures (was $P_FAIL) of $TOTAL certs.</div><p style=\"color:#444;font-size:13px\">Renewal fix is drafted in ~/.claude/yolo-queue/pending-approval/ (HTTP-01 fails on redirect-only vhosts; DNS-01 blocked by CF token scope). Deleting/converting certs is Steve-gated.</p>$ROWS</div>"
+  RESP="$(george_send steve-office "${CERT_TO:-steve@designerwallcoverings.com}" "⚠ Kamatera SSL renewal $VERDICT — $(date +%H:%M)" "$BODY")"
+  echo "$RESP" | grep -q '"success":true' && log "emailed" || log "email failed"
+fi
+log "surfaced worsening cert-renewal alert"
(oldest) · back to Dw Cert Renewal Canary · auto-save: 2026-07-16T09:42:29 (1 files) — data/latest.json dce5572 →