`run.sh`

73 lines
#!/bin/zsh
# dw-cert-renewal-canary — READ-ONLY watchdog that keeps the Kamatera Let's Encrypt
# fleet "always clean". SSHes read-only to 45.61.58.125, reads `certbot certificates`
# + the last `certbot renew` failure summary, and classifies:
#   EXPIRED   = any cert past expiry (already down)
#   EXPIRING  = any cert < EXPIRING_DAYS to expiry (about to go down)
#   FAILURES  = the last renewal run's "N renew failure(s)" count
# Alerts (CNCP parking-lot card + George email) ONLY on a WORSENING transition
# (expired↑, expiring↑, or failures↑) — never on steady-state, recovery, or the
# first baseline run. Touches nothing on prod (pure reads). Writes data/latest.json
# so the meta-watchdog sees it ran. Born 2026-07-15 after the 99/225 renew-failure find.
set -u
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
HERE="${0:A:h}"; DATA="$HERE/data"; mkdir -p "$DATA"
STATE="$DATA/state.json"; LATEST="$DATA/latest.json"; LOG="$HERE/run.log"
HOST="${CERT_HOST:-root@45.61.58.125}"
EXPIRING_DAYS="${EXPIRING_DAYS:-14}"
TS="$(date '+%Y-%m-%d %H:%M:%S')"
log(){ echo "[$TS] $*" >>"$LOG"; }

# --- read-only pull from Kamatera --------------------------------------------
RAW="$(ssh -o ConnectTimeout=20 -o BatchMode=yes "$HOST" '
  echo "FAILURES=$(grep -oE "[0-9]+ renew failure" /var/log/letsencrypt/letsencrypt.log 2>/dev/null | tail -1 | grep -oE "^[0-9]+")"
  echo "TOTAL=$(ls /etc/letsencrypt/renewal/*.conf 2>/dev/null | wc -l | tr -d " ")"
  certbot certificates 2>/dev/null | grep -E "Certificate Name:|Expiry Date:"
' 2>/dev/null)"
if [ -z "$RAW" ]; then
  log "UNREACHABLE — no data (ssh failed); writing UNKNOWN, no alert"
  echo "{\"ts\":\"$TS\",\"verdict\":\"UNKNOWN\",\"reachable\":false}" > "$LATEST"
  exit 0
fi

FAILURES=$(echo "$RAW" | sed -n 's/^FAILURES=//p'); FAILURES=${FAILURES:-0}
TOTAL=$(echo "$RAW" | sed -n 's/^TOTAL=//p'); TOTAL=${TOTAL:-0}
# count expired (INVALID) + expiring (VALID: n days, n<=EXPIRING_DAYS)
EXPIRED=$(echo "$RAW" | grep -c "INVALID")
EXPIRING=$(echo "$RAW" | grep -oE "VALID: [0-9]+ day" | grep -oE "[0-9]+" | awk -v t="$EXPIRING_DAYS" '$1<=t{c++} END{print c+0}')
# names of the expired/expiring for the alert body (cert name precedes its expiry line)
SOON=$(echo "$RAW" | awk -v t="$EXPIRING_DAYS" '
  /Certificate Name:/{n=$3}
  /Expiry Date:/{ if($0 ~ /INVALID/){print n" (EXPIRED)"} else { match($0,/VALID: [0-9]+/); d=substr($0,RSTART+7,RLENGTH-7)+0; if(d<=t) print n" ("d"d)" } }' | head -25)

if [ "$EXPIRED" -gt 0 ] || [ "$FAILURES" -gt 0 ] || [ "$EXPIRING" -gt 0 ]; then
  if [ "$EXPIRED" -gt 0 ]; then VERDICT="FAIL"; else VERDICT="WARN"; fi
else VERDICT="OK"; fi
log "certs total=$TOTAL expired=$EXPIRED expiring<=${EXPIRING_DAYS}d=$EXPIRING renew_failures=$FAILURES → $VERDICT"

# --- prior state (for worsening-only alerting) -------------------------------
P_EXP=$(jq -r '.expired // 0' "$STATE" 2>/dev/null); P_ING=$(jq -r '.expiring // 0' "$STATE" 2>/dev/null)
P_FAIL=$(jq -r '.failures // 0' "$STATE" 2>/dev/null); FIRST=1; [ -f "$STATE" ] && FIRST=0
echo "{\"ts\":\"$TS\",\"verdict\":\"$VERDICT\",\"total\":$TOTAL,\"expired\":$EXPIRED,\"expiring\":$EXPIRING,\"failures\":$FAILURES,\"reachable\":true}" | tee "$LATEST" > "$STATE"

# worsening = any of the three counts went UP (and not the very first baseline run)
WORSE=0
[ "$FIRST" -eq 0 ] && { [ "$EXPIRED" -gt "$P_EXP" ] && WORSE=1; [ "$EXPIRING" -gt "$P_ING" ] && WORSE=1; [ "$FAILURES" -gt "$P_FAIL" ] && WORSE=1; }
if [ "$FIRST" -eq 1 ]; then log "baseline run — recorded, no alert"; exit 0; fi
if [ "$WORSE" -eq 0 ]; then log "✓ no worsening (expired $P_EXP→$EXPIRED, expiring $P_ING→$EXPIRING, fail $P_FAIL→$FAILURES)"; exit 0; fi

# --- ALERT (CNCP + George) on worsening only ---------------------------------
CNCP="${CNCP_URL:-http://localhost:3333}"
NOTE="[CERT-RENEWAL $(date +%H:%M) $VERDICT] Kamatera LE fleet worsened — expired=$EXPIRED (was $P_EXP), expiring<=${EXPIRING_DAYS}d=$EXPIRING (was $P_ING), last-run renew_failures=$FAILURES (was $P_FAIL) of $TOTAL certs. Soon: $(echo "$SOON"|tr '\n' ',')"
curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
  -d "$(jq -n --arg u "certrenew://kamatera" --arg note "$NOTE" '{url:$u,note:$note}')" >/dev/null 2>&1 \
  && log "CNCP posted" || log "CNCP fail"
if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
  . "$HOME/.claude/skills/_shared/george-send.sh"
  ROWS=$(echo "$SOON" | sed 's/&/\&amp;/g;s/</\&lt;/g' | awk 'NF{print "<div>• "$0"</div>"}')
  BODY="<div style=\"font-family:-apple-system,sans-serif\"><h3 style=\"color:#b23b3b\">⚠ Kamatera SSL renewal worsening</h3><div><b>$EXPIRED</b> expired (was $P_EXP) · <b>$EXPIRING</b> expiring &le;${EXPIRING_DAYS}d (was $P_ING) · last renew run had <b>$FAILURES</b> failures (was $P_FAIL) of $TOTAL certs.</div><p style=\"color:#444;font-size:13px\">Renewal fix is drafted in ~/.claude/yolo-queue/pending-approval/ (HTTP-01 fails on redirect-only vhosts; DNS-01 blocked by CF token scope). Deleting/converting certs is Steve-gated.</p>$ROWS</div>"
  RESP="$(george_send steve-office "${CERT_TO:-steve@designerwallcoverings.com}" "⚠ Kamatera SSL renewal $VERDICT — $(date +%H:%M)" "$BODY")"
  echo "$RESP" | grep -q '"success":true' && log "emailed" || log "email failed"
fi
log "surfaced worsening cert-renewal alert"