`cert-fleet-fix.sh`

113 lines
#!/usr/bin/env bash
# cert-fleet-fix.sh — TK-12164. Finds Let's Encrypt certs on Kamatera that are failing to renew
# and moves each onto a validation method that actually reaches it, then renews it.
#
# Why failures happen here: public :80 traffic only lands on IP-bound vhosts
# (listen 45.61.58.125:80), so certbot's "nginx" authenticator (which edits the site's own vhost)
# and webroot paths other than /var/www/certbot (which 00-acme-default-80.conf serves) 404.
#
# Per lineage, each method is proven with a --dry-run against LE staging first (no rate-limit cost):
#   1. zone on Cloudflare DNS  -> dns-cloudflare (DNS-01; bypasses nginx entirely), trying each
#      /root/.secrets/cloudflare*.ini until one can edit the zone
#   2. otherwise               -> webroot /var/www/certbot, only if a probe file is reachable
# Only a method whose dry-run passes is used for the real renewal. Nothing passes -> reported, untouched.
#
# Usage: cert-fleet-fix.sh [--days N]   candidates = certs expiring within N days (default 30)
#        cert-fleet-fix.sh --report     dry-runs only: list candidates + chosen method, renew nothing
#        cert-fleet-fix.sh --cron       as default, but only proceeds at 01:xx America/Los_Angeles
#                                       (server is UTC; crontab fires at 08 and 09 UTC to cover DST)
# Undo:  renewal confs are backed up to /root/cert-backups/<ts>/ before any change:
#        cp /root/cert-backups/<ts>/*.conf /etc/letsencrypt/renewal/
set -uo pipefail

DAYS=30; REPORT=0; CRON=0
while [ $# -gt 0 ]; do case "$1" in
  --days) DAYS="$2"; shift 2;; --report) REPORT=1; shift;; --cron) CRON=1; shift;;
  *) echo "unknown arg $1"; exit 2;; esac; done

if [ "$CRON" = 1 ] && [ "$(TZ=America/Los_Angeles date +%H)" != 01 ]; then exit 0; fi

TS=$(date -u +%Y%m%dT%H%M%SZ)
LOG=/var/log/cert-fleet-fix.log
BK=/root/cert-backups/$TS
WEBROOT=/var/www/certbot
CF_INIS=(/root/.secrets/cloudflare-master.ini /root/.secrets/cloudflare-dw.ini /root/.secrets/cloudflare.ini)
say() { echo "$(date -u +%FT%TZ) $*" | tee -a "$LOG"; }

# Registrable zone for a name (last two labels; fine for this fleet's .com names).
zone_of() { echo "$1" | awk -F. '{print $(NF-1)"."$NF}'; }
on_cloudflare() { dig +short NS "$(zone_of "$1")" @1.1.1.1 | grep -q 'ns.cloudflare.com'; }

# Domains a lineage covers, from its live cert (authoritative, unlike parsing the conf).
domains_of() { openssl x509 -noout -ext subjectAltName -in "/etc/letsencrypt/live/$1/cert.pem" 2>/dev/null \
  | grep -oE 'DNS:[^,]+' | sed 's/DNS://' | tr '\n' ' '; }
days_left() { echo $(( ( $(date -d "$(openssl x509 -enddate -noout -in "/etc/letsencrypt/live/$1/cert.pem" | cut -d= -f2)" +%s) - $(date +%s) ) / 86400 )); }
dargs() { local a=""; for d in $1; do a="$a -d $d"; done; echo "$a"; }

# certbot allows one instance at a time; certbot.timer fires twice a day and a collision makes every
# attempt fail instantly with "Another instance of Certbot is already running" — which must never be
# misread as a token/probe failure. Wait (up to 60 min) for any other certbot process to exit.
wait_lock() {
  local waited=0
  while pgrep -f '(^|/)certbot( |$)' >/dev/null; do
    [ "$waited" -ge 3600 ] && { say "certbot lock still held after 60m"; return 1; }
    sleep 15; waited=$((waited+15))
  done
}
cb() { wait_lock || return 1; certbot "$@"; }

try_dns() { # lineage, domains -> prints the working ini
  local lin=$1 doms=$2 ini
  for ini in "${CF_INIS[@]}"; do
    [ -f "$ini" ] || continue
    if cb certonly --dry-run --non-interactive --dns-cloudflare --dns-cloudflare-credentials "$ini" \
        --dns-cloudflare-propagation-seconds 30 --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1; then
      echo "$ini"; return 0; fi
  done; return 1
}
try_webroot() { # lineage, domains
  local lin=$1 doms=$2 d p=probe-tk12164-$$
  mkdir -p "$WEBROOT/.well-known/acme-challenge"; echo ok > "$WEBROOT/.well-known/acme-challenge/$p"
  for d in $doms; do
    if [ "$(curl -s -m 8 -L "http://$d/.well-known/acme-challenge/$p")" != ok ]; then
      rm -f "$WEBROOT/.well-known/acme-challenge/$p"; echo "probe unreachable at http://$d"; return 1; fi
  done
  rm -f "$WEBROOT/.well-known/acme-challenge/$p"
  # Distinct from a probe failure: the route works but Let's Encrypt staging still refused.
  cb certonly --dry-run --non-interactive --webroot -w "$WEBROOT" --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 \
    || { echo "staging dry-run refused (route OK) — see $LOG"; return 1; }
}

# Candidates: certs expiring within DAYS. certbot.timer renews at 30 days left, so a cert under
# 30 days is one the timer has failed to renew.
# Most urgent first (expired, then fewest days left), so a long run fixes what's on fire before the rest.
mapfile -t LINS < <(for c in /etc/letsencrypt/live/*/cert.pem; do
  l=$(basename "$(dirname "$c")"); d=$(days_left "$l"); [ "$d" -lt "$DAYS" ] && echo "$d $l"; done \
  | sort -n | cut -d' ' -f2)

say "=== run $TS candidates=${#LINS[@]} (expiring <${DAYS}d) report_only=$REPORT"
if [ "$REPORT" = 0 ] && [ "${#LINS[@]}" -gt 0 ]; then
  mkdir -p "$BK"; cp /etc/letsencrypt/renewal/*.conf "$BK/"; say "backup: $BK"; fi

ok=0; fail=0; FAILED=()
for lin in "${LINS[@]}"; do
  doms=$(domains_of "$lin"); left=$(days_left "$lin"); first=${doms%% *}
  if on_cloudflare "$first"; then
    if ini=$(try_dns "$lin" "$doms"); then method="dns-cloudflare ($ini)"
      [ "$REPORT" = 1 ] || cb certonly --non-interactive --force-renewal --dns-cloudflare \
        --dns-cloudflare-credentials "$ini" --dns-cloudflare-propagation-seconds 30 \
        --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real dns-cloudflare"
    else method="FAILED no Cloudflare token can edit $(zone_of "$first")"; fi  else
    if why=$(try_webroot "$lin" "$doms"); then method="webroot $WEBROOT"
      [ "$REPORT" = 1 ] || cb certonly --non-interactive --force-renewal --webroot -w "$WEBROOT" \
        --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real webroot"
    else method="FAILED webroot: $why"; fi
  fi
  case "$method" in FAILED*) fail=$((fail+1)); FAILED+=("$lin");; *) ok=$((ok+1));; esac
  say "$lin | ${left}d | $doms| $method"
done

if [ "$REPORT" = 0 ] && [ "$ok" -gt 0 ]; then nginx -t >>"$LOG" 2>&1 && systemctl reload nginx && say "nginx reloaded"; fi
say "=== done ok=$ok failed=$fail ${FAILED[*]:-}"
[ "$fail" -eq 0 ]   # non-zero exit when anything is still failing, so cron/canaries see it