← back to Kamatera Ops
TK-12164: cert-fleet-fix.sh — move failing LE certs to dns-cloudflare / shared webroot, dry-run gated, conf backup
0f750a960d082902cc09f2eb44f4492dec7f7c5c · 2026-09-24 15:05:39 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
Diff
commit 0f750a960d082902cc09f2eb44f4492dec7f7c5c
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 15:05:39 2026 -0700
TK-12164: cert-fleet-fix.sh — move failing LE certs to dns-cloudflare / shared webroot, dry-run gated, conf backup
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
cert-fleet-fix.sh | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 97 insertions(+)
diff --git a/cert-fleet-fix.sh b/cert-fleet-fix.sh
new file mode 100755
index 0000000..cef1379
--- /dev/null
+++ b/cert-fleet-fix.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# cert-fleet-fix.sh — TK-12164. Finds Let's Encrypt certs on Kamatera that are failing to renew
+# and moves each onto a validation method that actually reaches it, then renews it.
+#
+# Why failures happen here: public :80 traffic only lands on IP-bound vhosts
+# (listen 45.61.58.125:80), so certbot's "nginx" authenticator (which edits the site's own vhost)
+# and webroot paths other than /var/www/certbot (which 00-acme-default-80.conf serves) 404.
+#
+# Per lineage, each method is proven with a --dry-run against LE staging first (no rate-limit cost):
+# 1. zone on Cloudflare DNS -> dns-cloudflare (DNS-01; bypasses nginx entirely), trying each
+# /root/.secrets/cloudflare*.ini until one can edit the zone
+# 2. otherwise -> webroot /var/www/certbot, only if a probe file is reachable
+# Only a method whose dry-run passes is used for the real renewal. Nothing passes -> reported, untouched.
+#
+# Usage: cert-fleet-fix.sh [--days N] candidates = certs expiring within N days (default 30)
+# cert-fleet-fix.sh --report dry-runs only: list candidates + chosen method, renew nothing
+# cert-fleet-fix.sh --cron as default, but only proceeds at 01:xx America/Los_Angeles
+# (server is UTC; crontab fires at 08 and 09 UTC to cover DST)
+# Undo: renewal confs are backed up to /root/cert-backups/<ts>/ before any change:
+# cp /root/cert-backups/<ts>/*.conf /etc/letsencrypt/renewal/
+set -uo pipefail
+
+DAYS=30; REPORT=0; CRON=0
+while [ $# -gt 0 ]; do case "$1" in
+ --days) DAYS="$2"; shift 2;; --report) REPORT=1; shift;; --cron) CRON=1; shift;;
+ *) echo "unknown arg $1"; exit 2;; esac; done
+
+if [ "$CRON" = 1 ] && [ "$(TZ=America/Los_Angeles date +%H)" != 01 ]; then exit 0; fi
+
+TS=$(date -u +%Y%m%dT%H%M%SZ)
+LOG=/var/log/cert-fleet-fix.log
+BK=/root/cert-backups/$TS
+WEBROOT=/var/www/certbot
+CF_INIS=(/root/.secrets/cloudflare-master.ini /root/.secrets/cloudflare-dw.ini /root/.secrets/cloudflare.ini)
+say() { echo "$(date -u +%FT%TZ) $*" | tee -a "$LOG"; }
+
+# Registrable zone for a name (last two labels; fine for this fleet's .com names).
+zone_of() { echo "$1" | awk -F. '{print $(NF-1)"."$NF}'; }
+on_cloudflare() { dig +short NS "$(zone_of "$1")" @1.1.1.1 | grep -q 'ns.cloudflare.com'; }
+
+# Domains a lineage covers, from its live cert (authoritative, unlike parsing the conf).
+domains_of() { openssl x509 -noout -ext subjectAltName -in "/etc/letsencrypt/live/$1/cert.pem" 2>/dev/null \
+ | grep -oE 'DNS:[^,]+' | sed 's/DNS://' | tr '\n' ' '; }
+days_left() { echo $(( ( $(date -d "$(openssl x509 -enddate -noout -in "/etc/letsencrypt/live/$1/cert.pem" | cut -d= -f2)" +%s) - $(date +%s) ) / 86400 )); }
+dargs() { local a=""; for d in $1; do a="$a -d $d"; done; echo "$a"; }
+
+try_dns() { # lineage, domains -> prints the working ini
+ local lin=$1 doms=$2 ini
+ for ini in "${CF_INIS[@]}"; do
+ [ -f "$ini" ] || continue
+ if certbot certonly --dry-run --non-interactive --dns-cloudflare --dns-cloudflare-credentials "$ini" \
+ --dns-cloudflare-propagation-seconds 30 --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1; then
+ echo "$ini"; return 0; fi
+ done; return 1
+}
+try_webroot() { # lineage, domains
+ local lin=$1 doms=$2 d p=probe-tk12164-$$
+ mkdir -p "$WEBROOT/.well-known/acme-challenge"; echo ok > "$WEBROOT/.well-known/acme-challenge/$p"
+ for d in $doms; do
+ if [ "$(curl -s -m 8 -L "http://$d/.well-known/acme-challenge/$p")" != ok ]; then
+ rm -f "$WEBROOT/.well-known/acme-challenge/$p"; return 1; fi
+ done
+ rm -f "$WEBROOT/.well-known/acme-challenge/$p"
+ certbot certonly --dry-run --non-interactive --webroot -w "$WEBROOT" --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1
+}
+
+# Candidates: certs expiring within DAYS. certbot.timer renews at 30 days left, so a cert under
+# 30 days is one the timer has failed to renew.
+mapfile -t LINS < <(for c in /etc/letsencrypt/live/*/cert.pem; do
+ l=$(basename "$(dirname "$c")"); [ "$(days_left "$l")" -lt "$DAYS" ] && echo "$l"; done)
+
+say "=== run $TS candidates=${#LINS[@]} (expiring <${DAYS}d) report_only=$REPORT"
+if [ "$REPORT" = 0 ] && [ "${#LINS[@]}" -gt 0 ]; then
+ mkdir -p "$BK"; cp /etc/letsencrypt/renewal/*.conf "$BK/"; say "backup: $BK"; fi
+
+ok=0; fail=0; FAILED=()
+for lin in "${LINS[@]}"; do
+ doms=$(domains_of "$lin"); left=$(days_left "$lin"); first=${doms%% *}
+ if on_cloudflare "$first"; then
+ if ini=$(try_dns "$lin" "$doms"); then method="dns-cloudflare ($ini)"
+ [ "$REPORT" = 1 ] || certbot certonly --non-interactive --force-renewal --dns-cloudflare \
+ --dns-cloudflare-credentials "$ini" --dns-cloudflare-propagation-seconds 30 \
+ --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real dns-cloudflare"
+ else method="FAILED no Cloudflare token can edit $(zone_of "$first")"; fi
+ else
+ if try_webroot "$lin" "$doms"; then method="webroot $WEBROOT"
+ [ "$REPORT" = 1 ] || certbot certonly --non-interactive --force-renewal --webroot -w "$WEBROOT" \
+ --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real webroot"
+ else method="FAILED probe not reachable over http (non-Cloudflare DNS)"; fi
+ fi
+ case "$method" in FAILED*) fail=$((fail+1)); FAILED+=("$lin");; *) ok=$((ok+1));; esac
+ say "$lin | ${left}d | $doms| $method"
+done
+
+if [ "$REPORT" = 0 ] && [ "$ok" -gt 0 ]; then nginx -t >>"$LOG" 2>&1 && systemctl reload nginx && say "nginx reloaded"; fi
+say "=== done ok=$ok failed=$fail ${FAILED[*]:-}"
+[ "$fail" -eq 0 ] # non-zero exit when anything is still failing, so cron/canaries see it
← b44b00d initial scaffold
·
back to Kamatera Ops
·
TK-12164: wait for certbot lock instead of misreporting a ti a9c5ac8 →