← back to Kamatera Ops
TK-12164: wait for certbot lock instead of misreporting a timer collision as a token failure
a9c5ac83b341036d9d54f885658d2dff7d4fccda · 2026-09-24 15:11:49 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Files touched
Diff
commit a9c5ac83b341036d9d54f885658d2dff7d4fccda
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 15:11:49 2026 -0700
TK-12164: wait for certbot lock instead of misreporting a timer collision as a token failure
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
cert-fleet-fix.sh | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/cert-fleet-fix.sh b/cert-fleet-fix.sh
index cef1379..d1ae669 100755
--- a/cert-fleet-fix.sh
+++ b/cert-fleet-fix.sh
@@ -44,11 +44,23 @@ domains_of() { openssl x509 -noout -ext subjectAltName -in "/etc/letsencrypt/liv
days_left() { echo $(( ( $(date -d "$(openssl x509 -enddate -noout -in "/etc/letsencrypt/live/$1/cert.pem" | cut -d= -f2)" +%s) - $(date +%s) ) / 86400 )); }
dargs() { local a=""; for d in $1; do a="$a -d $d"; done; echo "$a"; }
+# certbot allows one instance at a time; certbot.timer fires twice a day and a collision makes every
+# attempt fail instantly with "Another instance of Certbot is already running" — which must never be
+# misread as a token/probe failure. Wait (up to 60 min) for any other certbot process to exit.
+wait_lock() {
+ local waited=0
+ while pgrep -f '(^|/)certbot( |$)' >/dev/null; do
+ [ "$waited" -ge 3600 ] && { say "certbot lock still held after 60m"; return 1; }
+ sleep 15; waited=$((waited+15))
+ done
+}
+cb() { wait_lock || return 1; certbot "$@"; }
+
try_dns() { # lineage, domains -> prints the working ini
local lin=$1 doms=$2 ini
for ini in "${CF_INIS[@]}"; do
[ -f "$ini" ] || continue
- if certbot certonly --dry-run --non-interactive --dns-cloudflare --dns-cloudflare-credentials "$ini" \
+ if cb certonly --dry-run --non-interactive --dns-cloudflare --dns-cloudflare-credentials "$ini" \
--dns-cloudflare-propagation-seconds 30 --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1; then
echo "$ini"; return 0; fi
done; return 1
@@ -61,7 +73,7 @@ try_webroot() { # lineage, domains
rm -f "$WEBROOT/.well-known/acme-challenge/$p"; return 1; fi
done
rm -f "$WEBROOT/.well-known/acme-challenge/$p"
- certbot certonly --dry-run --non-interactive --webroot -w "$WEBROOT" --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1
+ cb certonly --dry-run --non-interactive --webroot -w "$WEBROOT" --cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1
}
# Candidates: certs expiring within DAYS. certbot.timer renews at 30 days left, so a cert under
@@ -78,13 +90,13 @@ for lin in "${LINS[@]}"; do
doms=$(domains_of "$lin"); left=$(days_left "$lin"); first=${doms%% *}
if on_cloudflare "$first"; then
if ini=$(try_dns "$lin" "$doms"); then method="dns-cloudflare ($ini)"
- [ "$REPORT" = 1 ] || certbot certonly --non-interactive --force-renewal --dns-cloudflare \
+ [ "$REPORT" = 1 ] || cb certonly --non-interactive --force-renewal --dns-cloudflare \
--dns-cloudflare-credentials "$ini" --dns-cloudflare-propagation-seconds 30 \
--cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real dns-cloudflare"
else method="FAILED no Cloudflare token can edit $(zone_of "$first")"; fi
else
if try_webroot "$lin" "$doms"; then method="webroot $WEBROOT"
- [ "$REPORT" = 1 ] || certbot certonly --non-interactive --force-renewal --webroot -w "$WEBROOT" \
+ [ "$REPORT" = 1 ] || cb certonly --non-interactive --force-renewal --webroot -w "$WEBROOT" \
--cert-name "$lin" $(dargs "$doms") >>"$LOG" 2>&1 || method="FAILED-real webroot"
else method="FAILED probe not reachable over http (non-Cloudflare DNS)"; fi
fi
← 0f750a9 TK-12164: cert-fleet-fix.sh — move failing LE certs to dns-c
·
back to Kamatera Ops
·
TK-12164: separate 'probe unreachable' from 'staging dry-run 47249a7 →