TK-10395 REOPEN: run publish-gate+alert BEFORE the push (a full Kamatera hung the push and swallowed the 07:10 alert); bound rsync with timeout; exit non-zero last

c46261711598655268db10542805e2158e44bc28 · 2026-08-11 07:16:22 -0700 · cre-agent
Files touched

M scripts/cron-run.sh
Diff

commit c46261711598655268db10542805e2158e44bc28
Author: cre-agent <steve@designerwallcoverings.com>
Date:   Tue Aug 11 07:16:22 2026 -0700

    TK-10395 REOPEN: run publish-gate+alert BEFORE the push (a full Kamatera hung the push and swallowed the 07:10 alert); bound rsync with timeout; exit non-zero last
---
 scripts/cron-run.sh | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/scripts/cron-run.sh b/scripts/cron-run.sh
index 72515da..9a6ea6f 100755
--- a/scripts/cron-run.sh
+++ b/scripts/cron-run.sh
@@ -17,22 +17,17 @@ node scripts/scan-leaks.mjs || { echo "LEAK GATE FAILED — aborting publish+pus
 # auto-post (Steve: auto-post on the nightly schedule). Runs on Mac3 where Norma/tiktok live.
 SOCIAL_AUTOPOST="${SOCIAL_AUTOPOST:-1}" node scripts/publish-social.mjs
 
-PUSH_HOST="${PUSH_HOST:-root@45.61.58.125}"
-PUSH_PATH="${PUSH_PATH:-/root/public-projects/dw-marketing-reels}"
-if ssh -o BatchMode=yes -o ConnectTimeout=6 "$PUSH_HOST" "test -d $PUSH_PATH" 2>/dev/null; then
-  rsync -az reels/ "$PUSH_HOST:$PUSH_PATH/reels/"
-  rsync -az data/ "$PUSH_HOST:$PUSH_PATH/data/"
-  echo "pushed reels+data -> $PUSH_HOST:$PUSH_PATH"
-fi
-
-# TK-10395 publish-result gate (LAST so it never blocks the push): the publisher exits 0 even
-# when the IG post fails, which is how the 2207076 failure went silently green. This makes the
-# nightly job report a non-zero exit (visible in launchd + reels-cron.log) when an ARMED IG post
-# did not actually land — incl. a 2207076 that survived the retry. Read-only, no network.
-# On failure it also PUSHES an alert (Steve-approved 2026-08-10 "PUSH OPEN"): CNCP parking-lot
-# card + George email to steve-office, so a silent no-post surfaces without reading the log.
+# TK-10395 publish-result gate — runs IMMEDIATELY after publish, BEFORE the push, so a hanging or
+# failing Kamatera push (e.g. host at 1% disk) can never suppress the alert (that exact ordering bug
+# swallowed the 2026-08-11 07:10 failure — the log died in the push and the gate never ran). The
+# publisher exits 0 even when the IG post fails (how 2207076 went silently green); this detects an
+# ARMED post that did NOT land and PUSHES an alert (Steve-approved "PUSH OPEN"): CNCP card + George
+# email. We record the failure and still do the push, then exit non-zero at the very end so launchd
+# sees the failure without losing the console push.
+PUBLISH_FAILED=0
 if ! node scripts/check-publish-status.mjs; then
-  MSG="TK-10395: nightly IG reel publish did NOT land (armed) on the $(date +%H:%M) run — likely a 2207076 that survived the 3x retry (possibly persistent -> gated Option C: IG re-auth). See reels.json publish.instagram + reels-cron.log."
+  PUBLISH_FAILED=1
+  MSG="TK-10395: nightly IG reel publish did NOT land (armed) on the $(date +%H:%M) run — a 2207076 that survived the 3x retry (now demonstrated PERSISTENT/flaky -> gated Option C: verify IG content-publish permission / re-auth). See reels.json publish.instagram + reels-cron.log."
   echo "IG PUBLISH CHECK FAILED — $MSG"
   curl -sS --max-time 10 "${CNCP_URL:-http://localhost:3333}/api/parking-lot" \
     -H 'Content-Type: application/json' \
@@ -44,5 +39,19 @@ if ! node scripts/check-publish-status.mjs; then
       "⚠ DW nightly reel IG publish FAILED — $(date +%Y-%m-%d)" \
       "<p>$MSG</p>" >/dev/null 2>&1 && echo "George alert sent" || echo "George send failed (non-fatal)"
   fi
-  exit 1
 fi
+
+# Push to the Kamatera console. Bounded so a full/hung host can't wedge the whole job (and can't
+# suppress the gate above, which already ran). rsync gets its own timeout + non-fatal guard.
+PUSH_HOST="${PUSH_HOST:-root@45.61.58.125}"
+PUSH_PATH="${PUSH_PATH:-/root/public-projects/dw-marketing-reels}"
+if ssh -o BatchMode=yes -o ConnectTimeout=6 "$PUSH_HOST" "test -d $PUSH_PATH" 2>/dev/null; then
+  timeout 120 rsync -az reels/ "$PUSH_HOST:$PUSH_PATH/reels/" || echo "reels rsync failed/timed out (non-fatal)"
+  timeout 120 rsync -az data/  "$PUSH_HOST:$PUSH_PATH/data/"  || echo "data rsync failed/timed out (non-fatal)"
+  echo "pushed reels+data -> $PUSH_HOST:$PUSH_PATH"
+else
+  echo "push skipped — $PUSH_HOST:$PUSH_PATH unreachable (non-fatal)"
+fi
+
+# Report the IG-publish failure to launchd LAST, after the push has had its chance.
+[ "$PUBLISH_FAILED" = 0 ] || { echo "nightly completed with IG PUBLISH FAILURE (see alert above)"; exit 1; }
← 9ab766d auto-data-snapshot: 2026-08-11T07:04:50 (1 data files) — dat · back to Dw Marketing Reels · auto-data-snapshot: 2026-08-11T07:35:48 (2 data files) — dat 176622f →