TK-11709: guard BLOCKs while exo-heavy-window lock is held (no Qwen re-POST stacked on the 40GB load); test 9/9

255a29319e9a38473e819ab2840603d7c8d8fd13 · 2026-09-26 09:04:00 -0700 · Steve Abrams

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Up8zEnKZ2CdnGTii6bDFgc

Files touched

M instance-load-guard.sh
M test-instance-load-guard.sh

Diff

commit 255a29319e9a38473e819ab2840603d7c8d8fd13
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Sep 26 09:04:00 2026 -0700

    TK-11709: guard BLOCKs while exo-heavy-window lock is held (no Qwen re-POST stacked on the 40GB load); test 9/9
    
    Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
    Claude-Session: https://claude.ai/code/session_01Up8zEnKZ2CdnGTii6bDFgc
---
 instance-load-guard.sh      | 14 +++++++++++++-
 test-instance-load-guard.sh |  7 +++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/instance-load-guard.sh b/instance-load-guard.sh
index fad0f26..84728c4 100755
--- a/instance-load-guard.sh
+++ b/instance-load-guard.sh
@@ -14,6 +14,8 @@
 #   2. memory       — kern.memorystatus_level (system-wide free %) < GUARD_MIN_MEMFREE (40)
 #   3. thrash       — >= GUARD_MAX_REPOSTS_PER_HOUR (4) re-POSTs already in the last 60 min
 #   4. NOT MEASURED — memfree or uptime unreadable (never assume it is safe to load 17GB)
+#   5. heavy window — ~/ai-cluster/scripts/exo-heavy-window.sh holds the lock (TK-11709): a
+#                     one-shot 40GB load is in flight; never stack the 17GB re-POST on top
 #
 # Test seam: ONLY with --test are GUARD_TEST_UPTIME / GUARD_TEST_MEMFREE / GUARD_TEST_ALERTS /
 # GUARD_TEST_NOW honoured. The launchd path never passes --test.
@@ -22,16 +24,26 @@ MIN_UPTIME=${GUARD_MIN_UPTIME_S:-900}
 MIN_MEMFREE=${GUARD_MIN_MEMFREE:-40}
 MAX_PER_HOUR=${GUARD_MAX_REPOSTS_PER_HOUR:-4}
 ALERTS_DEFAULT="$HOME/Projects/exo-cluster-watchdog/data/alerts.log"
+LOCK_DEFAULT="$HOME/Projects/exo-cluster-watchdog/data/heavy-window.lock"
 
 if [ "${1:-}" = "--test" ]; then
   uptime_s="${GUARD_TEST_UPTIME:-}"; memfree="${GUARD_TEST_MEMFREE:-}"
   alerts="${GUARD_TEST_ALERTS:-/dev/null}"; nowe="${GUARD_TEST_NOW:-$(date +%s)}"
+  lock="${GUARD_TEST_LOCK:-/nonexistent}"
 else
   nowe=$(date +%s)
   boot=$(/usr/sbin/sysctl -n kern.boottime 2>/dev/null | sed -E 's/^\{ sec = ([0-9]+),.*/\1/')
   if [[ "$boot" =~ ^[0-9]+$ ]]; then uptime_s=$((nowe - boot)); else uptime_s=""; fi
   memfree=$(/usr/sbin/sysctl -n kern.memorystatus_level 2>/dev/null)
-  alerts="$ALERTS_DEFAULT"
+  alerts="$ALERTS_DEFAULT"; lock="$LOCK_DEFAULT"
+fi
+
+# TK-11709: a live heavy-window lock (pid still running) wins over everything else
+if [ -f "$lock" ]; then
+  lpid=$(head -1 "$lock" 2>/dev/null)
+  if [[ "$lpid" =~ ^[0-9]+$ ]] && kill -0 "$lpid" 2>/dev/null; then
+    echo "BLOCK: heavy window in progress (pid $lpid holds $lock)"; exit 0
+  fi
 fi
 
 if ! [[ "$uptime_s" =~ ^[0-9]+$ ]] || ! [[ "$memfree" =~ ^[0-9]+$ ]]; then
diff --git a/test-instance-load-guard.sh b/test-instance-load-guard.sh
index 242aa7b..1e8d864 100755
--- a/test-instance-load-guard.sh
+++ b/test-instance-load-guard.sh
@@ -10,8 +10,8 @@ iso(){ python3 -c "import datetime,sys;print(datetime.datetime.fromtimestamp(int
 for i in 1 2 3 4 5; do echo "$(iso $((NOW - i*600))) WARN :: 0 instances on ring -> re-POSTed m (http 200)" >> "$T/thrash.log"; done
 echo "$(iso $((NOW - 7200))) WARN :: re-POSTed old (http 200)" > "$T/old.log"
 pass=0; fail=0
-check(){ # name expect(ALLOW|BLOCK) uptime memfree alerts
-  out=$(GUARD_TEST_NOW=$NOW GUARD_TEST_UPTIME="$3" GUARD_TEST_MEMFREE="$4" GUARD_TEST_ALERTS="$5" bash "$G" --test)
+check(){ # name expect(ALLOW|BLOCK) uptime memfree alerts [lock]
+  out=$(GUARD_TEST_LOCK="${6:-/nonexistent}" GUARD_TEST_NOW=$NOW GUARD_TEST_UPTIME="$3" GUARD_TEST_MEMFREE="$4" GUARD_TEST_ALERTS="$5" bash "$G" --test)
   if [[ "$out" == "$2"* ]]; then pass=$((pass+1)); echo "ok   $1 -> $out"; else fail=$((fail+1)); echo "FAIL $1 -> $out (expected $2)"; fi
 }
 check "healthy"              ALLOW 7200 60 "$T/empty.log"
@@ -21,4 +21,7 @@ check "memory 17% (11:45)"   BLOCK 7200 17 "$T/empty.log"
 check "thrash 5/hr"          BLOCK 7200 60 "$T/thrash.log"
 check "memfree unmeasured"   BLOCK 7200 "" "$T/empty.log"
 check "uptime unmeasured"    BLOCK ""   60 "$T/empty.log"
+echo $$ > "$T/live.lock"; echo 999999 > "$T/stale.lock"
+check "heavy-window lock live"  BLOCK 7200 60 "$T/empty.log" "$T/live.lock"
+check "heavy-window lock stale" ALLOW 7200 60 "$T/empty.log" "$T/stale.lock"
 echo "pass=$pass fail=$fail"; [ "$fail" = 0 ]

← 9cf6194 auto-data-snapshot: 2026-09-26T08:50:16 (2 data files) — dat · back to Exo Cluster Watchdog · auto-data-snapshot: 2026-09-26T09:21:27 (2 data files) — dat 14b6dbf →