`instance-load-guard.sh`

83 lines
#!/bin/bash
# instance-load-guard.sh — decide whether watchdog.sh may re-POST the wanted exo model
# instance right now. Prints exactly one line: "ALLOW" or "BLOCK: <reason>".  Exit 0 always.
#
# Why (TK-11996, 2026-09-25): the TK-11903 instance keepalive re-POSTed the 17GB
# Qwen3-30B instance ~200x/day (1,207 re-POSTs since 09-18) while the ring kept evicting it.
# On 09-25 each re-POST landed on a box already holding the SAME model in com.dw.mlx-qwen
# (~22GB) + dw-local-ai (~15GB): swap went 20GB -> 53GB within a minute of the 11:28
# CreateInstance, the Mac went unresponsive and was power-button reset at 11:35; after that
# boot the 11:41 + 11:47 re-POSTs drove mem-free to 17% and it was reset again at 11:52.
#
# Blocks the re-POST when ANY of:
#   1. boot grace   — uptime < GUARD_MIN_UPTIME_S (default 900s): post-boot RunAtLoad stampede
#   2. memory       — kern.memorystatus_level (system-wide free %) < GUARD_MIN_MEMFREE (40)
#   3. thrash       — >= GUARD_MAX_REPOSTS_PER_HOUR (4) re-POSTs already in the last 60 min
#   4. NOT MEASURED — memfree or uptime unreadable (never assume it is safe to load 17GB)
#   5. heavy window — ~/ai-cluster/scripts/exo-heavy-window.sh holds the lock (TK-11709): a
#                     one-shot 40GB load is in flight; never stack the 17GB re-POST on top
#
# Test seam: ONLY with --test are GUARD_TEST_UPTIME / GUARD_TEST_MEMFREE / GUARD_TEST_ALERTS /
# GUARD_TEST_NOW honoured. The launchd path never passes --test.
set -uo pipefail
MIN_UPTIME=${GUARD_MIN_UPTIME_S:-900}
MIN_MEMFREE=${GUARD_MIN_MEMFREE:-40}
MAX_PER_HOUR=${GUARD_MAX_REPOSTS_PER_HOUR:-4}
ALERTS_DEFAULT="$HOME/Projects/exo-cluster-watchdog/data/alerts.log"
LOCK_DEFAULT="$HOME/Projects/exo-cluster-watchdog/data/heavy-window.lock"

if [ "${1:-}" = "--test" ]; then
  uptime_s="${GUARD_TEST_UPTIME:-}"; memfree="${GUARD_TEST_MEMFREE:-}"
  alerts="${GUARD_TEST_ALERTS:-/dev/null}"; nowe="${GUARD_TEST_NOW:-$(date +%s)}"
  lock="${GUARD_TEST_LOCK:-/nonexistent}"
else
  nowe=$(date +%s)
  boot=$(/usr/sbin/sysctl -n kern.boottime 2>/dev/null | sed -E 's/^\{ sec = ([0-9]+),.*/\1/')
  if [[ "$boot" =~ ^[0-9]+$ ]]; then uptime_s=$((nowe - boot)); else uptime_s=""; fi
  memfree=$(/usr/sbin/sysctl -n kern.memorystatus_level 2>/dev/null)
  alerts="$ALERTS_DEFAULT"; lock="$LOCK_DEFAULT"
fi

# TK-11709: a live heavy-window lock (pid still running) wins over everything else
if [ -f "$lock" ]; then
  lpid=$(head -1 "$lock" 2>/dev/null)
  if [[ "$lpid" =~ ^[0-9]+$ ]] && kill -0 "$lpid" 2>/dev/null; then
    echo "BLOCK: heavy window in progress (pid $lpid holds $lock)"; exit 0
  fi
fi

if ! [[ "$uptime_s" =~ ^[0-9]+$ ]] || ! [[ "$memfree" =~ ^[0-9]+$ ]]; then
  echo "BLOCK: NOT MEASURED (uptime='${uptime_s}' memfree='${memfree}') — refusing to load a 17GB model blind"; exit 0
fi
if [ "$uptime_s" -lt "$MIN_UPTIME" ]; then
  echo "BLOCK: boot grace (uptime ${uptime_s}s < ${MIN_UPTIME}s)"; exit 0
fi
if [ "$memfree" -lt "$MIN_MEMFREE" ]; then
  echo "BLOCK: memory (system free ${memfree}% < ${MIN_MEMFREE}%)"; exit 0
fi
# count re-POSTs in the last hour (alerts.log lines start with an ISO-8601 UTC timestamp)
recent=$(NOWE="$nowe" python3 - "$alerts" <<'PY' 2>/dev/null
import os, sys, datetime
now = int(os.environ["NOWE"]); n = 0
try:
    for line in open(sys.argv[1], errors="replace"):
        if "re-POSTed" not in line: continue
        ts = line.split(" ", 1)[0]
        try:
            t = datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc).timestamp()
        except ValueError:
            continue
        if 0 <= now - t <= 3600: n += 1
except FileNotFoundError:
    pass
print(n)
PY
)
if ! [[ "$recent" =~ ^[0-9]+$ ]]; then
  echo "BLOCK: NOT MEASURED (re-POST history unreadable)"; exit 0
fi
if [ "$recent" -ge "$MAX_PER_HOUR" ]; then
  echo "BLOCK: thrash breaker (${recent} re-POSTs in last 60m >= ${MAX_PER_HOUR}; instance keeps getting evicted)"; exit 0
fi
echo "ALLOW (uptime ${uptime_s}s, free ${memfree}%, ${recent} re-POSTs/60m)"