`review.sh`

58 lines
#!/bin/bash
# Hourly local-model leaderboard watcher.
# The WRAPPER (this deterministic script) gathers LOCAL cluster state.
# The claude review then runs with NO shell/Bash — only WebSearch/WebFetch/Read/Write
# (research + report + gated recommendation). READ-ONLY: never downloads, never changes the cluster.
set -uo pipefail
DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
EXO_MODELS_DIR="${EXO_MODELS_DIR:-$HOME/.exo/models}"
TS="$(date '+%FT%T%z')"
echo "[$TS] === review start ===" >> "$DIR/data/run.log"

# 1) wrapper collects local state into a file for claude to READ (no Bash needed in the agent)
{
  echo "# Local cluster state @ $TS"
  echo "## Loaded exo model (curl :52415/state):"
  curl -s -m8 http://127.0.0.1:52415/state 2>/dev/null \
    | python3 -c "import json,sys
try:
  d=json.load(sys.stdin); inst=d.get('instances',{})
  print(list(inst.values())[0]['MlxRingInstance']['shardAssignments']['modelId'] if inst else 'none loaded')
except Exception as e: print('state-unreachable')" 2>/dev/null
  echo "## Downloaded models (~/.exo/models):"
  ls -la "$EXO_MODELS_DIR" 2>/dev/null | awk '{print $5, $NF}' | grep -viE '^\s*$|caches|\.$'
  echo "## Hardware: 96GB M3 Ultra (primary) + 2x 32GB nodes; target = fits ~55GB weights single-node."
} > "$DIR/data/local-state.txt" 2>&1

# 2) claude reviews — NO Bash, NO Skill: research + read the state file + write report/recommendation
timeout 600 claude -p "$(cat "$DIR/review-prompt.md")" \
  --allowedTools "WebSearch,WebFetch,Read,Write" \
  >> "$DIR/data/run.log" 2>&1
rc=$?

# 3) DETERMINISTIC load of today's chosen model — ONLY if it's already downloaded.
#    Never downloads: if model weights are missing or incomplete, we leave the current model and rely
#    on the gated pending-approval memo. This is the wrapper's job, not the agent's.
CHOSEN="$(head -1 "$DIR/data/chosen-model.txt" 2>/dev/null | tr -d '[:space:]')"
if [ -n "$CHOSEN" ]; then
  DNAME="$(printf '%s' "$CHOSEN" | sed 's#/#--#g')"
  if python3 "$DIR/verify-model-weights.py" "$EXO_MODELS_DIR/$DNAME" >> "$DIR/data/run.log" 2>&1; then
    LOADED="$(curl -s -m8 http://127.0.0.1:52415/state 2>/dev/null | python3 -c "import json,sys
try:
  d=json.load(sys.stdin); i=d.get('instances',{})
  print(list(i.values())[0]['MlxRingInstance']['shardAssignments']['modelId'] if i else '')
except: print('')" 2>/dev/null)"
    if [ "$CHOSEN" != "$LOADED" ]; then
      echo "[$TS] loading TODAY's chosen (on disk): $CHOSEN (was: ${LOADED:-none})" >> "$DIR/data/run.log"
      curl -s -m600 http://127.0.0.1:52415/v1/chat/completions -H 'Content-Type: application/json' \
        -d "{\"model\":\"$CHOSEN\",\"messages\":[{\"role\":\"user\",\"content\":\"ok\"}],\"max_tokens\":1}" >/dev/null 2>&1
      echo "[$TS] chosen model load requested" >> "$DIR/data/run.log"
    else
      echo "[$TS] chosen $CHOSEN already loaded — no change" >> "$DIR/data/run.log"
    fi
  else
    echo "[$TS] chosen $CHOSEN weights missing/incomplete — keeping current model; download is gated (see pending-approval)" >> "$DIR/data/run.log"
  fi
fi
echo "[$TS] === review done (exit $rc) ===" >> "$DIR/data/run.log"