`finish-exo-inference.sh`

46 lines
#!/usr/bin/env bash
# finish-exo-inference.sh — run ON .151 AFTER Xcode is installed + selected.
# Chains: build mlx fork -> restart exo -> load both models -> smoke-test.
# Everything after the (Steve-only) Apple-ID install + sudo license steps.
set -uo pipefail
API="http://localhost:52415"
say(){ printf '\n=== %s ===\n' "$1"; }

# 0. gate: Metal toolchain present (Steve did the sudo xcode-select + license)
if ! xcrun -sdk macosx metal --version >/dev/null 2>&1; then
  echo "!! Metal compiler not found yet. Finish these first (Steve):"
  echo "   sudo xcode-select -s /Applications/Xcode.app/Contents/Developer"
  echo "   sudo xcodebuild -license accept"
  exit 1
fi

say "1/3 build mlx fork + enable inference"
bash "$HOME/Projects/exo-keepalive/enable-exo-inference.sh" || exit 1

say "2/3 load models (text workhorse + vision)"
load_model(){
  local M="$1"
  curl -s -m10 "$API/instance/previews?model_id=$M" 2>/dev/null > /tmp/prev.json
  python3 - "$M" <<'PY'
import json,sys
d=json.load(open('/tmp/prev.json'))
ok=[p for p in d.get('previews',[]) if p.get('error') is None]
if ok: json.dump({'instance':ok[0]['instance']}, open('/tmp/inst.json','w')); print(f"  {sys.argv[1]}: {len(ok)} viable placement(s) -> creating")
else: print(f"  {sys.argv[1]}: NO viable placement (not enough RAM?)"); sys.exit(1)
PY
  [ -s /tmp/inst.json ] && curl -s -m15 -X POST "$API/instance" -H 'Content-Type: application/json' -d @/tmp/inst.json >/dev/null 2>&1
}
# vision first (small, fast), then the text workhorse (bigger download)
load_model "mlx-community/Qwen3-VL-4B-Instruct-4bit"
load_model "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"

say "3/3 smoke-test (waits for download+load, then generates)"
for i in $(seq 1 120); do   # up to ~30min for the 80B download
  out=$(curl -s -m120 -X POST "$API/v1/chat/completions" -H 'Content-Type: application/json' \
    -d '{"model":"mlx-community/Qwen3-VL-4B-Instruct-4bit","messages":[{"role":"user","content":"Reply with exactly: EXO_OK"}],"max_tokens":300,"stream":false}' 2>/dev/null \
    | python3 -c "import sys,json;print((json.load(sys.stdin).get('choices',[{}])[0].get('message',{}).get('content') or '').strip())" 2>/dev/null)
  if echo "$out" | grep -q "EXO_OK"; then echo "  ✅ exo inference LIVE — VL model answered: $out"; break; fi
  echo "  [$i] loading... (VL says: '${out:0:30}')"; sleep 15
done
echo "== done. exo cluster now serves local inference at $API/v1/chat/completions =="