← back to Exo Preflight
exo-preflight v1.0.0 — cluster-health gate for the model-routing policy
ee8a075065605726300bcdf7f8702627f671d5e1 · 2026-08-13 11:25:21 -0700 · Steve Abrams
Reads exo /state, judges HEALTHY/DEGRADED/UNREACHABLE (exit 0/2/3) so pipelines
never silently route a production-feeding job to a degraded ring. Closes the
contrarian's fix #2 from DTD verdict A.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A .gitignoreA README.mdA preflight.py
Diff
commit ee8a075065605726300bcdf7f8702627f671d5e1
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Aug 13 11:25:21 2026 -0700
exo-preflight v1.0.0 — cluster-health gate for the model-routing policy
Reads exo /state, judges HEALTHY/DEGRADED/UNREACHABLE (exit 0/2/3) so pipelines
never silently route a production-feeding job to a degraded ring. Closes the
contrarian's fix #2 from DTD verdict A.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
.gitignore | 9 +++
README.md | 43 ++++++++++++++
preflight.py | 179 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 231 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..47ff405
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+__pycache__/
+*.pyc
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d77634d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,43 @@
+# exo-preflight
+
+Cluster-health **gate** for the tiered-hybrid model-routing policy (DTD verdict A, 2026-08-13).
+
+Before a pipeline routes a **production-feeding** job to the exo ring at "$0 local",
+it must confirm the ring can actually hold the load. A degraded cluster (a node
+offline, the requested model not loaded) must **not** silently proceed — the caller
+queues the job or escalates to a paid API **and logs the fallback**. This script is
+that check. It's read-only: it only GETs exo's `/state`.
+
+## Usage
+
+```sh
+python3 preflight.py # is the ring healthy for any bulk job?
+python3 preflight.py --model llama-3.1-70b # ...and is THIS model loaded + ready?
+python3 preflight.py --json # machine-readable
+python3 preflight.py --expect-nodes 4 # require the full 4-node fleet
+```
+
+### Exit codes
+| Code | Verdict | Caller should… |
+|---|---|---|
+| `0` | HEALTHY | route the job to the ring ($0 local) |
+| `2` | DEGRADED | queue the job **or** escalate to a paid API + log the fallback |
+| `3` | UNREACHABLE | treat as DEGRADED |
+
+### In a pipeline
+```sh
+if python3 /Users/macstudio3/Projects/exo-preflight/preflight.py --model llama-3.1-70b -q; then
+ run_on_ring # $0 local
+else
+ escalate_to_paid_and_log # per the routing policy — never silently proceed
+fi
+```
+
+## Notes
+- Default healthy baseline is **3 live nodes** — the 4th M1 Max is often offline by
+ design. Override with `--expect-nodes N` or `EXO_EXPECT_NODES`.
+- OS-build mismatch across nodes is surfaced as **informational** (fine for routing;
+ it only blocks RDMA), never a hard fail.
+- Zero dependencies (Python stdlib only). Reads `http://<node>:52415/state`.
+
+See the standing policy + cluster facts in memory: `model-routing-policy`, `exo-cluster-topology`.
diff --git a/preflight.py b/preflight.py
new file mode 100644
index 0000000..e55a168
--- /dev/null
+++ b/preflight.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+exo-preflight — cluster-health gate for the model-routing policy.
+
+Contrarian's fix #2 (DTD verdict A, 2026-08-13): before a pipeline routes a
+PRODUCTION-FEEDING job to the exo ring at "$0 local", it must verify the ring
+can actually hold the load. A degraded cluster (a node offline, the requested
+model not loaded) must NOT silently proceed — the caller queues or escalates to
+a paid API and LOGS it.
+
+This script is that gate. It reads exo's /state, judges HEALTHY / DEGRADED /
+UNREACHABLE, prints a one-line verdict, and sets its exit code so a shell/py
+caller can branch. It touches nothing — pure read.
+
+Exit codes:
+ 0 HEALTHY -> safe to route the job to the ring ($0 local)
+ 2 DEGRADED -> caller must queue or escalate-to-paid + log the fallback
+ 3 UNREACHABLE -> ring not answering; treat as DEGRADED
+
+Usage:
+ preflight.py [--node HOST] [--model MODELID] [--expect-nodes N] [--json] [-q]
+
+Examples:
+ # Is the ring healthy enough for any bulk job right now?
+ preflight.py
+
+ # Is Llama-70B specifically loaded + all its shards ready?
+ preflight.py --model llama-3.1-70b
+
+ # In a pipeline:
+ if python3 preflight.py --model llama-3.1-70b -q; then
+ run_on_ring # $0 local
+ else
+ escalate_to_paid_and_log # per the routing policy
+ fi
+"""
+import argparse, json, os, sys, urllib.request
+
+# The two nodes most likely to be up (this box's loopback first, then the M1 Max
+# that is usually online). We try them in order until one answers.
+DEFAULT_NODES = ["127.0.0.1", "192.168.1.133", "192.168.1.54"]
+PORT = 52415
+# Normal live-ring size. The 4th M1 Max is often offline BY DESIGN, so the
+# baseline "healthy" count is 3. Override with --expect-nodes or EXO_EXPECT_NODES.
+DEFAULT_EXPECT = int(os.environ.get("EXO_EXPECT_NODES", "3"))
+
+
+def fetch_state(node, timeout=8):
+ url = f"http://{node}:{PORT}/state"
+ with urllib.request.urlopen(url, timeout=timeout) as r:
+ return json.load(r)
+
+
+def get_state(nodes):
+ """Try each node until one returns /state. Returns (state, node) or (None, None)."""
+ for n in nodes:
+ try:
+ return fetch_state(n), n
+ except Exception:
+ continue
+ return None, None
+
+
+def runner_ready(runners, rid):
+ """A runner is ready only if its entry is the RunnerReady variant."""
+ v = runners.get(rid)
+ return isinstance(v, dict) and "RunnerReady" in v
+
+
+def model_readiness(state, want):
+ """
+ Find instances whose modelId matches `want` (exact, or case-insensitive
+ substring) and report whether ALL their shard-runners are RunnerReady.
+ Returns (matched_model_id_or_None, all_shards_ready_bool, ready_count, total_count).
+ """
+ runners = state.get("runners", {}) or {}
+ instances = state.get("instances", {}) or {}
+ w = want.lower()
+ matched_id = None
+ total = ready = 0
+ for inst in instances.values():
+ ri = inst.get("MlxRingInstance") or {}
+ sa = ri.get("shardAssignments") or {}
+ mid = sa.get("modelId", "")
+ if mid and (mid == want or w in mid.lower()):
+ matched_id = mid
+ for rid in (sa.get("runnerToShard") or {}).keys():
+ total += 1
+ if runner_ready(runners, rid):
+ ready += 1
+ return matched_id, (total > 0 and ready == total), ready, total
+
+
+def main():
+ ap = argparse.ArgumentParser(description="exo cluster-health preflight gate")
+ ap.add_argument("--node", help="specific node host to query (else auto-try)")
+ ap.add_argument("--model", help="require this model loaded + all shards ready")
+ ap.add_argument("--expect-nodes", type=int, default=DEFAULT_EXPECT,
+ help=f"min live ring nodes for HEALTHY (default {DEFAULT_EXPECT})")
+ ap.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+ ap.add_argument("-q", "--quiet", action="store_true", help="suppress human line")
+ args = ap.parse_args()
+
+ nodes = [args.node] if args.node else DEFAULT_NODES
+ state, served_by = get_state(nodes)
+
+ result = {"verdict": None, "reason": None, "served_by": served_by,
+ "live_nodes": None, "expect_nodes": args.expect_nodes,
+ "model": args.model, "model_ready": None, "rdma_enabled": None,
+ "os_mismatch": None}
+
+ if state is None:
+ result["verdict"] = "UNREACHABLE"
+ result["reason"] = f"no exo node answered /state on {nodes}"
+ emit(result, args)
+ return 3
+
+ live = state.get("topology", {}).get("nodes", []) or []
+ result["live_nodes"] = len(live)
+
+ ids = state.get("nodeIdentities", {}) or {}
+ rdma = state.get("nodeRdmaCtl", {}) or {}
+ result["rdma_enabled"] = any(v.get("enabled") for v in rdma.values())
+ # Informational: do the live nodes share one macOS build? (RDMA needs it; routing doesn't)
+ builds = {ids.get(n, {}).get("osBuildVersion") for n in live if ids.get(n)}
+ result["os_mismatch"] = len([b for b in builds if b]) > 1
+
+ reasons = []
+ degraded = False
+
+ if len(live) < args.expect_nodes:
+ degraded = True
+ reasons.append(f"only {len(live)}/{args.expect_nodes} ring nodes live")
+
+ if args.model:
+ mid, ready, rc, tc = model_readiness(state, args.model)
+ result["model_ready"] = ready
+ if mid is None:
+ degraded = True
+ reasons.append(f"model '{args.model}' not loaded on the ring")
+ elif not ready:
+ degraded = True
+ reasons.append(f"model '{mid}' shards not all ready ({rc}/{tc})")
+
+ if degraded:
+ result["verdict"] = "DEGRADED"
+ result["reason"] = "; ".join(reasons)
+ emit(result, args)
+ return 2
+
+ result["verdict"] = "HEALTHY"
+ ok = f"{len(live)} nodes live"
+ if args.model:
+ ok += f"; model '{args.model}' ready"
+ result["reason"] = ok
+ emit(result, args)
+ return 0
+
+
+def emit(result, args):
+ if args.json:
+ print(json.dumps(result))
+ return
+ if args.quiet:
+ return
+ icon = {"HEALTHY": "✅", "DEGRADED": "⚠️", "UNREACHABLE": "✖"}.get(result["verdict"], "?")
+ line = f"{icon} exo {result['verdict']}: {result['reason']}"
+ extras = []
+ if result.get("os_mismatch"):
+ extras.append("OS builds differ across nodes (fine for routing, blocks RDMA)")
+ if result.get("rdma_enabled") is False:
+ extras.append("RDMA off")
+ if extras:
+ line += " [" + "; ".join(extras) + "]"
+ print(line, file=(sys.stderr if result["verdict"] != "HEALTHY" else sys.stdout))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
(oldest)
·
back to Exo Preflight
·
(newest)