← back to Local Model Leaderboard Watch
battle: 80B-A3B-4bit 12/12@21tok/s vs 35B 11/12@55tok/s — head-to-head results + harness fixes
bc626c5e1883b79115b03fe23a45f42fb7a5a198 · 2026-08-18 10:03:15 -0700 · Steve Abrams
Files touched
M battle/80b-watch.logA battle/80b.jsonA battle/run-80b-when-ready.sh
Diff
commit bc626c5e1883b79115b03fe23a45f42fb7a5a198
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Aug 18 10:03:15 2026 -0700
battle: 80B-A3B-4bit 12/12@21tok/s vs 35B 11/12@55tok/s — head-to-head results + harness fixes
---
battle/80b-watch.log | 38 +++++++++++
battle/80b.json | 150 +++++++++++++++++++++++++++++++++++++++++++
battle/run-80b-when-ready.sh | 28 ++++++++
3 files changed, 216 insertions(+)
diff --git a/battle/80b-watch.log b/battle/80b-watch.log
index 2088a3a..5814fe4 100644
--- a/battle/80b-watch.log
+++ b/battle/80b-watch.log
@@ -7,3 +7,41 @@
[09:41:34] dl=19682MB probe=''
[09:42:19] dl=20770MB probe=''
[09:43:04] dl=21954MB probe=''
+[09:43:49] dl=23122MB probe=''
+[09:44:34] dl=24290MB probe=''
+[09:45:19] dl=25442MB probe=''
+[09:46:04] dl=26610MB probe=''
+[09:46:49] dl=27762MB probe=''
+[09:47:34] dl=28930MB probe=''
+[09:48:19] dl=30050MB probe=''
+[09:49:04] dl=31186MB probe=''
+[09:49:49] dl=32354MB probe=''
+[09:50:34] dl=33506MB probe=''
+[09:51:19] dl=34642MB probe=''
+[09:52:04] dl=35810MB probe=''
+[09:52:49] dl=36914MB probe=''
+[09:53:34] dl=38125MB probe=''
+[09:54:19] dl=39281MB probe=''
+[09:55:04] dl=40449MB probe=''
+[09:55:49] dl=41393MB probe=''
+[09:56:34] dl=42289MB probe=''
+[09:57:20] dl=42815MB probe=''
+[09:58:04] dl=42815MB probe='ready'
+[09:58:04] SERVING — running eval
+
+=== mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit ===
+score 12/12 | tok/s 21.4 | longctx placed=true pass=true (~33990 tok)
+byCat: json 3/3 classify 3/3 constraint 2/2 reason 2/2 code 2/2
+ ✅ json_person [object Object]
+ ✅ json_order [object Object]
+ ✅ json_list [object Object]
+ ✅ cls_sentiment negative
+ ✅ cls_ticket billing
+ ✅ cls_spam yes
+ ✅ cons_oneword Paris
+ ✅ cons_numonly 391
+ ✅ reason_word wall: $ 135 - 108 = 27 $ sq ft. ### ✅ Final Answer: **27**
+ ✅ reason_logic Mia
+ ✅ code_sum function sumTo(n){let s=0;for(let i=1;i<=n;i++)s+=i;return s;}
+ ✅ code_null function total(o){return (o.price || 0) * (o.qty || 0);}
+[09:59:04] EVAL DONE — 80b.json written
diff --git a/battle/80b.json b/battle/80b.json
new file mode 100644
index 0000000..b4ed126
--- /dev/null
+++ b/battle/80b.json
@@ -0,0 +1,150 @@
+{
+ "model": "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit",
+ "when": "2026-08-18T16:59:04.261Z",
+ "score": "12/12",
+ "passed": 12,
+ "total": 12,
+ "tok_per_s": 21.4,
+ "tok_s_ms": 7526,
+ "longctx": {
+ "placed": true,
+ "pass": true,
+ "got": "MAUVE-7719",
+ "ms": 27782,
+ "approx_ctx_tokens": 33990
+ },
+ "byCat": {
+ "json": {
+ "pass": 3,
+ "tot": 3
+ },
+ "classify": {
+ "pass": 3,
+ "tot": 3
+ },
+ "constraint": {
+ "pass": 2,
+ "tot": 2
+ },
+ "reason": {
+ "pass": 2,
+ "tot": 2
+ },
+ "code": {
+ "pass": 2,
+ "tot": 2
+ }
+ },
+ "results": [
+ {
+ "id": "json_person",
+ "cat": "json",
+ "pass": true,
+ "got": {
+ "name": "Maria Gonzalez",
+ "age": 34,
+ "city": "Chicago"
+ },
+ "ms": 2179,
+ "out": "{\"name\": \"Maria Gonzalez\", \"age\": 34, \"city\": \"Chicago\"}"
+ },
+ {
+ "id": "json_order",
+ "cat": "json",
+ "pass": true,
+ "got": {
+ "sku": "CORK-500210",
+ "qty": 3,
+ "price_usd": 59.5
+ },
+ "ms": 3071,
+ "out": "{\"sku\": \"CORK-500210\", \"qty\": 3, \"price_usd\": 59.5}"
+ },
+ {
+ "id": "json_list",
+ "cat": "json",
+ "pass": true,
+ "got": {
+ "items": [
+ "silk wallpaper",
+ "cork tiles",
+ "grasscloth panels"
+ ]
+ },
+ "ms": 1587,
+ "out": "{\"items\": [\"silk wallpaper\", \"cork tiles\", \"grasscloth panels\"]}"
+ },
+ {
+ "id": "cls_sentiment",
+ "cat": "classify",
+ "pass": true,
+ "got": "negative",
+ "ms": 1074,
+ "out": "negative"
+ },
+ {
+ "id": "cls_ticket",
+ "cat": "classify",
+ "pass": true,
+ "got": "billing",
+ "ms": 1537,
+ "out": "billing"
+ },
+ {
+ "id": "cls_spam",
+ "cat": "classify",
+ "pass": true,
+ "got": "yes",
+ "ms": 756,
+ "out": "yes"
+ },
+ {
+ "id": "cons_oneword",
+ "cat": "constraint",
+ "pass": true,
+ "got": "Paris",
+ "ms": 787,
+ "out": "Paris"
+ },
+ {
+ "id": "cons_numonly",
+ "cat": "constraint",
+ "pass": true,
+ "got": "391",
+ "ms": 1300,
+ "out": "391"
+ },
+ {
+ "id": "reason_word",
+ "cat": "reason",
+ "pass": true,
+ "got": "wall:\n\n$ 135 - 108 = 27 $ sq ft.\n\n### ✅ Final Answer: **27**",
+ "ms": 6545,
+ "out": "We are given:\n\n- Each roll covers **27 sq ft**.\n- There are **5 rolls**, so total coverage available is: \n $ 5 \\times 27 = 135 $ sq ft.\n\n- The wall is **12 ft"
+ },
+ {
+ "id": "reason_logic",
+ "cat": "reason",
+ "pass": true,
+ "got": "Mia",
+ "ms": 891,
+ "out": "Mia"
+ },
+ {
+ "id": "code_sum",
+ "cat": "code",
+ "pass": true,
+ "got": "function sumTo(n){let s=0;for(let i=1;i<=n;i++)s+=i;return s;}",
+ "ms": 1586,
+ "out": "```js\nfunction sumTo(n){let s=0;for(let i=1;i<=n;i++)s+=i;return s;}\n```"
+ },
+ {
+ "id": "code_null",
+ "cat": "code",
+ "pass": true,
+ "got": "function total(o){return (o.price || 0) * (o.qty || 0);}",
+ "ms": 2700,
+ "out": "```js\nfunction total(o){return (o.price || 0) * (o.qty || 0);}\n```"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/battle/run-80b-when-ready.sh b/battle/run-80b-when-ready.sh
new file mode 100755
index 0000000..ea80e78
--- /dev/null
+++ b/battle/run-80b-when-ready.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Waits until the 80B ACTUALLY SERVES (a real completion returns non-empty) — not just
+# until the instance is placed (which /state reports immediately, before download+load).
+# Then runs the eval. Writes battle/80b.json. Exits when done.
+set -uo pipefail
+API=http://127.0.0.1:52415
+DIR=~/Projects/local-model-leaderboard-watch/battle
+MODEL="mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
+LOG="$DIR/80b-watch.log"
+echo "[$(date +%T)] waiting for $MODEL to SERVE (real probe)..." > "$LOG"
+serves(){ # returns non-empty completion text, or empty
+ curl -s -m25 $API/v1/chat/completions -H 'Content-Type: application/json' \
+ -d '{"model":"'"$MODEL"'","messages":[{"role":"user","content":"reply with the single word ready"}],"max_tokens":8,"temperature":0}' \
+ | node -e 'let c=[];process.stdin.on("data",d=>c.push(d)).on("end",()=>{try{const j=JSON.parse(c.join(""));process.stdout.write((j.choices?.[0]?.message?.content||"").trim())}catch(e){process.stdout.write("")}})' 2>/dev/null; }
+for i in $(seq 1 300); do # up to ~100 min (20s cadence)
+ sz=$(du -sk ~/.exo/models/mlx-community--Qwen3-Next-80B-A3B-Instruct-4bit 2>/dev/null | awk '{print int($1/1024)}')
+ probe="$(serves)"
+ echo "[$(date +%T)] dl=${sz:-0}MB probe='${probe:0:20}'" >> "$LOG"
+ if [ -n "$probe" ]; then
+ echo "[$(date +%T)] SERVING — running eval" >> "$LOG"
+ cd "$DIR" && node eval.mjs "$MODEL" 80b.json >> "$LOG" 2>&1
+ echo "[$(date +%T)] EVAL DONE — 80b.json written" >> "$LOG"
+ exit 0
+ fi
+ sleep 20
+done
+echo "[$(date +%T)] TIMEOUT — 80B never served" >> "$LOG"
+exit 1
← 524d2a5 auto-data-snapshot: 2026-08-18T09:42:51 (1 data files) — bat
·
back to Local Model Leaderboard Watch
·
auto-data-snapshot: 2026-08-19T04:49:28 (1 data files) — dat cccfb0f →