← back to Local Model Leaderboard Watch
battle: eval harness + 35B baseline (11/12, 55.2 tok/s, longctx pass)
6963c73c0faf7fd701d093aaea22ad9dd9106465 · 2026-08-18 09:23:58 -0700 · Steve Abrams
Files touched
A battle/35b.jsonA battle/eval.mjs
Diff
commit 6963c73c0faf7fd701d093aaea22ad9dd9106465
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Aug 18 09:23:58 2026 -0700
battle: eval harness + 35B baseline (11/12, 55.2 tok/s, longctx pass)
---
battle/35b.json | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
battle/eval.mjs | 120 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 266 insertions(+)
diff --git a/battle/35b.json b/battle/35b.json
new file mode 100644
index 0000000..c222e2e
--- /dev/null
+++ b/battle/35b.json
@@ -0,0 +1,146 @@
+{
+ "model": "mlx-community/Qwen3.6-35B-A3B-8bit",
+ "when": "2026-08-18T16:22:52.604Z",
+ "score": "11/12",
+ "passed": 11,
+ "total": 12,
+ "tok_per_s": 55.2,
+ "tok_s_ms": 7244,
+ "longctx": {
+ "placed": true,
+ "pass": true,
+ "got": "\n\nMAUVE-7719",
+ "ms": 18380,
+ "approx_ctx_tokens": 33990
+ },
+ "byCat": {
+ "json": {
+ "pass": 2,
+ "tot": 3
+ },
+ "classify": {
+ "pass": 3,
+ "tot": 3
+ },
+ "constraint": {
+ "pass": 2,
+ "tot": 2
+ },
+ "reason": {
+ "pass": 2,
+ "tot": 2
+ },
+ "code": {
+ "pass": 2,
+ "tot": 2
+ }
+ },
+ "results": [
+ {
+ "id": "json_person",
+ "cat": "json",
+ "pass": true,
+ "got": {
+ "name": "Maria Gonzalez",
+ "age": 34,
+ "city": "Chicago"
+ },
+ "ms": 14796,
+ "out": "\n\n{\"name\": \"Maria Gonzalez\", \"age\": 34, \"city\": \"Chicago\"}"
+ },
+ {
+ "id": "json_order",
+ "cat": "json",
+ "pass": false,
+ "got": null,
+ "ms": 16206,
+ "out": "."
+ },
+ {
+ "id": "json_list",
+ "cat": "json",
+ "pass": true,
+ "got": {
+ "items": [
+ "silk wallpaper",
+ "cork tiles",
+ "grasscloth panels"
+ ]
+ },
+ "ms": 10022,
+ "out": "\n\n{\"items\": [\"silk wallpaper\", \"cork tiles\", \"grasscloth panels\"]}"
+ },
+ {
+ "id": "cls_sentiment",
+ "cat": "classify",
+ "pass": true,
+ "got": "\n\nnegative",
+ "ms": 6510,
+ "out": "\n\nnegative"
+ },
+ {
+ "id": "cls_ticket",
+ "cat": "classify",
+ "pass": true,
+ "got": "\n\nbilling",
+ "ms": 6280,
+ "out": "\n\nbilling"
+ },
+ {
+ "id": "cls_spam",
+ "cat": "classify",
+ "pass": true,
+ "got": "\n\nyes",
+ "ms": 6431,
+ "out": "\n\nyes"
+ },
+ {
+ "id": "cons_oneword",
+ "cat": "constraint",
+ "pass": true,
+ "got": "\n\nParis",
+ "ms": 3565,
+ "out": "\n\nParis"
+ },
+ {
+ "id": "cons_numonly",
+ "cat": "constraint",
+ "pass": true,
+ "got": "\n\n391",
+ "ms": 3914,
+ "out": "\n\n391"
+ },
+ {
+ "id": "reason_word",
+ "cat": "reason",
+ "pass": true,
+ "got": "\n\n27",
+ "ms": 9808,
+ "out": "\n\n27"
+ },
+ {
+ "id": "reason_logic",
+ "cat": "reason",
+ "pass": true,
+ "got": "\n\nMia",
+ "ms": 9259,
+ "out": "\n\nMia"
+ },
+ {
+ "id": "code_sum",
+ "cat": "code",
+ "pass": true,
+ "got": "function sumTo(n){let s=0;for(let i=1;i<=n;i++)s+=i;return s;}",
+ "ms": 14847,
+ "out": "\n\n```js\nfunction sumTo(n){let s=0;for(let i=1;i<=n;i++)s+=i;return s;}\n```"
+ },
+ {
+ "id": "code_null",
+ "cat": "code",
+ "pass": true,
+ "got": "function total(o){return (o.price ?? 0) * (o.qty ?? 0);}",
+ "ms": 22830,
+ "out": "\n\n```js\nfunction total(o){return (o.price ?? 0) * (o.qty ?? 0);}\n```"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/battle/eval.mjs b/battle/eval.mjs
new file mode 100644
index 0000000..bdd1028
--- /dev/null
+++ b/battle/eval.mjs
@@ -0,0 +1,120 @@
+// Head-to-head local-model eval harness for the exo cluster.
+// Model-agnostic: hits exo's OpenAI-compatible /v1/chat/completions.
+// Deterministic grading (temp 0). Tasks: JSON extraction, classification,
+// code-fix (executed), constraint-following, reasoning, + tok/s + a long-context needle.
+// Usage: node eval.mjs "<exo-model-id>" <out.json>
+const API = process.env.EXO_API || "http://127.0.0.1:52415";
+const MODEL = process.argv[2];
+const OUT = process.argv[3] || "/dev/stdout";
+if (!MODEL) { console.error("usage: node eval.mjs <model-id> <out.json>"); process.exit(1); }
+import { writeFileSync } from "node:fs";
+
+async function ask(messages, max_tokens = 512, timeoutMs = 180000) {
+ const t0 = Date.now();
+ const ctl = new AbortController();
+ const to = setTimeout(() => ctl.abort(), timeoutMs);
+ let r, j;
+ try {
+ r = await fetch(`${API}/v1/chat/completions`, {
+ method: "POST", headers: { "Content-Type": "application/json" }, signal: ctl.signal,
+ body: JSON.stringify({ model: MODEL, messages, temperature: 0, max_tokens }),
+ });
+ j = await r.json();
+ } catch (e) { clearTimeout(to); return { err: String(e), ms: Date.now() - t0 }; }
+ clearTimeout(to);
+ const ms = Date.now() - t0;
+ const content = j?.choices?.[0]?.message?.content ?? "";
+ const ct = j?.usage?.completion_tokens ?? Math.round(content.split(/\s+/).filter(Boolean).length * 1.3);
+ return { content, ms, completion_tokens: ct, raw: j };
+}
+
+// ---- graders ----
+const firstJSON = (s) => { const m = s.match(/\{[\s\S]*\}/); if (!m) return null; try { return JSON.parse(m[0]); } catch { return null; } };
+const norm = (s) => String(s ?? "").toLowerCase().trim().replace(/[.\s]+$/,"");
+const has = (s, sub) => norm(s).includes(norm(sub));
+
+const TASKS = [
+ // ---- JSON extraction ----
+ { id: "json_person", cat: "json", max: 200,
+ msg: [{ role: "user", content: "Extract the person's details as a single JSON object with keys name, age, city. Output ONLY the JSON. Text: 'Maria Gonzalez, 34, lives in Chicago.'" }],
+ grade: (o) => { const j = firstJSON(o); return { pass: !!j && has(j.name,"maria gonzalez") && Number(j.age)===34 && has(j.city,"chicago"), got: j }; } },
+ { id: "json_order", cat: "json", max: 220,
+ msg: [{ role: "user", content: "Extract as JSON with keys sku, qty, price_usd. Output ONLY JSON. Text: 'Customer ordered 3 units of SKU CORK-500210 at $59.50 each.'" }],
+ grade: (o) => { const j = firstJSON(o); return { pass: !!j && has(String(j.sku),"cork-500210") && Number(j.qty)===3 && Math.abs(Number(j.price_usd)-59.5)<0.011, got: j }; } },
+ { id: "json_list", cat: "json", max: 260,
+ msg: [{ role: "user", content: "Return ONLY a JSON object with key 'items' = array of the product names mentioned. Text: 'We stock silk wallpaper, cork tiles, and grasscloth panels.'" }],
+ grade: (o) => { const j = firstJSON(o); const arr = j?.items; const blob = norm(JSON.stringify(arr||"")); return { pass: Array.isArray(arr) && blob.includes("silk") && blob.includes("cork") && blob.includes("grasscloth"), got: j }; } },
+ // ---- classification ----
+ { id: "cls_sentiment", cat: "classify", max: 60,
+ msg: [{ role: "user", content: "Classify the sentiment as exactly one word: positive, negative, or neutral. Reply with only that word. Review: 'The paper arrived torn and the color was completely wrong. Very disappointed.'" }],
+ grade: (o) => ({ pass: norm(o).startsWith("negative") || norm(o)==="negative", got: o.slice(0,40) }) },
+ { id: "cls_ticket", cat: "classify", max: 60,
+ msg: [{ role: "user", content: "Categorize this support ticket as exactly one of: billing, technical, account. Reply with only that word. Ticket: 'I was charged twice for my last order and need a refund.'" }],
+ grade: (o) => ({ pass: norm(o).startsWith("billing"), got: o.slice(0,40) }) },
+ { id: "cls_spam", cat: "classify", max: 40,
+ msg: [{ role: "user", content: "Is this email spam? Answer only 'yes' or 'no'. Email: 'CONGRATULATIONS!!! You WON a $1000 gift card, click here NOW to claim before it expires!!!'" }],
+ grade: (o) => ({ pass: norm(o).startsWith("yes"), got: o.slice(0,40) }) },
+ // ---- constraint / instruction-following ----
+ { id: "cons_oneword", cat: "constraint", max: 30,
+ msg: [{ role: "user", content: "Respond with exactly one word and nothing else: the capital of France." }],
+ grade: (o) => { const w = norm(o).split(/\s+/).filter(Boolean); return { pass: w.length===1 && w[0]==="paris", got: o.slice(0,40) }; } },
+ { id: "cons_numonly", cat: "constraint", max: 30,
+ msg: [{ role: "user", content: "Output only the number, no words, no punctuation: what is 17 * 23?" }],
+ grade: (o) => ({ pass: norm(o).replace(/[^0-9]/g,"")==="391", got: o.slice(0,40) }) },
+ // ---- reasoning ----
+ { id: "reason_word", cat: "reason", max: 400,
+ msg: [{ role: "user", content: "A shelf holds 5 rolls. Each roll covers 27 sq ft. A wall is 12 ft by 9 ft. After covering the wall, how many square feet of coverage remain? Answer with the final number of square feet." }],
+ grade: (o) => ({ pass: /(^|\D)27(\D|$)/.test(norm(o).replace(/135|108/g,"")), got: o.slice(-60) }) }, // 5*27=135 - 108 = 27
+ { id: "reason_logic", cat: "reason", max: 400,
+ msg: [{ role: "user", content: "Tom is older than Sara. Sara is older than Mia. Mia is older than Jon. Who is the second youngest? Answer with only the name." }],
+ grade: (o) => ({ pass: has(o,"mia") && !has(o,"jon "), got: o.slice(-60) }) },
+ // ---- code-fix (executed) ----
+ { id: "code_sum", cat: "code", max: 400,
+ msg: [{ role: "user", content: "This JS function is buggy: `function sumTo(n){let s=0;for(let i=0;i<n;i++)s+=i;return s;}` — it should return 1+2+...+n. Return ONLY the corrected function in a ```js code block." }],
+ grade: (o) => { try { const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code = (m?m[1]:o).trim(); const f = new Function(code + "; return sumTo;")(); return { pass: f(5)===15 && f(1)===1 && f(10)===55, got: code.slice(0,80) }; } catch (e) { return { pass:false, got:"ERR "+e.message }; } } },
+ { id: "code_null", cat: "code", max: 420,
+ msg: [{ role: "user", content: "Fix this JS so it returns 0 for a missing 'price' instead of crashing: `function total(o){return o.price*o.qty;}` should handle o.price or o.qty being undefined by treating them as 0. Return ONLY the corrected function in a ```js code block." }],
+ grade: (o) => { try { const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code=(m?m[1]:o).trim(); const f=new Function(code+"; return total;")(); return { pass: f({qty:2})===0 && f({price:5,qty:3})===15 && f({price:5})===0, got: code.slice(0,80) }; } catch(e){ return { pass:false, got:"ERR "+e.message }; } } },
+];
+
+// long-context needle: ~16K tokens of filler with a secret buried ~60% in
+function buildHaystack() {
+ const filler = "Designer Wallcoverings stocks a wide range of premium wall treatments including silk, cork, grasscloth, mica, and mylar finishes for luxury interiors. ";
+ const reps = 900; // ~ 24 words * 900 ~ 21600 words -> well past 16k tokens
+ let parts = [];
+ for (let i=0;i<reps;i++) { parts.push(filler); if (i===Math.floor(reps*0.6)) parts.push("IMPORTANT: The secret vault code is MAUVE-7719. Remember it. "); }
+ return parts.join("");
+}
+
+async function run() {
+ const results = []; let tps = null; let longctx = null;
+ // tok/s measurement (generation-heavy)
+ const gen = await ask([{ role:"user", content:"Write a 120-word product description for a luxury cork wallpaper called 'Coastal Cork'. Plain prose." }], 400);
+ if (!gen.err && gen.completion_tokens) tps = +(gen.completion_tokens / (gen.ms/1000)).toFixed(1);
+ // tasks
+ for (const t of TASKS) {
+ // Reasoning model: it emits hidden <think> tokens BEFORE the answer, which eat the
+ // budget. Give generous room so it thinks THEN answers (small budgets truncate mid-think).
+ const budget = Math.max(t.max, (t.cat === "reason" || t.cat === "code") ? 2000 : 1000);
+ const r = await ask(t.msg, budget);
+ if (r.err) { results.push({ id:t.id, cat:t.cat, pass:false, err:r.err, ms:r.ms }); continue; }
+ const g = t.grade(r.content);
+ results.push({ id:t.id, cat:t.cat, pass:g.pass, got:g.got, ms:r.ms, out:r.content.slice(0,160) });
+ }
+ // long-context placement + retrieval
+ const hay = buildHaystack();
+ const lc = await ask([{ role:"user", content: hay + "\n\nQuestion: What is the secret vault code mentioned above? Answer with only the code." }], 900, 300000);
+ if (lc.err) longctx = { placed:false, pass:false, err: lc.err, ms: lc.ms };
+ else longctx = { placed:true, pass: has(lc.content,"mauve-7719"), got: lc.content.slice(0,60), ms: lc.ms, approx_ctx_tokens: Math.round(hay.length/4) };
+
+ const passed = results.filter(r=>r.pass).length;
+ const summary = { model: MODEL, when: new Date().toISOString(), score: `${passed}/${results.length}`, passed, total: results.length,
+ tok_per_s: tps, tok_s_ms: gen.ms, longctx, byCat: {}, results };
+ for (const r of results) { const c=summary.byCat[r.cat] ||= {pass:0,tot:0}; c.tot++; if(r.pass)c.pass++; }
+ writeFileSync(OUT, JSON.stringify(summary, null, 2));
+ console.log(`\n=== ${MODEL} ===`);
+ console.log(`score ${summary.score} | tok/s ${tps} | longctx placed=${longctx.placed} pass=${longctx.pass} (~${longctx.approx_ctx_tokens||'?'} tok)`);
+ console.log("byCat:", Object.entries(summary.byCat).map(([k,v])=>`${k} ${v.pass}/${v.tot}`).join(" "));
+ for (const r of results) console.log(` ${r.pass?"✅":"❌"} ${r.id.padEnd(14)} ${r.err?("ERR "+r.err):(String(r.got).replace(/\s+/g,' ').slice(0,70))}`);
+}
+run();
← a52578d daily chooser 2026-08-18: chose Qwen3.6-35B-A3B-8bit; correc
·
back to Local Model Leaderboard Watch
·
auto-data-snapshot: 2026-08-18T09:42:51 (1 data files) — bat 524d2a5 →