← back to Model Arena
model-arena: openai-local adapter + dynamic multi-host model discovery
fa7e697062415bcc8fb0915a4b4eff9919e21bbd · 2026-09-18 14:32:32 -0700 · Steve Abrams
Add kind:'openai-local' (generateOpenAILocal -> POST /v1/chat/completions,
600s timeout, per-host onKey serialization, watchdog applies, strips <think>).
Discover every chat-capable local model across the 3 hosts every 5min: ollama
(Mac1+localhost)->local, MLX :8000 + exo :52415->openai-local; slug ids, exclude
embeddings, keep vision, dedupe vs curated by (host,model) and by generated id.
Prune the 4 dead static entries (gemma3-12b/hermes3-8b/hf-qwen-coder-32b/muse),
repoint qwen3-14b/qwen25-7b to Mac1 (localhost empty) to keep ledger history;
buildLedger seeds rows for any historical model id so pruning never erases
standings. Coalesce+cache availability lookups; /api/models reports host+discovered.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HTnyDUiR2At9Aj5f6RGRav
Files touched
Diff
commit fa7e697062415bcc8fb0915a4b4eff9919e21bbd
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 18 14:32:32 2026 -0700
model-arena: openai-local adapter + dynamic multi-host model discovery
Add kind:'openai-local' (generateOpenAILocal -> POST /v1/chat/completions,
600s timeout, per-host onKey serialization, watchdog applies, strips <think>).
Discover every chat-capable local model across the 3 hosts every 5min: ollama
(Mac1+localhost)->local, MLX :8000 + exo :52415->openai-local; slug ids, exclude
embeddings, keep vision, dedupe vs curated by (host,model) and by generated id.
Prune the 4 dead static entries (gemma3-12b/hermes3-8b/hf-qwen-coder-32b/muse),
repoint qwen3-14b/qwen25-7b to Mac1 (localhost empty) to keep ledger history;
buildLedger seeds rows for any historical model id so pruning never erases
standings. Coalesce+cache availability lookups; /api/models reports host+discovered.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HTnyDUiR2At9Aj5f6RGRav
---
server.js | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 116 insertions(+), 21 deletions(-)
diff --git a/server.js b/server.js
index c1f35fa..06be600 100644
--- a/server.js
+++ b/server.js
@@ -69,21 +69,19 @@ header h1{font-size:15px;margin:0;font-weight:600}header .by{color:#8a93a0;font-
// ---------- model roster ----------
// Local ollama models are free and enabled by default. Metered models require
// their env key AND explicit per-run selection in the UI (never pre-checked).
+// Mac1 is the only Ollama host with models loaded (Mac2 localhost:11434 is empty),
+// so the two curated ollama entries point at Mac1. Their ids are kept stable
+// (qwen3-14b / qwen25-7b) so the historical win-rate ledger stays continuous;
+// dynamic discovery dedupes them by (host,model) instead of re-registering dupes.
+const MAC1_OLLAMA = process.env.OLLAMA_MAC1_HOST || 'http://192.168.1.133:11434';
const MODELS = [
// small local models get the design PACK (material injected as text), not the live
// tool-calling loop — an 8B/14B is unreliable at function-calling and returned 0 chars
// through the tool loop (ALL-MUST-WORK fix, 2026-07-25). Live tool loop stays on the
// reliable API models (kimi/gpt/grok) + claude-code.
- { id: 'qwen3-14b', label: 'Qwen3 14B', kind: 'local', host: 'http://localhost:11434', model: 'qwen3:14b', estCost: 0, dtdRef: 'Qwen' },
- { id: 'gemma3-12b', label: 'Gemma3 12B', kind: 'local', host: 'http://localhost:11434', model: 'gemma3:12b', estCost: 0 },
- { id: 'hermes3-8b', label: 'Hermes3 8B', kind: 'local', host: 'http://localhost:11434', model: 'hermes3:8b', estCost: 0 },
- // was Mac1 (192.168.1.133) — that Ollama host wedges (model pinned in VRAM, generations hang);
- // repointed to Mac2-local qwen2.5 for reliable liveness. Set OLLAMA_MAC1=1 to use Mac1 again.
- { id: 'qwen25-7b', label: 'Qwen2.5 (local)', kind: 'local', host: process.env.OLLAMA_MAC1 ? 'http://192.168.1.133:11434' : 'http://localhost:11434', model: process.env.OLLAMA_MAC1 ? 'qwen2.5:7b' : 'qwen2.5:latest', estCost: 0 },
- // HuggingFace BigCode-leaderboard #1 open code model (pulled from HF as GGUF into Ollama). Auto-registers once the download finishes.
- { id: 'hf-qwen-coder-32b', label: 'Qwen2.5-Coder 32B (HF top coder)', kind: 'local', host: 'http://localhost:11434', model: 'hf.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF:Q4_K_M', estCost: 0, hf: true },
+ { id: 'qwen3-14b', label: 'Qwen3 14B', kind: 'local', host: MAC1_OLLAMA, model: 'qwen3:14b', estCost: 0, dtdRef: 'Qwen' },
+ { id: 'qwen25-7b', label: 'Qwen2.5 7B', kind: 'local', host: MAC1_OLLAMA, model: 'qwen2.5:7b', estCost: 0 },
// Claude on Steve's MAX PLAN via the Claude Code CLI — $0 marginal (subscription-covered, not API credits)
- { id: 'muse', label: 'Muse Glimmer 30B', kind: 'local', host: process.env.MUSE_HOST || 'http://localhost:11434', model: process.env.MUSE_MODEL || 'muse-glimmer:30b-mlx', estCost: 0, dtdRef: 'Muse' },
{ id: 'claude-code', label: 'Claude Opus (Max plan)', kind: 'cli', estCost: 0, dtdRef: 'Claude' },
// { id: 'claude', label: 'Claude Fable 5 (API)', kind: 'metered', provider: 'anthropic', model: 'claude-fable-5', envKey: 'ANTHROPIC_API_KEY', estCost: 0.15 }, // needs API credits
{ id: 'kimi', label: 'Kimi K2.5', kind: 'metered', provider: 'moonshot', model: 'kimi-k2.5', envKey: 'MOONSHOT_API_KEY', estCost: 0.03, tools: true, dtdRef: 'Kimi' },
@@ -254,25 +252,97 @@ function getJson(url, timeoutMs) {
req.on('error', () => resolve(null));
});
}
-// which model tags each ollama host currently has (cached ~15s) — lets the HF coder auto-register once its pull finishes
+// which model tags each host currently lists (cached ~15s, request-coalesced so a
+// 130-model /api/models fan-out hits each host once). Availability reflects live truth.
const tagCache = {};
-async function ollamaTags(host) {
- const c = tagCache[host];
- if (c && Date.now() - c.at < 15000) return c.set;
- const d = await getJson(host + '/api/tags', 4000);
- const set = new Set((d && d.models || []).map(m => m.name));
- tagCache[host] = { at: Date.now(), set };
- return set;
+function cachedSet(cache, host, fetchFn) {
+ const c = cache[host];
+ if (c && (c.promise || Date.now() - c.at < 15000)) return c.promise || Promise.resolve(c.set);
+ const promise = fetchFn().then(set => { cache[host] = { at: Date.now(), set, promise: null }; return set; })
+ .catch(() => { cache[host] = { at: Date.now(), set: new Set(), promise: null }; return new Set(); });
+ cache[host] = { ...(c || {}), promise };
+ return promise;
+}
+function ollamaTags(host) {
+ return cachedSet(tagCache, host, async () => {
+ const d = await getJson(host + '/api/tags', 4000);
+ return new Set((d && d.models || []).map(m => m.name));
+ });
+}
+// OpenAI-protocol hosts (MLX :8000, exo :52415) enumerate via /v1/models
+const openaiCache = {};
+function openaiModels(host) {
+ return cachedSet(openaiCache, host, async () => {
+ const d = await getJson(host + '/v1/models', 6000);
+ return new Set((d && d.data || []).map(m => m.id));
+ });
}
async function modelAvailable(m) {
if (m.kind === 'cli') return fs.existsSync(CLAUDE_BIN);
if (m.kind === 'metered') return !!process.env[m.envKey];
+ if (m.kind === 'openai-local') return (await openaiModels(m.host)).has(m.model);
const tags = await ollamaTags(m.host);
const base = m.model.split(':')[0];
for (const t of tags) if (t === m.model || t.split(':')[0] === base) return true;
return false;
}
+// ---------- dynamic multi-host model discovery ----------
+// Register EVERY chat-capable local model across all three hosts as a MODELS entry,
+// refreshed on startup and every ~5 min. ollama hosts -> kind 'local'; OpenAI-protocol
+// hosts (MLX :8000, exo :52415) -> kind 'openai-local'. Embedding models are excluded
+// (they cannot chat and would 100%-error every battle). Vision models are kept in.
+const OLLAMA_HOSTS = (process.env.ARENA_OLLAMA_HOSTS || MAC1_OLLAMA + ',http://localhost:11434').split(',').map(s => s.trim()).filter(Boolean);
+const OPENAI_LOCAL_HOSTS = (process.env.ARENA_OPENAI_HOSTS || 'http://127.0.0.1:8000,http://127.0.0.1:52415').split(',').map(s => s.trim()).filter(Boolean);
+const isEmbedding = id => /embed|nomic|bge|gte|e5|rerank/i.test(id);
+function hostTag(host) {
+ try {
+ const u = new URL(host);
+ const hp = u.hostname + ':' + (u.port || '');
+ const map = { '192.168.1.133:11434': 'mac1', '127.0.0.1:11434': 'mac2', 'localhost:11434': 'mac2', '127.0.0.1:8000': 'mlx', '127.0.0.1:52415': 'exo' };
+ return map[hp] || (u.hostname.replace(/\./g, '-') + (u.port ? '-' + u.port : ''));
+ } catch { return host.replace(/\W+/g, '-'); }
+}
+const slugModel = s => String(s).toLowerCase().replace(/^.*\//, '').replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+const niceModel = s => String(s).replace(/^.*\//, '');
+// (host,model) pairs already covered by a static curated entry — discovery skips these
+const staticPairs = new Set(MODELS.filter(m => m.host && m.model).map(m => m.host + '|' + m.model));
+async function refreshDiscovery() {
+ const found = [];
+ for (const host of OLLAMA_HOSTS) {
+ const d = await getJson(host + '/api/tags', 4000);
+ for (const mm of (d && d.models || [])) {
+ const name = mm.name; if (!name || isEmbedding(name)) continue;
+ found.push({ kind: 'local', host, model: name });
+ }
+ }
+ for (const host of OPENAI_LOCAL_HOSTS) {
+ const d = await getJson(host + '/v1/models', 6000);
+ for (const mm of (d && d.data || [])) {
+ const id = mm.id; if (!id || isEmbedding(id)) continue;
+ found.push({ kind: 'openai-local', host, model: id });
+ }
+ }
+ const seenPair = new Set(), seenId = new Set(MODELS.filter(m => !m.discovered).map(m => m.id));
+ const next = [];
+ for (const f of found) {
+ const pair = f.host + '|' + f.model;
+ if (staticPairs.has(pair) || seenPair.has(pair)) continue; // dedupe vs curated + within run
+ seenPair.add(pair);
+ const id = hostTag(f.host) + '__' + slugModel(f.model);
+ if (seenId.has(id)) continue; // two model strings that slug to the same id (e.g. org-path vs fs-path of one model)
+ seenId.add(id);
+ next.push({ id, label: hostTag(f.host) + ' · ' + niceModel(f.model), kind: f.kind, host: f.host, model: f.model, estCost: 0, discovered: true });
+ }
+ // swap discovered entries in place (preserve MODELS array identity for captured refs)
+ for (let i = MODELS.length - 1; i >= 0; i--) if (MODELS[i].discovered) MODELS.splice(i, 1);
+ for (const m of next) MODELS.push(m);
+ console.log('[discovery] registered ' + next.length + ' discovered models across ' + (OLLAMA_HOSTS.length + OPENAI_LOCAL_HOSTS.length) + ' hosts (' + MODELS.length + ' total)');
+ return next.length;
+}
+refreshDiscovery().catch(e => console.error('[discovery] initial failed', e));
+setInterval(() => refreshDiscovery().catch(e => console.error('[discovery] refresh failed', e)), 5 * 60 * 1000).unref();
+
// serialize local generations per host (ollama model-load thrash) AND metered
// generations per model (so two rapid retries queue instead of double-spending)
const queues = {};
@@ -300,6 +370,24 @@ async function generateLocal(m, prompt) {
return { text: r.json.message.content, cost: 0, tokens: { out: r.json.eval_count || 0 } };
}
+// OpenAI-protocol local hosts (MLX :8000, exo :52415). Same 600s timeout + per-host
+// serialization + stuck-run watchdog as generateLocal. Strips <think> blocks like the
+// qwen3 path (many MLX/exo reasoning builds emit them and would blow the HTML budget).
+async function generateOpenAILocal(m, prompt) {
+ const r = await httpJson(m.host + '/v1/chat/completions', {}, {
+ model: m.model,
+ messages: [{ role: 'user', content: prompt }],
+ temperature: 0.7,
+ max_tokens: 10240,
+ stream: false,
+ }, 600000);
+ const ch = r.json && r.json.choices && r.json.choices[0];
+ if (!ch || !ch.message) throw new Error('openai-local bad response: ' + (r.raw || r.status));
+ const raw = ch.message.content || ch.message.reasoning_content || '';
+ const text = String(raw).replace(/<think>[\s\S]*?<\/think>/gi, '').trim();
+ return { text, cost: 0, tokens: { out: (r.json.usage && r.json.usage.completion_tokens) || 0 } };
+}
+
// metered pricing per 1M tokens [in, out] — rough, for the cost line
const PRICING = {
anthropic: [5, 25], moonshot: [0.6, 2.5], openai: [1.75, 14], xai: [3, 15], openrouter: [0.5, 2],
@@ -506,6 +594,7 @@ function runModel(challenge, modelId) {
const t0 = Date.now();
try {
const callModel = (p) => m.kind === 'local' ? (dt && m.tools ? generateLocalTools(m, p) : generateLocal(m, p))
+ : m.kind === 'openai-local' ? generateOpenAILocal(m, p)
: m.kind === 'cli' ? generateCli(m, p)
: (dt && m.tools ? generateMeteredTools(m, p) : generateMetered(m, p));
let out = await callModel(prompt);
@@ -539,7 +628,9 @@ function runModel(challenge, modelId) {
saveChallenges(challenges);
if (run.status === 'error') maybeAutoJudge(challenge); // last run may have errored; judge the rest
};
- if (m.kind === 'local') onKey(m.host, exec); else onKey('metered:' + m.id, exec);
+ // serialize per host for both ollama and OpenAI-protocol local hosts (model-load
+ // thrash / single GPU); metered models serialize per model id.
+ if (m.kind === 'local' || m.kind === 'openai-local') onKey(m.host, exec); else onKey('metered:' + m.id, exec);
}
// stuck-run watchdog — a wedged Ollama host (e.g. Mac1) can leave a run
@@ -566,7 +657,11 @@ setInterval(() => {
// ---------- ledger ----------
function buildLedger(categoryFilter) {
const stats = {};
- for (const m of MODELS) stats[m.id] = { id: m.id, label: m.label, kind: m.kind, battles: 0, wins: 0, errors: 0, totalSeconds: 0, doneRuns: 0, spend: 0, elo: 1000, aiScoreSum: 0, aiScoreN: 0 };
+ const seed = (id, label, kind) => { if (!stats[id]) stats[id] = { id, label: label || id, kind: kind || 'local', battles: 0, wins: 0, errors: 0, totalSeconds: 0, doneRuns: 0, spend: 0, elo: 1000, aiScoreSum: 0, aiScoreN: 0 }; };
+ for (const m of MODELS) seed(m.id, m.label, m.kind);
+ // keep historical leaderboard rows for models that have battled but are no longer
+ // in the roster (e.g. pruned static entries), so pruning never erases standings.
+ for (const c of challenges) for (const r of c.runs) seed(r.model, mLabelServer(r.model));
let aiAgree = 0, aiJudged = 0;
const h2h = {}; // h2h[winner][loser] = {w,l} from the winner's perspective (pairwise)
const bump = (a, b, win) => { (h2h[a] = h2h[a] || {}); (h2h[a][b] = h2h[a][b] || { w: 0, l: 0 }); h2h[a][b][win ? 'w' : 'l']++; };
@@ -644,9 +739,9 @@ const server = http.createServer(async (req, res) => {
if (p === '/api/models') {
const out = await Promise.all(MODELS.map(async m => ({
- id: m.id, label: m.label, kind: m.kind, model: m.model,
+ id: m.id, label: m.label, kind: m.kind, model: m.model, host: m.host || null,
available: await modelAvailable(m),
- hf: !!m.hf, estCost: m.estCost, dtdRef: m.dtdRef || null,
+ hf: !!m.hf, discovered: !!m.discovered, estCost: m.estCost, dtdRef: m.dtdRef || null,
})));
return send(res, 200, out);
}
← f748b4b auto-data-snapshot: 2026-09-18T14:19:46 (3 data files) — dat
·
back to Model Arena
·
model-arena: continuous themed arena loop (art/culture/news/ 0fc4204 →