Pin the Claude arena competitor to Opus (CLAUDE_MODEL, default opus)

6c375f2989b6a775d7185e74550af31868c79aa7 · 2026-07-23 11:31:59 -0700 · Steve Abrams

The claude-code entrant was spawned with bare 'claude -p', so it raced on
whatever the CLI defaulted to (could drift to Sonnet), silently benchmarking
the wrong Claude and skewing the win-rate ledger. Both the headless -p run and
the interactive build-forward launch now pass --model opus, env-overridable.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Files touched

M server.js

Diff

commit 6c375f2989b6a775d7185e74550af31868c79aa7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jul 23 11:31:59 2026 -0700

    Pin the Claude arena competitor to Opus (CLAUDE_MODEL, default opus)
    
    The claude-code entrant was spawned with bare 'claude -p', so it raced on
    whatever the CLI defaulted to (could drift to Sonnet), silently benchmarking
    the wrong Claude and skewing the win-rate ledger. Both the headless -p run and
    the interactive build-forward launch now pass --model opus, env-overridable.
    
    Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 server.js | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/server.js b/server.js
index a814cbd..9d8881a 100644
--- a/server.js
+++ b/server.js
@@ -78,7 +78,7 @@ const MODELS = [
   // HuggingFace BigCode-leaderboard #1 open code model (pulled from HF as GGUF into Ollama). Auto-registers once the download finishes.
   { id: 'hf-qwen-coder-32b', label: 'Qwen2.5-Coder 32B (HF top coder)', kind: 'local', host: 'http://localhost:11434', model: 'hf.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF:Q4_K_M', estCost: 0, hf: true },
   // Claude on Steve's MAX PLAN via the Claude Code CLI — $0 marginal (subscription-covered, not API credits)
-  { id: 'claude-code', label: 'Claude (Max plan)', kind: 'cli', estCost: 0 },
+  { id: 'claude-code', label: 'Claude Opus (Max plan)', kind: 'cli', estCost: 0 },
   // { id: 'claude',  label: 'Claude Fable 5 (API)', kind: 'metered', provider: 'anthropic', model: 'claude-fable-5', envKey: 'ANTHROPIC_API_KEY', estCost: 0.15 }, // needs API credits
   { id: 'kimi',       label: 'Kimi K2.5',      kind: 'metered', provider: 'moonshot',  model: 'kimi-k2.5',        envKey: 'MOONSHOT_API_KEY',  estCost: 0.03, tools: true },
   { id: 'gpt',        label: 'GPT-5.1',        kind: 'metered', provider: 'openai',    model: process.env.OPENAI_MODEL || 'gpt-5.1', envKey: 'OPENAI_API_KEY', estCost: 0.10, tools: true },
@@ -88,11 +88,16 @@ const MODELS = [
 // Claude via the Claude Code CLI — runs on Steve's Max SUBSCRIPTION (OAuth), not
 // metered API credits. $0 marginal. Keys are unset so the CLI uses the plan login.
 const CLAUDE_BIN = process.env.CLAUDE_BIN || path.join(os.homedir(), '.local/bin/claude');
+// Always race Claude on Opus (Steve, 2026-07-23) — the CLI's own default can drift
+// to Sonnet / whatever /model was last set to, which would silently benchmark the
+// wrong Claude and poison the win-rate ledger. Pin it; `opus` alias tracks current
+// Opus across version bumps. Override with CLAUDE_MODEL (e.g. a frozen claude-opus-4-8).
+const CLAUDE_MODEL = process.env.CLAUDE_MODEL || 'opus';
 function generateCli(m, prompt) {
   return new Promise((resolve, reject) => {
     const env = { ...process.env };
     delete env.ANTHROPIC_API_KEY; delete env.ANTHROPIC_AUTH_TOKEN; // force Max-plan OAuth
-    const child = execFile(CLAUDE_BIN, ['-p', prompt], { env, timeout: 300000, maxBuffer: 32 * 1024 * 1024 },
+    const child = execFile(CLAUDE_BIN, ['--model', CLAUDE_MODEL, '-p', prompt], { env, timeout: 300000, maxBuffer: 32 * 1024 * 1024 },
       (err, stdout) => {
         if (err && !stdout) return reject(new Error('claude CLI: ' + String(err.message).slice(0, 160)));
         resolve({ text: stdout, cost: 0, tokens: {} }); // subscription-covered → $0
@@ -696,7 +701,7 @@ const server = http.createServer(async (req, res) => {
     }
     try { fs.writeFileSync(tmp, brief); } catch (e) { return send(res, 500, { error: 'temp write failed' }); }
     // iTerm runs: cd ~/Projects, print a banner, launch interactive claude seeded with the brief
-    const payload = `cd "${os.homedir()}/Projects" && clear && printf '\\n==== ARENA BUILD: ${safeTitle} ====\\n\\n' && claude "$(cat ${tmp})"`;
+    const payload = `cd "${os.homedir()}/Projects" && clear && printf '\\n==== ARENA BUILD: ${safeTitle} ====\\n\\n' && claude --model ${CLAUDE_MODEL} "$(cat ${tmp})"`;
     const osaEsc = s => s.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
     const script = `tell application "iTerm"
   activate

← 0524312 auto-save: 2026-07-23T11:20:37 (21 files) — data/challenges. · back to Model Arena · arena: show $ spend on every model chip (card grid) 92a5f7d →