← back to Model Arena

yolo idea 9: multi-vision CONSENSUS referee — panel of local vision models (qwen2.5vl + minicpm-v) vote per artifact; consensus score + per-panelist breakdown + spread/disagreement flag; consensus now agrees with human crown on the smoke test

d2bb06939d5247659e35f26cf34e002514deb8ad · 2026-07-22 23:54:48 -0700 · Steve

Files touched

• M data/challenges.json
• M public/index.html
• M server.js
• M yolo/IDEAS.md

Diff

commit d2bb06939d5247659e35f26cf34e002514deb8ad
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jul 22 23:54:48 2026 -0700

    yolo idea 9: multi-vision CONSENSUS referee — panel of local vision models (qwen2.5vl + minicpm-v) vote per artifact; consensus score + per-panelist breakdown + spread/disagreement flag; consensus now agrees with human crown on the smoke test
---
 data/challenges.json | 40 ++++++++++++++++++++++++++++++----------
 public/index.html    |  7 +++++--
 server.js            | 34 ++++++++++++++++++++++++----------
 yolo/IDEAS.md        |  1 +
 4 files changed, 60 insertions(+), 22 deletions(-)

diff --git a/data/challenges.json b/data/challenges.json
index 2db973b..e4aea9a 100644
--- a/data/challenges.json
+++ b/data/challenges.json
@@ -16,8 +16,13 @@
         "bytes": 2904,
         "finished_at": "2026-07-23T05:38:25.720Z",
         "thumb": true,
-        "aiScore": 4,
-        "aiReason": "The image is blank and does not show any interactive content or visual elements related to the challenge."
+        "aiScore": 3.8,
+        "aiReason": "The model is empty and does not display any fireworks or interactive elements.",
+        "aiScores": {
+          "qwen2.5vl:7b": 4,
+          "minicpm-v:latest": 3.5
+        },
+        "aiSpread": 0.5
       },
       {
         "model": "qwen25-7b",
@@ -38,8 +43,13 @@
         "bytes": 3117,
         "finished_at": "2026-07-23T05:36:44.762Z",
         "thumb": true,
-        "aiScore": 7,
-        "aiReason": "The model meets the challenge criteria but lacks some visual polish and detail."
+        "aiScore": 4.8,
+        "aiReason": "The model is partially functional but lacks the auto-show mode and some visual elements are missing.",
+        "aiScores": {
+          "qwen2.5vl:7b": 5,
+          "minicpm-v:latest": 4.5
+        },
+        "aiSpread": 0.5
       },
       {
         "model": "qwen3-14b",
@@ -51,8 +61,13 @@
         "bytes": 1295,
         "finished_at": "2026-07-23T05:35:14.449Z",
         "thumb": true,
-        "aiScore": 6,
-        "aiReason": "The model is partially functional but lacks visual quality and the complexity required for a physics-based interactive simulation."
+        "aiScore": 5.3,
+        "aiReason": "The file is empty and does not contain any code or content related to the challenge.",
+        "aiScores": {
+          "qwen2.5vl:7b": 4,
+          "minicpm-v:latest": 6.5
+        },
+        "aiSpread": 2.5
       },
       {
         "model": "kimi",
@@ -64,14 +79,19 @@
         "bytes": 1682,
         "finished_at": "2026-07-23T05:36:35.854Z",
         "thumb": true,
-        "aiScore": 7,
-        "aiReason": "The model successfully simulates fireworks with particles and trails but lacks the auto-show mode and is slightly cluttered."
+        "aiScore": 6.5,
+        "aiReason": "The model fulfills the challenge requirements but lacks some visual polish and detail.",
+        "aiScores": {
+          "qwen2.5vl:7b": 7,
+          "minicpm-v:latest": 6
+        },
+        "aiSpread": 1
       }
     ],
     "voted_at": "2026-07-23T05:51:00.009Z",
-    "judged_at": "2026-07-23T06:32:17.090Z",
+    "judged_at": "2026-07-23T06:54:28.527Z",
     "judging": false,
-    "aiPick": "gemma3-12b"
+    "aiPick": "kimi"
   },
   {
     "id": "966d54f224fd",
diff --git a/public/index.html b/public/index.html
index 2e0c797..930d46a 100644
--- a/public/index.html
+++ b/public/index.html
@@ -315,7 +315,7 @@ async function renderDetail(id, statusOnly){
   if (c.judging) ai.innerHTML = `🤖 <b>AI referee</b> is looking at each artifact… (local vision model, $0)`;
   else if (c.aiPick) ai.innerHTML = (blind&&!c.winner)
       ? `🤖 <b>AI referee</b> has scored every artifact — pick hidden until you crown a winner (blind mode). <button class="btn" id="rejudge">re-judge</button>`
-      : `🤖 <b>AI referee pick:</b> <span class="aiscore">${mLabel(c.aiPick)}</span> &nbsp;·&nbsp; a local vision model scored each rendered result. Your 👑 crown is the human vote — agree or overrule. <button class="btn" id="rejudge">re-judge</button>`;
+      : `🤖 <b>AI referee pick:</b> <span class="aiscore">${mLabel(c.aiPick)}</span> &nbsp;·&nbsp; a panel of local vision models scored each rendered result (hover a 🤖 score for the panel breakdown; ⚡ = panelists disagreed). Your 👑 crown is the human vote. <button class="btn" id="rejudge">re-judge</button>`;
   else if (anyArt) ai.innerHTML = `🤖 <b>AI referee</b> — have a local vision model score every artifact 0–10 ($0). <button class="btn" id="judge">Run AI referee</button>`;
   else ai.innerHTML = '';
   const jb = $('#judge')||$('#rejudge');
@@ -343,7 +343,10 @@ async function renderDetail(id, statusOnly){
     if (run.seconds!=null) stats.push(run.seconds+'s');
     if (run.bytes) stats.push(Math.round(run.bytes/1024)+' KB');
     stats.push(run.cost ? '$'+run.cost.toFixed(3) : '$0');
-    if (typeof run.aiScore==='number') stats.push('🤖 '+run.aiScore.toFixed(1));
+    if (typeof run.aiScore==='number'){
+      const panel = run.aiScores ? Object.entries(run.aiScores).map(([m,s])=>m.split(':')[0]+' '+s).join(', ') : '';
+      stats.push(`<span title="vision panel: ${panel}${run.aiSpread?' · spread '+run.aiSpread:''}">🤖 ${run.aiScore.toFixed(1)}${run.aiSpread>=4?' ⚡':''}</span>`);
+    }
     let bodyHtml;
     if (run.status==='done')
       // thumbnail poster (if rendered) with a click-to-run-live overlay — avoids
diff --git a/server.js b/server.js
index 68b8816..7ee6ddd 100644
--- a/server.js
+++ b/server.js
@@ -196,21 +196,35 @@ async function generateMetered(m, prompt) {
   return { text, cost, tokens: { in: inTok, out: outTok } };
 }
 
-// ---------- AI auto-referee (local vision model looks at each artifact) ----------
-const VISION_MODEL = process.env.VISION_MODEL || 'qwen2.5vl:7b';
+// ---------- AI auto-referee (a PANEL of local vision models looks at each artifact) ----------
+// multiple models vote → a robust consensus score + variance (qwen2.5vl leads, minicpm-v seconds).
+const VISION_MODELS = (process.env.VISION_MODELS || 'qwen2.5vl:7b,minicpm-v:latest').split(',').map(s => s.trim()).filter(Boolean);
 const VISION_HOST = process.env.VISION_HOST || 'http://localhost:11434';
-async function judgeArtifact(challengePrompt, pngPath) {
-  const b64 = fs.readFileSync(pngPath).toString('base64');
-  const prompt = `You are the impartial referee of an AI build-off. The challenge was:\n"${challengePrompt}"\n\nThis image is a screenshot of one model's rendered single-file HTML result. Score how well it fulfills the challenge AND its visual quality, 0-10 (a blank/empty/near-black or broken page scores 0-2; a polished, on-brief result scores 8-10). Respond ONLY with JSON: {"score": <number 0-10>, "reason": "<one short sentence>"}.`;
+async function askVision(model, prompt, b64) {
   const r = await httpJson(VISION_HOST + '/api/generate', {}, {
-    model: VISION_MODEL, prompt, images: [b64], stream: false, format: 'json', options: { temperature: 0.2 },
+    model, prompt, images: [b64], stream: false, format: 'json', options: { temperature: 0.2 },
   }, 120000);
-  if (!r.json || !r.json.response) throw new Error('vision no response');
-  let j; try { j = JSON.parse(r.json.response); } catch { throw new Error('vision bad json: ' + r.json.response.slice(0, 80)); }
+  if (!r.json || !r.json.response) throw new Error('no response');
+  const j = JSON.parse(r.json.response);
   const score = Math.max(0, Math.min(10, Number(j.score)));
-  if (!isFinite(score)) throw new Error('vision non-numeric score');
+  if (!isFinite(score)) throw new Error('non-numeric');
   return { score: +score.toFixed(1), reason: String(j.reason || '').slice(0, 200) };
 }
+// returns {score: mean, scores: {model:score}, spread, reason} — panel consensus
+async function judgeArtifact(challengePrompt, pngPath) {
+  const b64 = fs.readFileSync(pngPath).toString('base64');
+  const prompt = `You are an impartial referee of an AI build-off. The challenge was:\n"${challengePrompt}"\n\nThis image is a screenshot of one model's rendered single-file HTML result. Score how well it fulfills the challenge AND its visual quality, 0-10 (a blank/empty/near-black or broken page scores 0-2; a polished, on-brief result scores 8-10). Respond ONLY with JSON: {"score": <number 0-10>, "reason": "<one short sentence>"}.`;
+  const scores = {}; let reason = '';
+  for (const vm of VISION_MODELS) {
+    try { const v = await askVision(vm, prompt, b64); scores[vm] = v.score; if (vm === VISION_MODELS[0] || !reason) reason = v.reason; }
+    catch { /* skip a flaky panelist */ }
+  }
+  const vals = Object.values(scores);
+  if (!vals.length) throw new Error('all vision panelists failed');
+  const mean = vals.reduce((a, b) => a + b, 0) / vals.length;
+  const spread = vals.length > 1 ? +(Math.max(...vals) - Math.min(...vals)).toFixed(1) : 0;
+  return { score: +mean.toFixed(1), scores, spread, reason };
+}
 // judge every done+thumbnailed run of a challenge, sequentially on the vision host
 function judgeChallenge(challenge) {
   if (challenge.judging) return;
@@ -228,7 +242,7 @@ function judgeChallenge(challenge) {
     const run = targets[i++];
     try {
       const v = await judgeArtifact(challenge.prompt, path.join(ART, challenge.id, run.model + '.png'));
-      run.aiScore = v.score; run.aiReason = v.reason;
+      run.aiScore = v.score; run.aiReason = v.reason; run.aiScores = v.scores; run.aiSpread = v.spread;
     } catch (e) { run.aiScore = null; run.aiReason = 'judge failed: ' + String(e.message).slice(0, 80); }
     saveChallenges(challenges); next();
   });
diff --git a/yolo/IDEAS.md b/yolo/IDEAS.md
index 52c6994..ceb592d 100644
--- a/yolo/IDEAS.md
+++ b/yolo/IDEAS.md
@@ -31,3 +31,4 @@ Answering the /contrarian critique ("one builder vote = theater", "toy demos not
 - [ ] Head-to-head 1v1 quick mode.
 
 ## Status @ ~27min: 8 features shipped + committed, 7/7 six-way clean, 1 gated draft (daily launchd). Live: modelarena.agentabrams.com
+- [DONE] #9 Multi-vision CONSENSUS referee — a PANEL of local vision models (qwen2.5vl:7b + minicpm-v) scores each artifact; consensus = mean, with per-panelist breakdown (hover) + a ⚡ disagreement flag on high spread. On the Smoke Test the 2-model consensus pick (kimi) AGREES with the human crown where the single model didn't — more robust. VISION_MODELS env-configurable. $0 local.

← 0c1d691 yolo: log backlog + mid-run status (8 shipped, continuing)  ·  back to Model Arena  ·  yolo idea A: per-category leaderboards — auto-tag challenges dfbd21a →