[object Object]

← back to Model Arena

yolo idea 3: ELO ranking (K=24) + cost-per-win + avg AI-referee score + human/AI agreement % — leaderboard is now a real ranking, not a flat win-rate

f109f27e5001bdeb16eeabf174a60e6d005eb3c0 · 2026-07-22 23:34:32 -0700 · Steve

Files touched

Diff

commit f109f27e5001bdeb16eeabf174a60e6d005eb3c0
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jul 22 23:34:32 2026 -0700

    yolo idea 3: ELO ranking (K=24) + cost-per-win + avg AI-referee score + human/AI agreement % — leaderboard is now a real ranking, not a flat win-rate
---
 public/index.html | 14 ++++++++------
 server.js         | 27 ++++++++++++++++++++++-----
 yolo/IDEAS.md     |  1 +
 3 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/public/index.html b/public/index.html
index 06586d7..174a96a 100644
--- a/public/index.html
+++ b/public/index.html
@@ -162,7 +162,7 @@ tr.first td{color:var(--gold)}
     <h2>Real-World Leaderboard</h2>
     <div class="note" id="b-note"></div>
     <table>
-      <thead><tr><th>#</th><th>Model</th><th>Kind</th><th>Battles judged</th><th>Wins</th><th>Win rate</th><th>Errors</th><th>Avg gen time</th><th>Spend</th></tr></thead>
+      <thead><tr><th>#</th><th>Model</th><th>Kind</th><th>ELO</th><th>Battles</th><th>Wins</th><th>Win %</th><th>🤖 avg</th><th>$/win</th><th>Errors</th><th>Avg time</th><th>Spend</th></tr></thead>
       <tbody id="b-rows"></tbody>
     </table>
   </section>
@@ -352,14 +352,16 @@ async function renderDetail(id, statusOnly){
 // ---- leaderboard ----
 async function loadBoard(){
   const d = await (await fetch(API+'/api/ledger')).json();
-  $('#b-note').textContent = d.votes + ' battle' + (d.votes===1?'':'s') + ' judged. Win rate = wins / battles the model completed an artifact in AND a winner was crowned.';
+  const agree = d.aiAgreePct==null ? '' : ` · 🤖 AI referee agrees with the human crown ${d.aiAgreePct}% of ${d.aiJudged} judged battle${d.aiJudged===1?'':'s'}.`;
+  $('#b-note').textContent = d.votes + ' battle' + (d.votes===1?'':'s') + ' crowned. Ranked by ELO (crowned model beats every other that rendered, K=24). 🤖 avg = mean vision-referee score.' + agree;
   const tb = $('#b-rows'); tb.innerHTML='';
   d.models.forEach((s,i)=>{
     const tr = document.createElement('tr');
-    if (i===0 && s.winRate!=null) tr.className='first';
-    tr.innerHTML = `<td>${i+1}</td><td>${s.label}</td><td>${s.kind}</td><td>${s.battles}</td><td>${s.wins}</td>
-      <td>${s.winRate==null?'—':s.winRate+'%'}</td><td>${s.errors}</td>
-      <td>${s.avgSeconds==null?'—':s.avgSeconds+'s'}</td><td>${s.spend?'$'+s.spend.toFixed(2):'$0'}</td>`;
+    if (i===0 && s.wins>0) tr.className='first';
+    tr.innerHTML = `<td>${i+1}</td><td>${s.label}</td><td>${s.kind}</td><td><b>${s.elo}</b></td><td>${s.battles}</td><td>${s.wins}</td>
+      <td>${s.winRate==null?'—':s.winRate+'%'}</td><td>${s.avgAiScore==null?'—':s.avgAiScore}</td>
+      <td>${s.costPerWin==null?'—':(s.costPerWin?'$'+s.costPerWin.toFixed(3):'$0')}</td>
+      <td>${s.errors}</td><td>${s.avgSeconds==null?'—':s.avgSeconds+'s'}</td><td>${s.spend?'$'+s.spend.toFixed(2):'$0'}</td>`;
     tb.appendChild(tr);
   });
 }
diff --git a/server.js b/server.js
index d57756c..fef703f 100644
--- a/server.js
+++ b/server.js
@@ -301,21 +301,38 @@ setInterval(() => {
 // ---------- ledger ----------
 function buildLedger() {
   const stats = {};
-  for (const m of MODELS) stats[m.id] = { id: m.id, label: m.label, kind: m.kind, battles: 0, wins: 0, errors: 0, totalSeconds: 0, doneRuns: 0, spend: 0 };
+  for (const m of MODELS) stats[m.id] = { id: m.id, label: m.label, kind: m.kind, battles: 0, wins: 0, errors: 0, totalSeconds: 0, doneRuns: 0, spend: 0, elo: 1000, aiScoreSum: 0, aiScoreN: 0 };
+  let aiAgree = 0, aiJudged = 0;
+  // chronological pass (array is creation order) so ELO updates in battle order
   for (const c of challenges) {
     for (const r of c.runs) {
       const s = stats[r.model]; if (!s) continue;
-      if (r.status === 'done') { s.doneRuns++; s.totalSeconds += r.seconds || 0; s.spend += r.cost || 0; }
+      if (r.status === 'done') { s.doneRuns++; s.totalSeconds += r.seconds || 0; s.spend += r.cost || 0; if (typeof r.aiScore === 'number') { s.aiScoreSum += r.aiScore; s.aiScoreN++; } }
       if (r.status === 'error') s.errors++;
       if (c.winner && r.status === 'done') { s.battles++; if (c.winner === r.model) s.wins++; }
     }
+    // ELO: the crowned model beats every other model that produced an artifact (K=24)
+    if (c.winner && stats[c.winner]) {
+      const losers = c.runs.filter(r => r.status === 'done' && r.model !== c.winner && stats[r.model]);
+      for (const l of losers) {
+        const w = stats[c.winner], lo = stats[l.model];
+        const ew = 1 / (1 + Math.pow(10, (lo.elo - w.elo) / 400));
+        w.elo += 24 * (1 - ew); lo.elo += 24 * (0 - (1 - ew));
+      }
+    }
+    // AI-vs-human agreement on judged+crowned battles
+    if (c.winner && c.aiPick) { aiJudged++; if (c.winner === c.aiPick) aiAgree++; }
   }
-  return Object.values(stats).map(s => ({
+  const models = Object.values(stats).map(s => ({
     ...s,
+    elo: Math.round(s.elo),
     winRate: s.battles ? +(100 * s.wins / s.battles).toFixed(1) : null,
     avgSeconds: s.doneRuns ? Math.round(s.totalSeconds / s.doneRuns) : null,
+    avgAiScore: s.aiScoreN ? +(s.aiScoreSum / s.aiScoreN).toFixed(1) : null,
+    costPerWin: s.wins ? +(s.spend / s.wins).toFixed(4) : null,
     spend: +s.spend.toFixed(4),
-  })).sort((a, b) => (b.winRate || 0) - (a.winRate || 0) || b.wins - a.wins);
+  })).sort((a, b) => b.elo - a.elo || (b.winRate || 0) - (a.winRate || 0) || b.wins - a.wins);
+  return { models, aiAgreePct: aiJudged ? Math.round(100 * aiAgree / aiJudged) : null, aiJudged };
 }
 
 // ---------- http ----------
@@ -416,7 +433,7 @@ const server = http.createServer(async (req, res) => {
     return send(res, 200, c);
   }
 
-  if (p === '/api/ledger') return send(res, 200, { models: buildLedger(), votes: challenges.filter(c => c.winner).length });
+  if (p === '/api/ledger') { const l = buildLedger(); return send(res, 200, { ...l, votes: challenges.filter(c => c.winner).length }); }
 
   if ((m = p.match(/^\/thumb\/([a-f0-9]+)\/([\w-]+)$/))) {
     const fp = path.join(ART, m[1], m[2] + '.png');
diff --git a/yolo/IDEAS.md b/yolo/IDEAS.md
index f93cb02..ed2d76f 100644
--- a/yolo/IDEAS.md
+++ b/yolo/IDEAS.md
@@ -15,3 +15,4 @@ Answering the /contrarian critique ("one builder vote = theater", "toy demos not
 ## Log
 - [DONE] #1 Artifact auto-screenshot — thumbnails on cards (.shots strip) + battle panes (poster→click-to-run-live), startup backfill, /thumb route. Commit next.
 - [DONE] #2 AI auto-referee — local qwen2.5vl:7b scores each rendered artifact 0-10 (JSON), auto-runs on battle settle, picks a winner. Verified on Smoke Test: AI pick (gemma) AGREES with human crown = two-signal validation, directly answers 'one vote = theater'. $0 local vision.
+- [DONE] #3 ELO + cost-efficiency + AI-agreement ledger — ELO (K=24, crowned beats all rendered) as headline rank, $/win, avg 🤖 score, and 'AI referee agrees with human X% of N battles' meta-metric. Grok leads @1118.

← dc3d732 yolo idea 2: AI auto-referee — local vision model (qwen2.5vl  ·  back to Model Arena  ·  yolo idea 4: blind judging mode — anonymize models to Model 7ec7ada →