← back to Model Arena
yolo idea A: per-category leaderboards — auto-tag challenges (Games/Real Work/Custom), category-filtered ELO ledger; 'which model is best at real work?' is now answerable
dfbd21a76416332adfa28a6f56d7215b1dd0190f · 2026-07-23 00:23:22 -0700 · Steve
Files touched
M data/challenges.jsonM public/index.htmlM server.jsM yolo/IDEAS.md
Diff
commit dfbd21a76416332adfa28a6f56d7215b1dd0190f
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jul 23 00:23:22 2026 -0700
yolo idea A: per-category leaderboards — auto-tag challenges (Games/Real Work/Custom), category-filtered ELO ledger; 'which model is best at real work?' is now answerable
---
data/challenges.json | 42 ++++++++++++++++++++++++++++--------------
public/index.html | 11 ++++++++---
server.js | 29 ++++++++++++++++++++++++++---
yolo/IDEAS.md | 1 +
4 files changed, 63 insertions(+), 20 deletions(-)
diff --git a/data/challenges.json b/data/challenges.json
index e4aea9a..db91bb6 100644
--- a/data/challenges.json
+++ b/data/challenges.json
@@ -91,7 +91,8 @@
"voted_at": "2026-07-23T05:51:00.009Z",
"judged_at": "2026-07-23T06:54:28.527Z",
"judging": false,
- "aiPick": "kimi"
+ "aiPick": "kimi",
+ "category": "Games"
},
{
"id": "966d54f224fd",
@@ -179,7 +180,8 @@
"thumb": true
}
],
- "voted_at": "2026-07-23T05:51:21.751Z"
+ "voted_at": "2026-07-23T05:51:21.751Z",
+ "category": "Games"
},
{
"id": "74df3b61e7ca",
@@ -268,7 +270,8 @@
"thumb": true
}
],
- "voted_at": "2026-07-23T05:51:45.195Z"
+ "voted_at": "2026-07-23T05:51:45.195Z",
+ "category": "Games"
},
{
"id": "93946ee6d736",
@@ -363,7 +366,8 @@
"bytes": 24146,
"thumb": true
}
- ]
+ ],
+ "category": "Real Work"
},
{
"id": "acebf42d306a",
@@ -450,7 +454,8 @@
"started_at": "2026-07-23T06:26:33.968Z",
"finished_at": "2026-07-23T06:24:29.510Z"
}
- ]
+ ],
+ "category": "Real Work"
},
{
"id": "35303b0ef3b8",
@@ -519,7 +524,8 @@
"seconds": 0,
"cost": null
}
- ]
+ ],
+ "category": "Custom"
},
{
"id": "8dd797019e83",
@@ -572,7 +578,8 @@
"bytes": 121,
"thumb": true
}
- ]
+ ],
+ "category": "Custom"
},
{
"id": "e4acfac81892",
@@ -637,7 +644,8 @@
"seconds": 0,
"cost": null
}
- ]
+ ],
+ "category": "Custom"
},
{
"id": "02bc35a3d0b2",
@@ -724,7 +732,8 @@
"thumb": true
}
],
- "voted_at": "2026-07-23T06:13:14.106Z"
+ "voted_at": "2026-07-23T06:13:14.106Z",
+ "category": "Custom"
},
{
"id": "f1c13a3b7ac5",
@@ -754,7 +763,8 @@
"bytes": 471,
"thumb": true
}
- ]
+ ],
+ "category": "Custom"
},
{
"id": "aed52f240396",
@@ -792,7 +802,8 @@
],
"judging": false,
"aiPick": "gemma3-12b",
- "judged_at": "2026-07-23T06:37:16.754Z"
+ "judged_at": "2026-07-23T06:37:16.754Z",
+ "category": "Custom"
},
{
"id": "248ea25bf69b",
@@ -843,7 +854,8 @@
],
"judging": false,
"aiPick": "gemma3-12b",
- "judged_at": "2026-07-23T06:40:30.864Z"
+ "judged_at": "2026-07-23T06:40:30.864Z",
+ "category": "Real Work"
},
{
"id": "08d1c975f6db",
@@ -903,7 +915,8 @@
],
"judging": false,
"aiPick": "gemma3-12b",
- "judged_at": "2026-07-23T06:44:54.324Z"
+ "judged_at": "2026-07-23T06:44:54.324Z",
+ "category": "Custom"
},
{
"id": "661e816553cf",
@@ -943,6 +956,7 @@
],
"judging": false,
"aiPick": "gemma3-12b",
- "judged_at": "2026-07-23T06:49:28.896Z"
+ "judged_at": "2026-07-23T06:49:28.896Z",
+ "category": "Real Work"
}
]
\ No newline at end of file
diff --git a/public/index.html b/public/index.html
index 930d46a..6958a43 100644
--- a/public/index.html
+++ b/public/index.html
@@ -167,6 +167,8 @@ tr.first td{color:var(--gold)}
<section id="board" class="view">
<h2>Real-World Leaderboard</h2>
+ <div class="controls"><label>Category <select id="b-cat"><option value="">All</option></select></label>
+ <span class="tag" id="b-agree"></span></div>
<div class="note" id="b-note"></div>
<table>
<thead><tr><th>#</th><th>Model</th><th>Kind</th><th>ELO</th><th>Battles</th><th>Wins</th><th>Win %</th><th>🤖 avg</th><th>$/win</th><th>Errors</th><th>Avg time</th><th>Spend</th></tr></thead>
@@ -268,7 +270,7 @@ async function loadModels(){
for (const p of PRESETS){
if (p.cat!==lastCat){ lastCat=p.cat; const h=document.createElement('span'); h.className='pcat'; h.textContent=p.cat; pr.appendChild(h); }
const b = document.createElement('span'); b.className='preset'; b.textContent=p.t;
- b.onclick = ()=>{ $('#f-title').value=p.t; $('#f-prompt').value=p.p; };
+ b.onclick = ()=>{ $('#f-title').value=p.t; $('#f-prompt').value=p.p; window.curCat = p.cat==='🏭 Real Work'?'Real Work':(p.cat==='🎮 Games'?'Games':''); };
pr.appendChild(b);
}
}
@@ -283,7 +285,7 @@ $('#f-go').onclick = async ()=>{
const title=$('#f-title').value.trim(), prompt=$('#f-prompt').value.trim(), ids=picked();
if (!title||!prompt||!ids.length) return alert('Need a title, a challenge prompt, and at least one model.');
$('#f-go').disabled = true;
- const r = await fetch(API+'/api/challenges',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({title,prompt,models:ids})});
+ const r = await fetch(API+'/api/challenges',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({title,prompt,models:ids,category:window.curCat||''})});
$('#f-go').disabled = false;
if (!r.ok) return alert('Error: '+(await r.text()));
const c = await r.json();
@@ -397,7 +399,10 @@ async function renderDetail(id, statusOnly){
// ---- leaderboard ----
async function loadBoard(){
- const d = await (await fetch(API+'/api/ledger')).json();
+ const cat = $('#b-cat').value || '';
+ const d = await (await fetch(API+'/api/ledger'+(cat?'?category='+encodeURIComponent(cat):''))).json();
+ // populate category dropdown once
+ const sel=$('#b-cat'); if (sel.options.length<=1 && d.categories){ d.categories.forEach(c=>{ const o=document.createElement('option'); o.value=c; o.textContent=c; sel.appendChild(o); }); sel.value=cat; sel.onchange=loadBoard; }
const agree = d.aiAgreePct==null ? '' : ` · 🤖 AI referee agrees with the human crown ${d.aiAgreePct}% of ${d.aiJudged} judged battle${d.aiJudged===1?'':'s'}.`;
$('#b-note').textContent = d.votes + ' battle' + (d.votes===1?'':'s') + ' crowned. Ranked by ELO (crowned model beats every other that rendered, K=24). 🤖 avg = mean vision-referee score.' + agree;
const tb = $('#b-rows'); tb.innerHTML='';
diff --git a/server.js b/server.js
index 7ee6ddd..a16fe33 100644
--- a/server.js
+++ b/server.js
@@ -75,6 +75,21 @@ saveChallenges(challenges);
function appendJsonl(file, obj) { try { fs.appendFileSync(file, JSON.stringify(obj) + '\n'); } catch {} }
+// classify a challenge into a coarse category from its title/prompt (for per-category ranking)
+function inferCategory(c) {
+ const t = ((c.title || '') + ' ' + (c.prompt || '')).toLowerCase();
+ if (/\b(product page|catalog|storefront|email|marketing|room|swatch|json|card grid|dashboard|invoice|table|spec)\b/.test(t)) return 'Real Work';
+ if (/\b(game|driving|dragon|world|fireworks|synthwave|bounc|starfield|conway|life|tron|neon|visualizer|particle|snake|maze|physics)\b/.test(t)) return 'Games';
+ return 'Custom';
+}
+// one-time: tag existing challenges that predate the category field
+function backfillCategories() {
+ let dirty = false;
+ for (const c of challenges) if (!c.category) { c.category = inferCategory(c); dirty = true; }
+ if (dirty) saveChallenges(challenges);
+}
+backfillCategories();
+
// one-time: backfill thumbnails for done artifacts that don't have one yet (throttled, sequential)
function backfillThumbs() {
const jobs = [];
@@ -315,12 +330,13 @@ setInterval(() => {
}, 30000).unref();
// ---------- ledger ----------
-function buildLedger() {
+function buildLedger(categoryFilter) {
const stats = {};
for (const m of MODELS) stats[m.id] = { id: m.id, label: m.label, kind: m.kind, battles: 0, wins: 0, errors: 0, totalSeconds: 0, doneRuns: 0, spend: 0, elo: 1000, aiScoreSum: 0, aiScoreN: 0 };
let aiAgree = 0, aiJudged = 0;
+ const pool = categoryFilter ? challenges.filter(c => (c.category || 'Custom') === categoryFilter) : challenges;
// chronological pass (array is creation order) so ELO updates in battle order
- for (const c of challenges) {
+ for (const c of pool) {
for (const r of c.runs) {
const s = stats[r.model]; if (!s) continue;
if (r.status === 'done') { s.doneRuns++; s.totalSeconds += r.seconds || 0; s.spend += r.cost || 0; if (typeof r.aiScore === 'number') { s.aiScoreSum += r.aiScore; s.aiScoreN++; } }
@@ -395,6 +411,7 @@ const server = http.createServer(async (req, res) => {
const c = {
id: crypto.randomBytes(6).toString('hex'),
title, prompt,
+ category: (String(body.category || '').slice(0, 40)) || inferCategory({ title, prompt }),
created_at: new Date().toISOString(),
winner: null,
runs: ids.map(id => ({ model: id, status: 'queued', error: null, seconds: null, cost: null })),
@@ -474,7 +491,13 @@ const server = http.createServer(async (req, res) => {
return send(res, 200, c);
}
- if (p === '/api/ledger') { const l = buildLedger(); return send(res, 200, { ...l, votes: challenges.filter(c => c.winner).length }); }
+ if (p === '/api/ledger') {
+ const cat = u.searchParams.get('category') || '';
+ const l = buildLedger(cat || null);
+ const cats = [...new Set(challenges.map(c => c.category || 'Custom'))].sort();
+ const pool = cat ? challenges.filter(c => (c.category || 'Custom') === cat) : challenges;
+ return send(res, 200, { ...l, votes: pool.filter(c => c.winner).length, categories: cats, category: cat });
+ }
if ((m = p.match(/^\/thumb\/([a-f0-9]+)\/([\w-]+)$/))) {
const fp = path.join(ART, m[1], m[2] + '.png');
diff --git a/yolo/IDEAS.md b/yolo/IDEAS.md
index ceb592d..29b8bc9 100644
--- a/yolo/IDEAS.md
+++ b/yolo/IDEAS.md
@@ -32,3 +32,4 @@ Answering the /contrarian critique ("one builder vote = theater", "toy demos not
## Status @ ~27min: 8 features shipped + committed, 7/7 six-way clean, 1 gated draft (daily launchd). Live: modelarena.agentabrams.com
- [DONE] #9 Multi-vision CONSENSUS referee — a PANEL of local vision models (qwen2.5vl:7b + minicpm-v) scores each artifact; consensus = mean, with per-panelist breakdown (hover) + a ⚡ disagreement flag on high spread. On the Smoke Test the 2-model consensus pick (kimi) AGREES with the human crown where the single model didn't — more robust. VISION_MODELS env-configurable. $0 local.
+- [DONE] #A Per-category leaderboards — challenges auto-tagged Games/Real Work/Custom (inferCategory + backfill of existing); ledger accepts ?category, leaderboard has a category filter. Answers 'which model is best at REAL WORK vs games'. Games: Grok #1, 100% AI/human agreement.
← d2bb069 yolo idea 9: multi-vision CONSENSUS referee — panel of local
·
back to Model Arena
·
yolo idea B: export a battle as a self-contained standalone 2a631a5 →