`battle/eval.mjs`

121 lines
// Head-to-head local-model eval harness for the exo cluster.
// Model-agnostic: hits exo's OpenAI-compatible /v1/chat/completions.
// Deterministic grading (temp 0). Tasks: JSON extraction, classification,
// code-fix (executed), constraint-following, reasoning, + tok/s + a long-context needle.
// Usage:  node eval.mjs "<exo-model-id>" <out.json>
const API = process.env.EXO_API || "http://127.0.0.1:52415";
const MODEL = process.argv[2];
const OUT = process.argv[3] || "/dev/stdout";
if (!MODEL) { console.error("usage: node eval.mjs <model-id> <out.json>"); process.exit(1); }
import { writeFileSync } from "node:fs";

async function ask(messages, max_tokens = 512, timeoutMs = 180000) {
  const t0 = Date.now();
  const ctl = new AbortController();
  const to = setTimeout(() => ctl.abort(), timeoutMs);
  let r, j;
  try {
    r = await fetch(`${API}/v1/chat/completions`, {
      method: "POST", headers: { "Content-Type": "application/json" }, signal: ctl.signal,
      body: JSON.stringify({ model: MODEL, messages, temperature: 0, max_tokens }),
    });
    j = await r.json();
  } catch (e) { clearTimeout(to); return { err: String(e), ms: Date.now() - t0 }; }
  clearTimeout(to);
  const ms = Date.now() - t0;
  const content = j?.choices?.[0]?.message?.content ?? "";
  const ct = j?.usage?.completion_tokens ?? Math.round(content.split(/\s+/).filter(Boolean).length * 1.3);
  return { content, ms, completion_tokens: ct, raw: j };
}

// ---- graders ----
const firstJSON = (s) => { const m = s.match(/\{[\s\S]*\}/); if (!m) return null; try { return JSON.parse(m[0]); } catch { return null; } };
const norm = (s) => String(s ?? "").toLowerCase().trim().replace(/[.\s]+$/,"");
const has = (s, sub) => norm(s).includes(norm(sub));

const TASKS = [
  // ---- JSON extraction ----
  { id: "json_person", cat: "json", max: 200,
    msg: [{ role: "user", content: "Extract the person's details as a single JSON object with keys name, age, city. Output ONLY the JSON. Text: 'Maria Gonzalez, 34, lives in Chicago.'" }],
    grade: (o) => { const j = firstJSON(o); return { pass: !!j && has(j.name,"maria gonzalez") && Number(j.age)===34 && has(j.city,"chicago"), got: j }; } },
  { id: "json_order", cat: "json", max: 220,
    msg: [{ role: "user", content: "Extract as JSON with keys sku, qty, price_usd. Output ONLY JSON. Text: 'Customer ordered 3 units of SKU CORK-500210 at $59.50 each.'" }],
    grade: (o) => { const j = firstJSON(o); return { pass: !!j && has(String(j.sku),"cork-500210") && Number(j.qty)===3 && Math.abs(Number(j.price_usd)-59.5)<0.011, got: j }; } },
  { id: "json_list", cat: "json", max: 260,
    msg: [{ role: "user", content: "Return ONLY a JSON object with key 'items' = array of the product names mentioned. Text: 'We stock silk wallpaper, cork tiles, and grasscloth panels.'" }],
    grade: (o) => { const j = firstJSON(o); const arr = j?.items; const blob = norm(JSON.stringify(arr||"")); return { pass: Array.isArray(arr) && blob.includes("silk") && blob.includes("cork") && blob.includes("grasscloth"), got: j }; } },
  // ---- classification ----
  { id: "cls_sentiment", cat: "classify", max: 60,
    msg: [{ role: "user", content: "Classify the sentiment as exactly one word: positive, negative, or neutral. Reply with only that word. Review: 'The paper arrived torn and the color was completely wrong. Very disappointed.'" }],
    grade: (o) => ({ pass: norm(o).startsWith("negative") || norm(o)==="negative", got: o.slice(0,40) }) },
  { id: "cls_ticket", cat: "classify", max: 60,
    msg: [{ role: "user", content: "Categorize this support ticket as exactly one of: billing, technical, account. Reply with only that word. Ticket: 'I was charged twice for my last order and need a refund.'" }],
    grade: (o) => ({ pass: norm(o).startsWith("billing"), got: o.slice(0,40) }) },
  { id: "cls_spam", cat: "classify", max: 40,
    msg: [{ role: "user", content: "Is this email spam? Answer only 'yes' or 'no'. Email: 'CONGRATULATIONS!!! You WON a $1000 gift card, click here NOW to claim before it expires!!!'" }],
    grade: (o) => ({ pass: norm(o).startsWith("yes"), got: o.slice(0,40) }) },
  // ---- constraint / instruction-following ----
  { id: "cons_oneword", cat: "constraint", max: 30,
    msg: [{ role: "user", content: "Respond with exactly one word and nothing else: the capital of France." }],
    grade: (o) => { const w = norm(o).split(/\s+/).filter(Boolean); return { pass: w.length===1 && w[0]==="paris", got: o.slice(0,40) }; } },
  { id: "cons_numonly", cat: "constraint", max: 30,
    msg: [{ role: "user", content: "Output only the number, no words, no punctuation: what is 17 * 23?" }],
    grade: (o) => ({ pass: norm(o).replace(/[^0-9]/g,"")==="391", got: o.slice(0,40) }) },
  // ---- reasoning ----
  { id: "reason_word", cat: "reason", max: 400,
    msg: [{ role: "user", content: "A shelf holds 5 rolls. Each roll covers 27 sq ft. A wall is 12 ft by 9 ft. After covering the wall, how many square feet of coverage remain? Answer with the final number of square feet." }],
    grade: (o) => ({ pass: /(^|\D)27(\D|$)/.test(norm(o).replace(/135|108/g,"")), got: o.slice(-60) }) }, // 5*27=135 - 108 = 27
  { id: "reason_logic", cat: "reason", max: 400,
    msg: [{ role: "user", content: "Tom is older than Sara. Sara is older than Mia. Mia is older than Jon. Who is the second youngest? Answer with only the name." }],
    grade: (o) => ({ pass: has(o,"mia") && !has(o,"jon "), got: o.slice(-60) }) },
  // ---- code-fix (executed) ----
  { id: "code_sum", cat: "code", max: 400,
    msg: [{ role: "user", content: "This JS function is buggy: `function sumTo(n){let s=0;for(let i=0;i<n;i++)s+=i;return s;}` — it should return 1+2+...+n. Return ONLY the corrected function in a ```js code block." }],
    grade: (o) => { try { const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code = (m?m[1]:o).trim(); const f = new Function(code + "; return sumTo;")(); return { pass: f(5)===15 && f(1)===1 && f(10)===55, got: code.slice(0,80) }; } catch (e) { return { pass:false, got:"ERR "+e.message }; } } },
  { id: "code_null", cat: "code", max: 420,
    msg: [{ role: "user", content: "Fix this JS so it returns 0 for a missing 'price' instead of crashing: `function total(o){return o.price*o.qty;}` should handle o.price or o.qty being undefined by treating them as 0. Return ONLY the corrected function in a ```js code block." }],
    grade: (o) => { try { const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code=(m?m[1]:o).trim(); const f=new Function(code+"; return total;")(); return { pass: f({qty:2})===0 && f({price:5,qty:3})===15 && f({price:5})===0, got: code.slice(0,80) }; } catch(e){ return { pass:false, got:"ERR "+e.message }; } } },
];

// long-context needle: ~16K tokens of filler with a secret buried ~60% in
function buildHaystack() {
  const filler = "Designer Wallcoverings stocks a wide range of premium wall treatments including silk, cork, grasscloth, mica, and mylar finishes for luxury interiors. ";
  const reps = 900; // ~ 24 words * 900 ~ 21600 words -> well past 16k tokens
  let parts = [];
  for (let i=0;i<reps;i++) { parts.push(filler); if (i===Math.floor(reps*0.6)) parts.push("IMPORTANT: The secret vault code is MAUVE-7719. Remember it. "); }
  return parts.join("");
}

async function run() {
  const results = []; let tps = null; let longctx = null;
  // tok/s measurement (generation-heavy)
  const gen = await ask([{ role:"user", content:"Write a 120-word product description for a luxury cork wallpaper called 'Coastal Cork'. Plain prose." }], 400);
  if (!gen.err && gen.completion_tokens) tps = +(gen.completion_tokens / (gen.ms/1000)).toFixed(1);
  // tasks
  for (const t of TASKS) {
    // Reasoning model: it emits hidden <think> tokens BEFORE the answer, which eat the
    // budget. Give generous room so it thinks THEN answers (small budgets truncate mid-think).
    const budget = Math.max(t.max, (t.cat === "reason" || t.cat === "code") ? 2000 : 1000);
    const r = await ask(t.msg, budget);
    if (r.err) { results.push({ id:t.id, cat:t.cat, pass:false, err:r.err, ms:r.ms }); continue; }
    const g = t.grade(r.content);
    results.push({ id:t.id, cat:t.cat, pass:g.pass, got:g.got, ms:r.ms, out:r.content.slice(0,160) });
  }
  // long-context placement + retrieval
  const hay = buildHaystack();
  const lc = await ask([{ role:"user", content: hay + "\n\nQuestion: What is the secret vault code mentioned above? Answer with only the code." }], 900, 300000);
  if (lc.err) longctx = { placed:false, pass:false, err: lc.err, ms: lc.ms };
  else longctx = { placed:true, pass: has(lc.content,"mauve-7719"), got: lc.content.slice(0,60), ms: lc.ms, approx_ctx_tokens: Math.round(hay.length/4) };

  const passed = results.filter(r=>r.pass).length;
  const summary = { model: MODEL, when: new Date().toISOString(), score: `${passed}/${results.length}`, passed, total: results.length,
    tok_per_s: tps, tok_s_ms: gen.ms, longctx, byCat: {}, results };
  for (const r of results) { const c=summary.byCat[r.cat] ||= {pass:0,tot:0}; c.tot++; if(r.pass)c.pass++; }
  writeFileSync(OUT, JSON.stringify(summary, null, 2));
  console.log(`\n=== ${MODEL} ===`);
  console.log(`score ${summary.score}  |  tok/s ${tps}  |  longctx placed=${longctx.placed} pass=${longctx.pass} (~${longctx.approx_ctx_tokens||'?'} tok)`);
  console.log("byCat:", Object.entries(summary.byCat).map(([k,v])=>`${k} ${v.pass}/${v.tot}`).join("  "));
  for (const r of results) console.log(`  ${r.pass?"✅":"❌"} ${r.id.padEnd(14)} ${r.err?("ERR "+r.err):(String(r.got).replace(/\s+/g,' ').slice(0,70))}`);
}
run();