← back to Model Arena

auto-resume runs interrupted by a server restart: on boot, re-queue running/queued/mid-run-error runs (cap 4 resumes per run) instead of abandoning them as errors; heal stuck judging flag and re-judge affected battles

0dbc486fd347fd288191cbf15d8e1186f9321f03 · 2026-07-23 07:06:02 -0700 · Steve Abrams

Files touched

• M server.js

Diff

commit 0dbc486fd347fd288191cbf15d8e1186f9321f03
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jul 23 07:06:02 2026 -0700

    auto-resume runs interrupted by a server restart: on boot, re-queue running/queued/mid-run-error runs (cap 4 resumes per run) instead of abandoning them as errors; heal stuck judging flag and re-judge affected battles
---
 server.js | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/server.js b/server.js
index 0783089..e2d1599 100644
--- a/server.js
+++ b/server.js
@@ -90,12 +90,29 @@ function saveChallenges(list) {
 }
 let challenges = loadChallenges();
 // anything left "running"/"queued" from a previous process died with it; also
-// heal corrupt records (finished before started) that a mid-write restart left behind
-for (const c of challenges) for (const r of c.runs) {
-  const corrupt = r.started_at && r.finished_at && r.finished_at < r.started_at;
-  if (r.status === 'running' || r.status === 'queued' || (r.status !== 'done' && corrupt)) {
-    r.status = 'error'; r.error = 'server restarted mid-run';
+// heal corrupt records (finished before started) that a mid-write restart left behind.
+// Interrupted runs are RE-QUEUED on boot (auto-resume until completed), capped so a
+// run interrupted over and over can't re-spend a metered model forever.
+const MAX_RESUMES = 4;
+const resumeOnBoot = [];
+for (const c of challenges) {
+  if (c.judging) c.judging = false; // a restart mid-judge would otherwise block judging forever
+  for (const r of c.runs) {
+    const corrupt = r.started_at && r.finished_at && r.finished_at < r.started_at;
+    const interrupted = r.status === 'running' || r.status === 'queued' ||
+      (r.status === 'error' && r.error === 'server restarted mid-run') ||
+      (r.status !== 'done' && corrupt);
+    if (!interrupted) continue;
+    r.resumes = (r.resumes || 0) + 1;
     if (r.seconds == null) r.seconds = 0;
+    if (r.resumes > MAX_RESUMES) {
+      r.status = 'error'; r.error = 'server restarted mid-run (gave up after ' + MAX_RESUMES + ' resumes)';
+    } else {
+      r.status = 'error'; r.error = 'server restarted mid-run'; // visible until the boot resume re-queues it
+      resumeOnBoot.push({ c, model: r.model });
+      // the battle will re-settle once this run finishes — let it be judged again then
+      delete c.judged_at; c.aiPick = null;
+    }
   }
 }
 saveChallenges(challenges);
@@ -670,4 +687,8 @@ iframe{width:100%;height:460px;border:0;background:#000}.c.win iframe{height:560
 
 server.listen(PORT, function () {
   console.log('[model-arena] http://localhost:' + this.address().port + (BASIC_AUTH ? '  (basic auth on)' : ''));
+  if (resumeOnBoot.length) {
+    console.log('[model-arena] resuming ' + resumeOnBoot.length + ' run(s) interrupted by restart');
+    for (const j of resumeOnBoot) runModel(j.c, j.model);
+  }
 });

← 41ff05b arena: harden with unhandledRejection/uncaughtException guar  ·  back to Model Arena  ·  add 'Re-run entire challenge' button (re-queues every model bde0e0c →