watchdog: age queued runs from their own queued_at, not challenge created_at — retries/resumes on battles older than 20 min were being falsely killed as 'never started'; bump queue ceiling to 45 min for deep serialized queues

ca4eed6f1e05b9cfea0b2e7a0d2861acad37a13f · 2026-07-23 07:14:04 -0700 · Steve Abrams

Files touched

M server.js

Diff

commit ca4eed6f1e05b9cfea0b2e7a0d2861acad37a13f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jul 23 07:14:04 2026 -0700

    watchdog: age queued runs from their own queued_at, not challenge created_at — retries/resumes on battles older than 20 min were being falsely killed as 'never started'; bump queue ceiling to 45 min for deep serialized queues
---
 server.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/server.js b/server.js
index e2d1599..bb3c664 100644
--- a/server.js
+++ b/server.js
@@ -352,7 +352,9 @@ function runModel(challenge, modelId) {
   const fkey = challenge.id + ':' + modelId;
   if (inFlight.has(fkey)) return;
   inFlight.add(fkey);
-  run.status = 'queued'; run.error = null; run.started_at = null; run.finished_at = null; saveChallenges(challenges);
+  run.status = 'queued'; run.error = null; run.started_at = null; run.finished_at = null;
+  run.queued_at = new Date().toISOString(); // real queue-entry time — the stuck-queue watchdog ages from THIS, not challenge age
+  saveChallenges(challenges);
   const prompt = ARENA_PROMPT(challenge.prompt);
   const exec = async () => {
     run.status = 'running'; run.started_at = new Date().toISOString(); saveChallenges(challenges);
@@ -388,7 +390,7 @@ function runModel(challenge, modelId) {
 // 30s, fail any run stuck well past its expected ceiling so it surfaces as an
 // honest error instead of an infinite spinner.
 const STUCK_RUNNING_MS = 660000; // local timeout is 600s; grace 60s
-const STUCK_QUEUED_MS = 1200000; // 20 min waiting to even start = host is dead
+const STUCK_QUEUED_MS = 2700000; // 45 min waiting to even start = host is dead (deep serialized queues after a boot resume can legitimately wait 20+ min)
 setInterval(() => {
   const now = Date.now();
   let dirty = false;
@@ -396,8 +398,8 @@ setInterval(() => {
     if (r.status === 'running' && r.started_at && now - Date.parse(r.started_at) > STUCK_RUNNING_MS) {
       r.status = 'error'; r.error = 'timed out (watchdog) — model host unresponsive'; r.finished_at = new Date().toISOString(); inFlight.delete(c.id + ':' + r.model); dirty = true;
     } else if (r.status === 'queued' && r.started_at == null) {
-      // stamp a queue-entry time so we can age it out; reuse challenge age as the floor
-      const floor = Date.parse(c.created_at || new Date().toISOString());
+      // age from the run's own queue-entry time; challenge age only as a legacy fallback
+      const floor = Date.parse(r.queued_at || c.created_at || new Date().toISOString());
       if (now - floor > STUCK_QUEUED_MS) { r.status = 'error'; r.error = 'never started — model host busy/down'; r.seconds = 0; r.finished_at = new Date().toISOString(); inFlight.delete(c.id + ':' + r.model); dirty = true; }
     }
   }

← bde0e0c add 'Re-run entire challenge' button (re-queues every model · back to Model Arena · auto-save: 2026-07-23T07:19:17 (50 files) — data/challenges. 72239ef →