proxy: idle-timeout guard (90s no-data -> clean 'model overloaded, retry' instead of a multi-minute hang)

442d90568620b709fb950a36dc016d8a5e56f7a0 · 2026-08-19 12:17:55 -0700 · steve

Files touched

M server.js

Diff

commit 442d90568620b709fb950a36dc016d8a5e56f7a0
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Aug 19 12:17:55 2026 -0700

    proxy: idle-timeout guard (90s no-data -> clean 'model overloaded, retry' instead of a multi-minute hang)
---
 server.js | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/server.js b/server.js
index 70c297b..520c99b 100644
--- a/server.js
+++ b/server.js
@@ -74,10 +74,20 @@ app.post('/api/chat', async (req, res) => {
   const think = req.body.think === true;   // default false = fast, no reasoning
   res.setHeader('Content-Type', 'application/x-ndjson');
   res.setHeader('Cache-Control', 'no-cache');
+  // Idle-timeout guard: if Ollama sends NO data for IDLE_MS (wedged / overloaded /
+  // thrashing on a cold load) we abort and return a clean, actionable error instead
+  // of hanging the browser for minutes. Reset on every chunk, so a slow-but-steady
+  // stream never trips it.
+  const IDLE_MS = Number(process.env.IDLE_MS) || 90000;
+  const controller = new AbortController();
+  let idle, aborted = false;
+  const bump = () => { clearTimeout(idle); idle = setTimeout(() => { aborted = true; controller.abort(); }, IDLE_MS); };
   try {
+    bump();
     const upstream = await fetch(`${OLLAMA}/api/chat`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
+      signal: controller.signal,
       body: JSON.stringify({ model: MODEL, messages, stream: true, options, think,
         keep_alive: KEEP_ALIVE === '-1' ? -1 : KEEP_ALIVE }),  // -1 (number) = keep forever
     });
@@ -88,18 +98,24 @@ app.post('/api/chat', async (req, res) => {
     }
     // Abort upstream if the client disconnects
     const reader = upstream.body.getReader();
-    const onClose = () => reader.cancel().catch(() => {});
+    const onClose = () => { clearTimeout(idle); reader.cancel().catch(() => {}); };
     req.once('close', onClose);
     const dec = new TextDecoder();
     for (;;) {
       const { done, value } = await reader.read();
       if (done) break;
+      bump();                                 // got data -> reset the idle timer
       res.write(dec.decode(value, { stream: true }));
     }
+    clearTimeout(idle);
     req.off('close', onClose);
     res.end();
   } catch (e) {
-    res.write(JSON.stringify({ error: String(e) }) + '\n');
+    clearTimeout(idle);
+    const msg = aborted
+      ? `model is not responding (overloaded or loading) — no output for ${IDLE_MS/1000}s, please retry`
+      : String(e);
+    res.write(JSON.stringify({ error: msg }) + '\n');
     res.end();
   }
 });

← 4956adc chore: v1.1.0 (session close) — already lint/refactor-clean · back to Qwen38 Viewer · creds-safe fetch guard: resolve relative fetch vs credential ee652bf →