[object Object]

← back to Ventura Claw

fix(llm): demo classify returned 0/5 — qwen3 reasoning ate token budget + health-gate ignored fallback

f04d321fcfedec9e7e7bed656805f4192ff88755 · 2026-05-30 21:27:26 -0700 · SteveStudio2

Root causes (demo widget = Show HN launch blocker):
- classifyIntent used num_predict:220 against qwen3:14b (a reasoning model); reasoning
  tokens consumed the whole budget so response came back EMPTY (done_reason=length).
  Fix: pass think:false to /api/generate (clean JSON in ~1.2s) + raise budget to 512.
- _isHealthy() gated only on the PRIMARY endpoint; when Mac1 Ollama returned HTTP 500 it
  returned null in 0ms and never reached the healthy Mac2 fallback. Fix: healthy if EITHER
  primary or fallback /api/tags answers.
- prewarm() logged success without checking r.ok, masking a dead backend (silent 0/5).
  Fix: check r.ok, warn loudly on unhealthy backend.
- Defense-in-depth: strip <think> in tryParse + /no_think suffix (for builds that ignore think:false).

Note: OLLAMA_FALLBACK_URL/MODEL added to server/.env (gitignored) — Mac1 stays primary
per project default, Mac2 (also holds qwen3:14b) is the fallback. Verified PRIMED 5/5.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit f04d321fcfedec9e7e7bed656805f4192ff88755
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Sat May 30 21:27:26 2026 -0700

    fix(llm): demo classify returned 0/5 — qwen3 reasoning ate token budget + health-gate ignored fallback
    
    Root causes (demo widget = Show HN launch blocker):
    - classifyIntent used num_predict:220 against qwen3:14b (a reasoning model); reasoning
      tokens consumed the whole budget so response came back EMPTY (done_reason=length).
      Fix: pass think:false to /api/generate (clean JSON in ~1.2s) + raise budget to 512.
    - _isHealthy() gated only on the PRIMARY endpoint; when Mac1 Ollama returned HTTP 500 it
      returned null in 0ms and never reached the healthy Mac2 fallback. Fix: healthy if EITHER
      primary or fallback /api/tags answers.
    - prewarm() logged success without checking r.ok, masking a dead backend (silent 0/5).
      Fix: check r.ok, warn loudly on unhealthy backend.
    - Defense-in-depth: strip <think> in tryParse + /no_think suffix (for builds that ignore think:false).
    
    Note: OLLAMA_FALLBACK_URL/MODEL added to server/.env (gitignored) — Mac1 stays primary
    per project default, Mac2 (also holds qwen3:14b) is the fallback. Verified PRIMED 5/5.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server/llm.js | 42 +++++++++++++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/server/llm.js b/server/llm.js
index c005ab2..60cee29 100644
--- a/server/llm.js
+++ b/server/llm.js
@@ -13,11 +13,17 @@ const OLLAMA_FALLBACK_MODEL = process.env.OLLAMA_FALLBACK_MODEL || "gemma3:12b";
 let _healthCache = { at: 0, ok: false };
 async function _isHealthy() {
   if (Date.now() - _healthCache.at < 15000) return _healthCache.ok;
-  try {
-    const r = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(1500) });
-    _healthCache = { at: Date.now(), ok: r.ok };
-    return r.ok;
-  } catch { _healthCache = { at: Date.now(), ok: false }; return false; }
+  const check = async (url) => {
+    try { const r = await fetch(`${url}/api/tags`, { signal: AbortSignal.timeout(1500) }); return r.ok; }
+    catch { return false; }
+  };
+  // Healthy if EITHER primary or fallback answers. Previously this gated only on the
+  // primary (Mac1); when Mac1 returned HTTP 500 the whole function returned null in 0ms
+  // and never reached the healthy Mac2 fallback — the demo went 0/5 even with a good box up.
+  let ok = await check(OLLAMA_URL);
+  if (!ok && OLLAMA_FALLBACK_URL) ok = await check(OLLAMA_FALLBACK_URL);
+  _healthCache = { at: Date.now(), ok };
+  return ok;
 }
 
 async function generate(prompt, opts = {}) {
@@ -30,6 +36,9 @@ async function generate(prompt, opts = {}) {
     options: { temperature: opts.temperature ?? 0.1, num_predict: opts.maxTokens || 256 }
   };
   if (opts.json) body.format = "json";
+  // Disable reasoning for thinking models (qwen3): otherwise the token budget is spent on
+  // <think> tokens (routed to a separate `thinking` field) and `response` comes back EMPTY.
+  if (opts.think === false) body.think = false;
   const r = await fetch(`${url}/api/generate`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
@@ -47,13 +56,21 @@ async function prewarm() {
   if (OLLAMA_FALLBACK_URL) targets.push([OLLAMA_FALLBACK_URL, OLLAMA_FALLBACK_MODEL]);
   for (const [url, model] of targets) {
     try {
-      await fetch(`${url}/api/generate`, {
+      const r = await fetch(`${url}/api/generate`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ model, prompt: "ok", stream: false, keep_alive: "30m", options: { num_predict: 1 } }),
         signal: AbortSignal.timeout(60000),
       });
-      console.log(`[llm] pre-warmed ${model} on ${url}`);
+      if (r.ok) {
+        console.log(`[llm] pre-warmed ${model} on ${url}`);
+      } else {
+        // r.ok=false = endpoint reachable but UNHEALTHY (e.g. Ollama HTTP 500, model not
+        // pulled/evicted). This used to be logged as success, masking a dead backend and
+        // producing a silent 0/5 demo-cache prime. Warn loudly so the failure is visible.
+        const body = await r.text().catch(() => "");
+        console.warn(`[llm] pre-warm UNHEALTHY for ${model} on ${url}: HTTP ${r.status} ${body.slice(0, 200)}`);
+      }
     } catch (e) {
       console.warn(`[llm] pre-warm failed for ${url}: ${e.message}`);
     }
@@ -107,10 +124,17 @@ async function classifyIntent(message, ctx = {}) {
   if (connectors.length === 0) return null;
 
   const prefix = _buildStaticPrefix(connectors);
-  const dynamic = `\nCONNECTED for this user: ${[...userConnected].join(", ") || "(none)"}\n\nUser: ${message}\nJSON:`;
+  // ` /no_think` disables qwen3's reasoning pass so it emits compact JSON directly (faster,
+  // and no <think> block to strip). Harmless literal text for non-thinking fallback models.
+  const dynamic = `\nCONNECTED for this user: ${[...userConnected].join(", ") || "(none)"}\n\nUser: ${message} /no_think\nJSON:`;
 
   const tryParse = (out) => {
     if (!out) return null;
+    // Belt-and-suspenders: if a build ignores /no_think, qwen3 still wraps replies in
+    // <think>…</think>. Strip it (as approvalHaiku/summarizeResult do) before brace-matching —
+    // otherwise the greedy matcher spans the reasoning prose and JSON.parse fails. THIS, plus
+    // a dead Mac1 backend, was the cause of the silent 0/5 demo-cache prime.
+    out = out.replace(/<think>[\s\S]*?<\/think>/g, "").replace(/<think>[\s\S]*$/g, "").trim();
     const m = out.match(/\{[\s\S]*\}/);
     if (!m) return null;
     try { return JSON.parse(m[0]); } catch { return null; }
@@ -121,7 +145,7 @@ async function classifyIntent(message, ctx = {}) {
   // is probably being evicted by the codex 8-way; fall back to Mac2 fast.
   const tryGen = async (url, model, timeoutMs, suffix = "") => {
     try {
-      const out = await generate(prefix + dynamic + suffix, { maxTokens: 220, url, model, timeoutMs });
+      const out = await generate(prefix + dynamic + suffix, { maxTokens: 512, think: false, url, model, timeoutMs });
       const p = tryParse(out);
       return p && _validateIntent(p, connectors) ? p : null;
     } catch { return null; }

← 7a6c4ad Update Claude model IDs to claude-opus-4-8 (Opus 4.8 upgrade  ·  back to Ventura Claw  ·  fix: add rel=noopener noreferrer to target=_blank link in ad 49d825b →