← back to Butlr

AI-gather fixes: hermes3:8b model (0.3s warm vs 6.9s qwen3:14b), speechTimeout=3 (no more mid-sentence clipping), 4s LLM AbortController timeout (no more 24s call-killers), warm keep_alive=15m

4d03f06d1d5d4e70a37f01d9ec06b75f8f8aad2e · 2026-05-12 15:56:29 -0700 · Steve

Files touched

• M routes/twilio-webhooks.js

Diff

commit 4d03f06d1d5d4e70a37f01d9ec06b75f8f8aad2e
Author: Steve <steve@designerwallcoverings.com>
Date:   Tue May 12 15:56:29 2026 -0700

    AI-gather fixes: hermes3:8b model (0.3s warm vs 6.9s qwen3:14b), speechTimeout=3 (no more mid-sentence clipping), 4s LLM AbortController timeout (no more 24s call-killers), warm keep_alive=15m
---
 routes/twilio-webhooks.js | 43 ++++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/routes/twilio-webhooks.js b/routes/twilio-webhooks.js
index 2ed1c21..0baf406 100644
--- a/routes/twilio-webhooks.js
+++ b/routes/twilio-webhooks.js
@@ -103,7 +103,7 @@ async function twimlHandler(req, res) {
   <Gather input="speech dtmf"
           action="${pub}/twilio/gather/${callId}"
           method="POST"
-          speechTimeout="auto"
+          speechTimeout="3"
           timeout="20"
           actionOnEmptyResult="true"
           speechModel="phone_call"
@@ -152,31 +152,44 @@ You are speaking on a phone call. Be brief — 1-2 short sentences max per turn.
 
 You will hear what ${call.business_name} just said. Respond appropriately. Keep responses SHORT (under 25 words).`;
 
+  // ── LLM call with hard 4s timeout ────────────────────────────────────
+  // Model: hermes3:8b — warm response ~0.3s on M2 Max vs qwen3:14b ~6.9s.
+  // Phone-call latency budget is sub-3s; anything slower and the human
+  // hears silence and hangs up.
+  // OLLAMA_HOST override supported for non-default endpoints.
   let aiText = '';
   let aiAction = null;
+  const OLLAMA_URL = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
+  const MODEL = process.env.BUTLR_LLM_MODEL || 'hermes3:8b';
+  const ac = new AbortController();
+  const timeoutMs = parseInt(process.env.BUTLR_LLM_TIMEOUT_MS || '4000', 10);
+  const timeoutId = setTimeout(() => ac.abort(), timeoutMs);
   try {
     const messages = [{ role: 'system', content: systemPrompt }, ...turns.slice(-12)];
-    const r = await fetch('http://host.docker.internal:11434/api/chat', {
+    const r = await fetch(`${OLLAMA_URL}/api/chat`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ model: 'qwen3:14b', stream: false, messages, options: { temperature: 0.3, num_predict: 200 }}),
-    }).catch(async () => {
-      // Fallback to localhost if not running in container
-      return fetch('http://127.0.0.1:11434/api/chat', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ model: 'qwen3:14b', stream: false, messages, options: { temperature: 0.3, num_predict: 200 }}),
-      });
+      body: JSON.stringify({ model: MODEL, stream: false, messages, options: { temperature: 0.3, num_predict: 120 }, keep_alive: '15m' }),
+      signal: ac.signal,
     });
     if (r && r.ok) {
       const j = await r.json();
       aiText = (j.message && j.message.content || '').replace(/<think>[\s\S]*?<\/think>/g, '').trim();
-      // Try to parse as JSON action
-      const m = aiText.match(/\{[^{}]*"action"[^{}]*\}/);
-      if (m) try { aiAction = JSON.parse(m[0]); } catch {}
+      // Try to parse as JSON action (be forgiving with whitespace, single quotes)
+      const m = aiText.match(/\{\s*"?action"?\s*:\s*"[^"]+"[\s\S]*?\}/);
+      if (m) try { aiAction = JSON.parse(m[0].replace(/'/g, '"')); } catch {}
+    } else {
+      console.error('[ai-agent] LLM http error:', r && r.status);
     }
   } catch (e) {
-    console.error('[ai-agent] LLM error:', e.message);
+    if (e.name === 'AbortError') {
+      console.error(`[ai-agent] LLM timed out after ${timeoutMs}ms — falling back`);
+      aiText = "One moment please, I'm checking on that.";
+    } else {
+      console.error('[ai-agent] LLM error:', e.message);
+    }
+  } finally {
+    clearTimeout(timeoutId);
   }
 
   if (!aiText) aiText = 'Sorry, could you repeat that?';
@@ -207,7 +220,7 @@ You will hear what ${call.business_name} just said. Respond appropriately. Keep
     : `<Gather input="speech dtmf"
               action="${pub}/twilio/gather/${callId}"
               method="POST"
-              speechTimeout="auto"
+              speechTimeout="3"
               timeout="20"
               actionOnEmptyResult="true"
               speechModel="phone_call">

← 391b859 feat(listen): live AI-agent transcript fallback when Twilio  ·  back to Butlr  ·  security(P0): kill ?reveal=1 leak + owner-auth + robots noin 5ef12b5 →