Add LLM_BACKEND=ollama switch; Big Red answers from Mac2 qwen3:14b over tailnet (no Anthropic-credit dependency)

b7f86db28abb115f53d38a5bd5ca267fac61e76b · 2026-06-24 16:42:54 -0700 · Steve Abrams

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

M server.js

Diff

commit b7f86db28abb115f53d38a5bd5ca267fac61e76b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 24 16:42:54 2026 -0700

    Add LLM_BACKEND=ollama switch; Big Red answers from Mac2 qwen3:14b over tailnet (no Anthropic-credit dependency)
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server.js | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 2 deletions(-)

diff --git a/server.js b/server.js
index 3f20436..c2cb374 100644
--- a/server.js
+++ b/server.js
@@ -322,12 +322,73 @@ app.get('/api/pair/new', async (req, res) => {
 // /api/chat request 500'd with ENOENT. Now we call the Anthropic Messages API
 // directly via @anthropic-ai/sdk. Same signature as the old runClaude so all
 // three callsites (chat, analyze, warm-up) keep working unchanged.
+// Backend switch. Default 'anthropic' (Messages API) keeps the original
+// behaviour; set LLM_BACKEND=ollama to answer from a local/self-hosted Ollama
+// model instead (no metered API spend, no Anthropic-credit dependency). On
+// Kamatera prod we point OLLAMA_URL at the Mac2 Studio's Ollama exposed over
+// the tailnet (tailscale serve --tcp). Anything else falls through to Anthropic.
+const LLM_BACKEND = (process.env.LLM_BACKEND || 'anthropic').toLowerCase();
+const OLLAMA_URL = (process.env.OLLAMA_URL || 'http://127.0.0.1:11434').replace(/\/+$/, '');
+const OLLAMA_MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
+const OLLAMA_VISION_MODEL = process.env.OLLAMA_VISION_MODEL || 'qwen2.5vl:7b';
+
 const ANTHROPIC_KEY = process.env.ANTHROPIC_API_KEY || '';
-if (!ANTHROPIC_KEY) {
-  console.warn('[big-red] WARNING: ANTHROPIC_API_KEY not set — /api/chat and /api/analyze will 500. Add it to .env.');
+if (LLM_BACKEND === 'anthropic' && !ANTHROPIC_KEY) {
+  console.warn('[big-red] WARNING: ANTHROPIC_API_KEY not set — /api/chat and /api/analyze will 500. Add it to .env or set LLM_BACKEND=ollama.');
+}
+if (LLM_BACKEND === 'ollama') {
+  console.log(`[big-red] LLM backend: ollama (${OLLAMA_URL}, text=${OLLAMA_MODEL}, vision=${OLLAMA_VISION_MODEL})`);
 }
 const anthropic = ANTHROPIC_KEY ? new Anthropic({ apiKey: ANTHROPIC_KEY }) : null;
 
+// Local/self-hosted backend via Ollama's OpenAI-compatible endpoint. Same
+// contract as runClaude: takes the assembled prompt, returns the reply text.
+// Vision requests (imagePath set) route to a vision-capable model; text chat
+// uses the main text model. Thinking models (qwen3) may wrap their reasoning in
+// <think>…</think> — we strip it so clients only see the final answer.
+async function runOllama(prompt, { imagePath = null, timeoutMs = 240_000 } = {}) {
+  const model = imagePath ? OLLAMA_VISION_MODEL : OLLAMA_MODEL;
+  let content;
+  if (imagePath) {
+    const buf = fs.readFileSync(imagePath);
+    const ext = path.extname(imagePath).slice(1).toLowerCase();
+    const media_type =
+      ext === 'png' ? 'image/png' :
+      ext === 'webp' ? 'image/webp' :
+      ext === 'gif' ? 'image/gif' :
+      'image/jpeg';
+    content = [
+      { type: 'text', text: prompt },
+      { type: 'image_url', image_url: { url: `data:${media_type};base64,${buf.toString('base64')}` } },
+    ];
+  } else {
+    content = prompt;
+  }
+  const ac = new AbortController();
+  const t = setTimeout(() => ac.abort(), timeoutMs);
+  try {
+    const r = await fetch(`${OLLAMA_URL}/v1/chat/completions`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ model, max_tokens: 1024, stream: false, messages: [{ role: 'user', content }] }),
+      signal: ac.signal,
+    });
+    if (!r.ok) {
+      const body = await r.text().catch(() => '');
+      throw new Error(`Ollama ${r.status}: ${body.slice(0, 200)}`);
+    }
+    const d = await r.json();
+    let text = d?.choices?.[0]?.message?.content || '';
+    text = text.replace(/<think>[\s\S]*?<\/think>/gi, '').trim();
+    return text;
+  } catch (e) {
+    if (e.name === 'AbortError') throw new Error('Ollama request timed out');
+    throw e;
+  } finally {
+    clearTimeout(t);
+  }
+}
+
 // Map the legacy `model: 'haiku'` shorthand (used by old CLI callers) to a
 // current API model ID. Anything else is passed through verbatim so callers
 // can ask for a specific snapshot.
@@ -339,6 +400,10 @@ function resolveModelId(m) {
 }
 
 function runClaude(prompt, { imagePath = null, model = 'haiku', timeoutMs = 240_000 } = {}) {
+  // Local backend takes over when selected — same return contract (reply text).
+  if (LLM_BACKEND === 'ollama') {
+    return runOllama(prompt, { imagePath, timeoutMs });
+  }
   if (!anthropic) {
     return Promise.reject(new Error('ANTHROPIC_API_KEY missing — cannot call Anthropic API'));
   }

← 3315d11 auto-save: 2026-06-21T18:53:05 (1 files) — .claude/ · back to Big Red · (newest)