`server/lib/smart-router.js`

123 lines
// VenturaClaw — Smart LLM Router
// Goal: route prompts to the cheapest viable model. Local Ollama for the easy 80%,
// Claude/GPT only for the hard 20%. Saves Anthropic spend without sacrificing UX.
//
// Wiring:
//   const router = require("./lib/smart-router");
//   const { answer, routed_to, est_cost_usd } = await router.route(prompt, { allowAnthropic: true });
//
// Toggle env:
//   SMART_ROUTER_ALLOW_ANTHROPIC=1   - opt-in for Claude fallback (default: false)
//   SMART_ROUTER_ALLOW_OPENAI=1      - opt-in for OpenAI fallback (default: false)
//
// Hard rule: never spawn `claude` CLI from this module without explicit opt-in.

const OLLAMA_URL   = process.env.OLLAMA_URL   || "http://192.168.1.133:11434";
const OLLAMA_MODEL = process.env.OLLAMA_MODEL || "qwen3:14b";

// Tiers, ordered cheapest → most capable.
const TIERS = [
  { id: "local-fast", label: "Ollama qwen3:14b (Mac1)", url: OLLAMA_URL, model: OLLAMA_MODEL, cost_per_1k: 0,      max_tokens: 800 },
  { id: "local-big",  label: "Ollama deepseek-r1:14b", url: OLLAMA_URL, model: "deepseek-r1:14b",                cost_per_1k: 0,      max_tokens: 1200 },
  { id: "anthropic",  label: "Claude (via CLI)",        url: null,       model: "claude-opus-4-8",              cost_per_1k: 0.003,  max_tokens: 4000 },
];

// Estimate complexity 0..1 from a prompt without calling any LLM. Heuristic, not perfect — but free.
function estimateComplexity(prompt) {
  if (!prompt) return 0;
  const len = prompt.length;
  let score = 0;
  if (len > 200) score += 0.15;
  if (len > 600) score += 0.20;
  if (len > 1500) score += 0.25;
  // signals of complex synthesis / reasoning / code
  const keywords = /\b(legal|contract|tax|liability|jurisdiction|comply|implications|architecture|migration|debug|refactor|design|tradeoff|why does|prove|optimize|because)\b/i;
  if (keywords.test(prompt)) score += 0.20;
  // signals of coding / technical depth
  if (/\bcode|function|regex|algorithm|complexity|big-o|recursion|async\b/i.test(prompt)) score += 0.20;
  // signals of nuance / multi-step / comparison
  if (/\b(vs\.?|compare|differ|pros|cons|alternatives?|deciding between)\b/i.test(prompt)) score += 0.15;
  // signals of simple factual lookup → keep low
  if (/^(what is|how do i|where can i|when is|who is|is there)/i.test(prompt) && len < 120) score -= 0.10;
  return Math.max(0, Math.min(1, score));
}

function pickTier(complexity, opts = {}) {
  const allowAnthropic = opts.allowAnthropic ?? (process.env.SMART_ROUTER_ALLOW_ANTHROPIC === "1");
  // Tuned for "qwen3:14b is pre-warmed, deepseek-r1 cold-loads ~45s on Mac1".
  // Stay on the warm model for ~90% of traffic.
  if (complexity < 0.55) return TIERS[0];                 // pre-warmed local
  if (complexity < 0.80) return TIERS[1];                 // bigger local — accept cold load risk
  if (allowAnthropic)    return TIERS[2];                 // Claude only for the genuinely hardest prompts
  return TIERS[0];                                        // fall back to fast/warm if Claude not allowed
}

async function callOllama(tier, prompt, opts = {}) {
  const r = await fetch(`${tier.url}/api/generate`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: tier.model,
      prompt,
      stream: false,
      keep_alive: "30m",
      options: { temperature: opts.temperature ?? 0.3, num_predict: opts.maxTokens || tier.max_tokens },
    }),
    signal: AbortSignal.timeout(opts.timeoutMs || 60000),
  });
  if (!r.ok) throw new Error(`ollama ${r.status}: ${(await r.text()).slice(0, 200)}`);
  const d = await r.json();
  return (d.response || "").trim();
}

// Claude fallback — uses the Max plan via the `claude` CLI to avoid burning Anthropic API credits.
// Spawns one-shot, never persists a session. Strips ANTHROPIC_API_KEY env to ensure CLI uses Max plan.
async function callClaudeCLI(prompt, opts = {}) {
  const { spawn } = require("child_process");
  return new Promise((resolve, reject) => {
    const env = { ...process.env };
    delete env.ANTHROPIC_API_KEY;
    delete env.ANTHROPIC_AUTH_TOKEN;
    const child = spawn("claude", ["--model", "opus", "-p", "--output-format", "text"], { env });
    let out = "", err = "";
    child.stdout.on("data", d => out += d);
    child.stderr.on("data", d => err += d);
    child.on("close", code => code === 0 ? resolve(out.trim()) : reject(new Error(`claude CLI exit ${code}: ${err.slice(0, 200)}`)));
    child.on("error", reject);
    const timer = setTimeout(() => { child.kill(); reject(new Error("claude CLI timeout")); }, opts.timeoutMs || 90000);
    child.on("close", () => clearTimeout(timer));
    child.stdin.end(prompt);
  });
}

async function route(prompt, opts = {}) {
  const start = Date.now();
  const complexity = estimateComplexity(prompt);
  const tier = pickTier(complexity, opts);
  let answer, error;
  try {
    if (tier.id === "anthropic") {
      answer = await callClaudeCLI(prompt, opts);
    } else {
      answer = await callOllama(tier, prompt, opts);
    }
  } catch (e) {
    error = e.message;
    // graceful fallback: if local-fast fails, try local-big; if local-big fails, give up (don't escalate to Claude on error — bill control).
    if (tier.id === "local-fast") {
      try { answer = await callOllama(TIERS[1], prompt, opts); } catch (e2) { error += " · also " + e2.message; }
    }
  }
  return {
    answer: answer || `(error) ${error}`,
    routed_to: tier.id,
    routed_to_label: tier.label,
    complexity_score: complexity,
    est_cost_usd: tier.cost_per_1k * Math.ceil((prompt.length + (answer || "").length) / 4 / 1000),
    elapsed_ms: Date.now() - start,
    error: error || null,
  };
}

module.exports = { route, estimateComplexity, pickTier, TIERS };