`verification/verify.mjs`

76 lines
import assert from 'node:assert/strict';
import { writeFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { DEFAULT_MODEL, DEEPSEEK_MODEL } from '../exo-client.mjs';

const root = fileURLToPath(new URL('../', import.meta.url));
const evidence = { ticket: 'TK-11325', timestamp: new Date().toISOString(), risk_tier: 'R2', intent: 'Coding client -> stdio MCP -> explicitly selected DeepSeek or default Qwen -> live Exo answer; model absence and invalid input fail visibly', checks: [] };
const clients = [];
const unpack = response => JSON.parse(response.content.find(c => c.type === 'text').text);
async function connect(label, env = {}) {
  const client = new Client({ name: label, version: '1.0.0' });
  const transport = new StdioClientTransport({ command: process.execPath, args: [root + 'server.mjs'], env: { ...process.env, ...env }, stderr: 'pipe' });
  let stderr = '';
  transport.stderr?.on('data', data => { stderr += data.toString(); });
  clients.push(client);
  await client.connect(transport);
  evidence.checks.push({ name: label + ' initialize', verdict: 'PASS', server: client.getServerVersion() });
  return { client, stderr: () => stderr };
}

try {
  const main = await connect('exo-helper-verification', { EXO_TIMEOUT_MS: '45000' });
  const list = await main.client.listTools();
  assert.deepEqual(list.tools.map(t => t.name).sort(), ['ask_exo', 'exo_status']);
  evidence.checks.push({ name: 'Tool discovery', verdict: 'PASS', tools: list.tools.map(t => t.name) });
  const status = await main.client.callTool({ name: 'exo_status', arguments: {} });
  assert.ok(!status.isError); const health = unpack(status);
  assert.equal(health.configured_model, DEFAULT_MODEL); assert.equal(health.model_ready, true);
  evidence.checks.push({ name: 'Live model readiness', verdict: 'PASS', result: health });
  const answer = await main.client.callTool({ name: 'ask_exo', arguments: { prompt: 'What is 17 plus 25? Reply with only the number.', max_tokens: 32 } }, undefined, { timeout: 110000 });
  assert.ok(!answer.isError, JSON.stringify(answer)); const data = unpack(answer);
  assert.equal(data.answer.trim(), '42'); assert.equal(data.model, DEFAULT_MODEL);
  assert.ok(data.response_id); assert.equal(data.truncated, false);
  evidence.checks.push({ name: 'MCP -> live Exo inference -> MCP answer', verdict: 'PASS', result: data });
  const summary = await main.client.callTool({ name: 'ask_exo', arguments: { prompt: 'Extract the HTTP status code and filename. Output only STATUS=<code>; FILE=<filename>.', context: 'The request failed with HTTP 404 while loading settings.json.', max_tokens: 60 } }, undefined, { timeout: 110000 });
  assert.ok(!summary.isError); const extracted = unpack(summary);
  assert.match(extracted.answer, /STATUS=404/); assert.match(extracted.answer, /FILE=settings\.json/);
  evidence.checks.push({ name: 'Bounded extraction task', verdict: 'PASS', result: extracted });
  const deepseekStatus = await main.client.callTool({ name: 'exo_status', arguments: { model: 'deepseek' } });
  assert.ok(!deepseekStatus.isError);
  assert.equal(unpack(deepseekStatus).configured_model, DEEPSEEK_MODEL);
  assert.equal(unpack(deepseekStatus).model_ready, true);
  evidence.checks.push({ name: 'DeepSeek selected readiness', verdict: 'PASS', result: unpack(deepseekStatus) });
  const deepseek = await main.client.callTool({ name: 'ask_exo', arguments: { model: 'deepseek', prompt: 'What is 17 plus 25? Answer briefly.' } }, undefined, { timeout: 60000 });
  assert.ok(!deepseek.isError, JSON.stringify(deepseek));
  const ds = unpack(deepseek);
  assert.equal(ds.model, DEEPSEEK_MODEL); assert.match(ds.answer, /\b42\b/);
  assert.equal(ds.truncated, false); assert.ok(ds.response_id);
  evidence.checks.push({ name: 'Explicit DeepSeek MCP inference', verdict: 'PASS', result: ds });
  const invalidModel = await main.client.callTool({ name: 'ask_exo', arguments: { model: 'unknown', prompt: 'hello' } });
  assert.equal(invalidModel.isError, true);
  evidence.checks.push({ name: 'Unknown model rejected by MCP schema', verdict: 'PASS' });
  const oversized = await main.client.callTool({ name: 'ask_exo', arguments: { prompt: 'x'.repeat(16000), context: 'too much' } });
  assert.equal(oversized.isError, true); assert.equal(unpack(oversized).code, 'INPUT_TOO_LARGE');
  evidence.checks.push({ name: 'Oversized combined input rejected', verdict: 'PASS' });
  const missing = await connect('exo-helper-missing-model', { EXO_MODEL: 'verification/nonexistent-model' });
  const absent = await missing.client.callTool({ name: 'ask_exo', arguments: { prompt: 'hello' } });
  assert.equal(absent.isError, true); assert.equal(unpack(absent).code, 'MODEL_NOT_READY');
  evidence.checks.push({ name: 'Missing model fails without load/download/fallback', verdict: 'PASS', error: unpack(absent) });
  const unavailable = await connect('exo-helper-unreachable', { EXO_BASE_URL: 'http://127.0.0.1:1' });
  const offline = await unavailable.client.callTool({ name: 'exo_status', arguments: {} });
  assert.equal(offline.isError, true); assert.equal(unpack(offline).code, 'EXO_UNREACHABLE');
  evidence.checks.push({ name: 'Unavailable Exo fails visibly', verdict: 'PASS' });
  assert.equal(main.stderr(), '');
  evidence.verdict = 'PASS';
} catch (error) {
  evidence.verdict = 'FAIL'; evidence.error = error.message; process.exitCode = 1;
} finally {
  await Promise.all(clients.map(c => c.close().catch(() => {})));
  evidence.retained_state = 'Qwen VL4B and DeepSeek R1-0528 8B remain loaded locally for helper use. Test MCP processes closed. No filesystem tools, automatic model load, paid fallback, or Ollama runtime.';
  await writeFile(root + 'verification/integration.json', JSON.stringify(evidence, null, 2) + '\n');
  console.log(JSON.stringify({ verdict: evidence.verdict, checks: evidence.checks.map(c => ({ name: c.name, verdict: c.verdict })), error: evidence.error }, null, 2));
}