← back to Exo Helper

verification/integration.json

262 lines

{
  "ticket": "TK-11325",
  "timestamp": "2026-09-09T18:24:35.093Z",
  "risk_tier": "R2",
  "intent": "Coding client -> stdio MCP -> explicitly selected DeepSeek or default Qwen -> live Exo answer; model absence and invalid input fail visibly",
  "checks": [
    {
      "name": "exo-helper-verification initialize",
      "verdict": "PASS",
      "server": {
        "name": "exo-helper",
        "version": "1.1.0"
      }
    },
    {
      "name": "Tool discovery",
      "verdict": "PASS",
      "tools": [
        "exo_status",
        "ask_exo"
      ]
    },
    {
      "name": "Live model readiness",
      "verdict": "PASS",
      "result": {
        "cluster_online": true,
        "live_nodes": [
          {
            "id": "f6e9949f46b313c459680a7e2a25600a",
            "chip": "Apple M1 Max"
          },
          {
            "id": "68c4727dee8b15c5126f6dde8fcb2218",
            "chip": "Apple M2 Max"
          },
          {
            "id": "9c39e67fb28f1bdc2c270236d4bce0a9",
            "chip": "Apple M3 Ultra"
          }
        ],
        "configured_model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
        "model_ready": true,
        "instances": [
          {
            "id": "cca7f90e-cddd-430f-8c65-fafb9c2ab642",
            "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
            "ready": true,
            "runners": 1,
            "nodes": [
              "9c39e67fb28f1bdc2c270236d4bce0a9"
            ]
          },
          {
            "id": "53563c5b-9700-41da-8a9a-7c55bbaacec9",
            "model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
            "ready": true,
            "runners": 1,
            "nodes": [
              "9c39e67fb28f1bdc2c270236d4bce0a9"
            ]
          }
        ],
        "rdma_enabled": false,
        "selectable_models": [
          {
            "option": "qwen",
            "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
            "ready": true
          },
          {
            "option": "deepseek",
            "model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
            "ready": true
          }
        ]
      }
    },
    {
      "name": "MCP -> live Exo inference -> MCP answer",
      "verdict": "PASS",
      "result": {
        "request_id": "05f91270-29cc-47d9-a293-026dbd56b84c",
        "response_id": "e71e27de-e80d-4a9f-b645-7405687628a6",
        "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
        "answer": "42",
        "finish_reason": "stop",
        "truncated": false,
        "usage": {
          "prompt_tokens": 103,
          "completion_tokens": 3,
          "total_tokens": 106,
          "prompt_tokens_details": {
            "cached_tokens": 101,
            "audio_tokens": 0
          },
          "completion_tokens_details": {
            "reasoning_tokens": 0,
            "audio_tokens": 0,
            "accepted_prediction_tokens": 0,
            "rejected_prediction_tokens": 0
          }
        },
        "elapsed_ms": 1324,
        "review_required": true
      }
    },
    {
      "name": "Bounded extraction task",
      "verdict": "PASS",
      "result": {
        "request_id": "df0fcc22-48b7-4ff1-ab02-42e1d0bf62a6",
        "response_id": "9a149ddc-bf21-4a46-b44f-996753352d3a",
        "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
        "answer": "STATUS=404; FILE=settings.json",
        "finish_reason": "stop",
        "truncated": false,
        "usage": {
          "prompt_tokens": 124,
          "completion_tokens": 10,
          "total_tokens": 134,
          "prompt_tokens_details": {
            "cached_tokens": 122,
            "audio_tokens": 0
          },
          "completion_tokens_details": {
            "reasoning_tokens": 0,
            "audio_tokens": 0,
            "accepted_prediction_tokens": 0,
            "rejected_prediction_tokens": 0
          }
        },
        "elapsed_ms": 1115,
        "review_required": true
      }
    },
    {
      "name": "DeepSeek selected readiness",
      "verdict": "PASS",
      "result": {
        "cluster_online": true,
        "live_nodes": [
          {
            "id": "f6e9949f46b313c459680a7e2a25600a",
            "chip": "Apple M1 Max"
          },
          {
            "id": "68c4727dee8b15c5126f6dde8fcb2218",
            "chip": "Apple M2 Max"
          },
          {
            "id": "9c39e67fb28f1bdc2c270236d4bce0a9",
            "chip": "Apple M3 Ultra"
          }
        ],
        "configured_model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
        "model_ready": true,
        "instances": [
          {
            "id": "cca7f90e-cddd-430f-8c65-fafb9c2ab642",
            "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
            "ready": false,
            "runners": 1,
            "nodes": [
              "9c39e67fb28f1bdc2c270236d4bce0a9"
            ]
          },
          {
            "id": "53563c5b-9700-41da-8a9a-7c55bbaacec9",
            "model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
            "ready": true,
            "runners": 1,
            "nodes": [
              "9c39e67fb28f1bdc2c270236d4bce0a9"
            ]
          }
        ],
        "rdma_enabled": false,
        "selectable_models": [
          {
            "option": "qwen",
            "model": "mlx-community/Qwen3-VL-4B-Instruct-4bit",
            "ready": false
          },
          {
            "option": "deepseek",
            "model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
            "ready": true
          }
        ]
      }
    },
    {
      "name": "Explicit DeepSeek MCP inference",
      "verdict": "PASS",
      "result": {
        "request_id": "867afa9a-7269-422f-8cdf-cf12361814db",
        "response_id": "cafe6493-9f03-40e8-9101-f0e630ae16f6",
        "model": "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",
        "answer": "\n42",
        "finish_reason": "stop",
        "truncated": false,
        "usage": {
          "prompt_tokens": 15,
          "completion_tokens": 771,
          "total_tokens": 786,
          "prompt_tokens_details": {
            "cached_tokens": 9,
            "audio_tokens": 0
          },
          "completion_tokens_details": {
            "reasoning_tokens": 765,
            "audio_tokens": 0,
            "accepted_prediction_tokens": 0,
            "rejected_prediction_tokens": 0
          }
        },
        "elapsed_ms": 20438,
        "review_required": true
      }
    },
    {
      "name": "Unknown model rejected by MCP schema",
      "verdict": "PASS"
    },
    {
      "name": "Oversized combined input rejected",
      "verdict": "PASS"
    },
    {
      "name": "exo-helper-missing-model initialize",
      "verdict": "PASS",
      "server": {
        "name": "exo-helper",
        "version": "1.1.0"
      }
    },
    {
      "name": "Missing model fails without load/download/fallback",
      "verdict": "PASS",
      "error": {
        "code": "MODEL_NOT_READY",
        "message": "verification/nonexistent-model has no ready instance in Exo. Load it in the Exo dashboard, then retry. No download or provider fallback was attempted."
      }
    },
    {
      "name": "exo-helper-unreachable initialize",
      "verdict": "PASS",
      "server": {
        "name": "exo-helper",
        "version": "1.1.0"
      }
    },
    {
      "name": "Unavailable Exo fails visibly",
      "verdict": "PASS"
    }
  ],
  "verdict": "PASS",
  "retained_state": "Qwen VL4B and DeepSeek R1-0528 8B remain loaded locally for helper use. Test MCP processes closed. No filesystem tools, automatic model load, paid fallback, or Ollama runtime."
}