add ministral-8b

6bd0f07a326631c47b98e682b533e0c6c7297fed · 2024-10-17 14:31:23 -0700 · Alex Cheema

Files touched

M exo/models.py

Diff

commit 6bd0f07a326631c47b98e682b533e0c6c7297fed
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Oct 17 14:31:23 2024 -0700

    add ministral-8b
---
 exo/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/exo/models.py b/exo/models.py
index 728958f5..aa0edfdf 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -32,6 +32,7 @@ model_base_shards = {
   ### mistral
   "mistral-nemo": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Mistral-Nemo-Instruct-2407-4bit", start_layer=0, end_layer=0, n_layers=40),},
   "mistral-large": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Mistral-Large-Instruct-2407-4bit", start_layer=0, end_layer=0, n_layers=88),},
+  "ministral-8b": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Ministral-8B-Instruct-2410-4bit", start_layer=0, end_layer=0, n_layers=32),},
   ### deepseek
   "deepseek-coder-v2-lite": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/DeepSeek-Coder-V2-Lite-Instruct-4bit-mlx", start_layer=0, end_layer=0, n_layers=27),},
   "deepseek-coder-v2.5": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/DeepSeek-V2.5-MLX-AQ4_1_64", start_layer=0, end_layer=0, n_layers=60),},

← 1e4524b5 add nemotron-70b and nemotron-70b-bf16 to tinychat · back to Exo · DummyInferenceEngine commit 1 7d610475 →