add more qwen2.5 models: mlx-community/Qwen2.5-7B-Instruct-4bit mlx-community/Qwen2.5-Math-7B-Instruct-4bit mlx-community/Qwen2.5-72B-Instruct-4bit mlx-community/Qwen2.5-Math-72B-Instruct-4bit

dee83e4840b61b46e6c6786abdd1f08396a81832 · 2024-09-18 23:48:19 +0100 · Alex Cheema

Files touched

M exo/models.py
M tinychat/examples/tinychat/index.html

Diff

commit dee83e4840b61b46e6c6786abdd1f08396a81832
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Sep 18 23:48:19 2024 +0100

    add more qwen2.5 models: mlx-community/Qwen2.5-7B-Instruct-4bit mlx-community/Qwen2.5-Math-7B-Instruct-4bit mlx-community/Qwen2.5-72B-Instruct-4bit mlx-community/Qwen2.5-Math-72B-Instruct-4bit
---
 exo/models.py                         | 12 ++++++++++++
 tinychat/examples/tinychat/index.html |  4 ++++
 2 files changed, 16 insertions(+)

diff --git a/exo/models.py b/exo/models.py
index 3fe95d6b..30e927e4 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -31,7 +31,19 @@ model_base_shards = {
   ### llava
   "llava-1.5-7b-hf": {"MLXDynamicShardInferenceEngine": Shard(model_id="llava-hf/llava-1.5-7b-hf", start_layer=0, end_layer=0, n_layers=32),},
   ### qwen
+  "qwen-2.5-7b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-7B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=28),
+  },
+  "qwen-2.5-math-7b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-Math-7B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=28),
+  },
   "qwen-2.5-14b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-14B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=48),
   },
+  "qwen-2.5-72b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-72B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=80),
+  },
+  "qwen-2.5-math-72b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-Math-72B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=80),
+  },
 }
diff --git a/tinychat/examples/tinychat/index.html b/tinychat/examples/tinychat/index.html
index 4a7a95f3..3e5164d6 100644
--- a/tinychat/examples/tinychat/index.html
+++ b/tinychat/examples/tinychat/index.html
@@ -37,7 +37,11 @@
 <option value="mistral-large">Mistral Large</option>
 <option value="deepseek-coder-v2-lite">Deepseek Coder V2 Lite</option>
 <option value="llava-1.5-7b-hf">LLaVa 1.5 7B (Vision Model)</option>
+<option value="qwen-2.5-7b">Qwen 2.5 7B</option>
+<option value="qwen-2.5-math-7b">Qwen 2.5 7B (Math)</option>
 <option value="qwen-2.5-14b">Qwen 2.5 14B</option>
+<option value="qwen-2.5-72b">Qwen 2.5 72B</option>
+<option value="qwen-2.5-math-72b">Qwen 2.5 72B (Math)</option>
 </select>
 </div>
 <div @popstate.window="

← 3597fba3 add support for qwen2.5, initially adding mlx-community/Qwen · back to Exo · ignore Qwen models in tokenizers test until bos issue is fix 68028cc9 →