add support for llama 3.2

777102c9970863094ce52c3ffbafddfacdbb38ba · 2024-09-25 19:48:04 +0100 · Alex Cheema

Files touched

M exo/models.py
M tinychat/examples/tinychat/index.html

Diff

commit 777102c9970863094ce52c3ffbafddfacdbb38ba
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Sep 25 19:48:04 2024 +0100

    add support for llama 3.2
---
 exo/models.py                         | 6 ++++++
 tinychat/examples/tinychat/index.html | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/exo/models.py b/exo/models.py
index c55424e5..7bf424ce 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -2,6 +2,12 @@ from exo.inference.shard import Shard
 
 model_base_shards = {
   ### llama
+  "llama-3.2-1b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Llama-3.2-1B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=16),
+  },
+  "llama-3.2-3b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Llama-3.2-3B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=28),
+  },
   "llama-3.1-8b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=32),
     "TinygradDynamicShardInferenceEngine": Shard(model_id="mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated", start_layer=0, end_layer=0, n_layers=32),
diff --git a/tinychat/examples/tinychat/index.html b/tinychat/examples/tinychat/index.html
index 2c80d801..f51b7c4c 100644
--- a/tinychat/examples/tinychat/index.html
+++ b/tinychat/examples/tinychat/index.html
@@ -27,7 +27,9 @@
 <main x-data="state" x-init="console.log(endpoint)">
 <div class="model-selector">
 <select @change="if (cstate) cstate.selectedModel = $event.target.value" x-model="cstate.selectedModel">
-<option selected="" value="llama-3.1-8b">Llama 3.1 8B</option>
+<option selected="" value="llama-3.2-1b">Llama 3.2 1B</option>
+<option value="llama-3.2-3b">Llama 3.2 3B</option>
+<option value="llama-3.1-8b">Llama 3.1 8B</option>
 <option value="llama-3.1-70b">Llama 3.1 70B</option>
 <option value="llama-3.1-70b-bf16">Llama 3.1 70B (BF16)</option>
 <option value="llama-3.1-405b">Llama 3.1 405B</option>

← 9831c26d update README with Install Certificates SSL troubleshooting · back to Exo · Add error toast popup 7f9810c6 →