← back to Exo

add llama-3.2-1b-8bit, llama-3.2-3b-8bit, llama-3.2-3b-bf16

a0bada3b2af443b3007d1d4e82e64aafb87f7807 · 2024-12-12 17:13:34 +0000 · Alex Cheema

Files touched

• M exo/models.py

Diff

commit a0bada3b2af443b3007d1d4e82e64aafb87f7807
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Dec 12 17:13:34 2024 +0000

    add llama-3.2-1b-8bit, llama-3.2-3b-8bit, llama-3.2-3b-bf16
---
 exo/models.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/exo/models.py b/exo/models.py
index 0d4c6e3e..0f984d48 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -17,7 +17,28 @@ model_cards = {
       "TinygradDynamicShardInferenceEngine": "unsloth/Llama-3.2-1B-Instruct",
     },
   },
+  "llama-3.2-1b-8bit": {
+    "layers": 16,
+    "repo": {
+      "MLXDynamicShardInferenceEngine": "mlx-community/Llama-3.2-1B-Instruct-8bit",
+      "TinygradDynamicShardInferenceEngine": "unsloth/Llama-3.2-1B-Instruct",
+    },
+  },
   "llama-3.2-3b": {
+    "layers": 28,
+    "repo": {
+       "MLXDynamicShardInferenceEngine": "mlx-community/Llama-3.2-3B-Instruct-4bit",
+       "TinygradDynamicShardInferenceEngine": "unsloth/Llama-3.2-3B-Instruct",
+    },
+  },
+  "llama-3.2-3b-8bit": {
+    "layers": 28,
+    "repo": {
+       "MLXDynamicShardInferenceEngine": "mlx-community/Llama-3.2-3B-Instruct-8bit",
+       "TinygradDynamicShardInferenceEngine": "unsloth/Llama-3.2-3B-Instruct",
+    },
+  },
+  "llama-3.2-3b-bf16": {
     "layers": 28,
     "repo": {
        "MLXDynamicShardInferenceEngine": "mlx-community/Llama-3.2-3B-Instruct",
@@ -94,7 +115,10 @@ model_cards = {
 pretty_name = {
   "llama-3.3-70b": "Llama 3.3 70B",
   "llama-3.2-1b": "Llama 3.2 1B",
+  "llama-3.2-1b-8bit": "Llama 3.2 1B (8-bit)",
   "llama-3.2-3b": "Llama 3.2 3B",
+  "llama-3.2-3b-8bit": "Llama 3.2 3B (8-bit)",
+  "llama-3.2-3b-bf16": "Llama 3.2 3B (BF16)",
   "llama-3.1-8b": "Llama 3.1 8B",
   "llama-3.1-70b": "Llama 3.1 70B",
   "llama-3.1-70b-bf16": "Llama 3.1 70B (BF16)",

← b6f2385c run llama-3.1-8b on 3 m4 pro cluster  ·  back to Exo  ·  add llama-3.3-70b to 3 M4 Pro cluster e5d54c77 →