add llama 3.1 405b 8bit at mlx-community/Meta-Llama-3.1-405B-Instruct-8bit

98948441e38ac5f47323604a935a50a24ffbdab7 · 2024-11-10 11:46:24 +0400 · Alex Cheema

Files touched

M exo/models.py

Diff

commit 98948441e38ac5f47323604a935a50a24ffbdab7
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sun Nov 10 11:46:24 2024 +0400

    add llama 3.1 405b 8bit at mlx-community/Meta-Llama-3.1-405B-Instruct-8bit
---
 exo/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/models.py b/exo/models.py
index 1d6e934c..3cfe6bb8 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -17,7 +17,7 @@ model_base_shards = {
     "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B-Instruct", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-405b": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-4bit", start_layer=0, end_layer=0, n_layers=126),},
-  "llama-3.1-405b-8bit": {"MLXDynamicShardInferenceEngine": Shard(model_id="IntuitIntel/Meta-Llama-3.1-405B-Instruct-8bit", start_layer=0, end_layer=0, n_layers=126),},
+  "llama-3.1-405b-8bit": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-Instruct-8bit", start_layer=0, end_layer=0, n_layers=126),},
   "llama-3-8b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3-8B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=32),
     "TinygradDynamicShardInferenceEngine": Shard(model_id="TriAiExperiments/SFR-Iterative-DPO-LLaMA-3-8B-R", start_layer=0, end_layer=0, n_layers=32),

← 0d8a1ee4 Added clear all history button · back to Exo · ignore 8bit llama 405b from tokenizers test 47235914 →