fix model id for llama 3.1 405b now its finally on the hub

94ac9463a73bedf1e7f939042feda608849a66ba · 2024-07-31 10:16:30 +0100 · Alex Cheema

Files touched

M exo/api/chatgpt_api.py

Diff

commit 94ac9463a73bedf1e7f939042feda608849a66ba
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Jul 31 10:16:30 2024 +0100

    fix model id for llama 3.1 405b now its finally on the hub
---
 exo/api/chatgpt_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/api/chatgpt_api.py b/exo/api/chatgpt_api.py
index 2d80f2d5..d4958732 100644
--- a/exo/api/chatgpt_api.py
+++ b/exo/api/chatgpt_api.py
@@ -21,7 +21,7 @@ shard_mappings = {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-70B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-405b": {
-    "MLXDynamicShardInferenceEngine": Shard(model_id="/Users/alex/405b-instruct-4bit", start_layer=0, end_layer=0, n_layers=126),
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-4bit", start_layer=0, end_layer=0, n_layers=126),
   },
   "llama-3-8b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3-8B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=32),

← 178fb75c fix image api prompt encoding · back to Exo · smart prompt longest prefix matching to avoid sending the sa 5c67e24c →