Point `llama-3.1-70b-bf16` model to the actually bf16 version

82c7ce69f6ea966daa1d8b96f3a837d20b193bc7 · 2024-10-05 11:43:46 -0700 · Mark Van Aken

Files touched

M exo/models.py

Diff

commit 82c7ce69f6ea966daa1d8b96f3a837d20b193bc7
Author: Mark Van Aken <1727697+vanakema@users.noreply.github.com>
Date:   Sat Oct 5 11:43:46 2024 -0700

    Point `llama-3.1-70b-bf16` model to the actually bf16 version
---
 exo/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/models.py b/exo/models.py
index 1eaabed3..857317d8 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -17,7 +17,7 @@ model_base_shards = {
     "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B-Instruct", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-70b-bf16": {
-    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-70B-Instruct-bf16", start_layer=0, end_layer=0, n_layers=80),
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-70B-Instruct-bf16-CORRECTED", start_layer=0, end_layer=0, n_layers=80),
     "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B-Instruct", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-405b": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-4bit", start_layer=0, end_layer=0, n_layers=126),},

← 9ffd8116 Use official nvidia-ml-py instead of pynvml · back to Exo · replace tailscale.devices with good old http, removing the n e8a87023 →