add llama-3.1-405b-8bit

c8438b6d2396deb5b3bce2453e831e77574e2c0e · 2024-11-06 12:29:21 +0400 · Alex Cheema

Files touched

M exo/models.py
M exo/tinychat/index.html

Diff

commit c8438b6d2396deb5b3bce2453e831e77574e2c0e
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Nov 6 12:29:21 2024 +0400

    add llama-3.1-405b-8bit
---
 exo/models.py           | 1 +
 exo/tinychat/index.html | 1 +
 2 files changed, 2 insertions(+)

diff --git a/exo/models.py b/exo/models.py
index 5791bdd5..e512f918 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -17,6 +17,7 @@ model_base_shards = {
     "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B-Instruct", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-405b": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-4bit", start_layer=0, end_layer=0, n_layers=126),},
+  "llama-3.1-405b-8bit": {"MLXDynamicShardInferenceEngine": Shard(model_id="IntuitIntel/Llama-3.1-405B-Instruct", start_layer=0, end_layer=0, n_layers=126),},
   "llama-3-8b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3-8B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=32),
     "TinygradDynamicShardInferenceEngine": Shard(model_id="TriAiExperiments/SFR-Iterative-DPO-LLaMA-3-8B-R", start_layer=0, end_layer=0, n_layers=32),
diff --git a/exo/tinychat/index.html b/exo/tinychat/index.html
index 1f698f3c..811f206f 100644
--- a/exo/tinychat/index.html
+++ b/exo/tinychat/index.html
@@ -36,6 +36,7 @@
 <option value="llama-3.1-70b">Llama 3.1 70B</option>
 <option value="llama-3.1-70b-bf16">Llama 3.1 70B (BF16)</option>
 <option value="llama-3.1-405b">Llama 3.1 405B</option>
+<option value="llama-3.1-405b-8bit">Llama 3.1 405B (8-bit)</option>
 <option value="llama-3-8b">Llama 3 8B</option>
 <option value="llama-3-70b">Llama 3 70B</option>
 <option value="nemotron-70b">Nemotron 70B</option>

← 6ae6ebeb revert back to CORRECTED · back to Exo · update llama-3.1-405b-8bit model id to IntuitIntel/Meta-Llam 36c1f68c →