add nemotron-70b and nemotron-70b-bf16

61ee67c955f0ba4731b9ef6c2722dfaa359464fd · 2024-10-16 12:36:49 -0700 · Alex Cheema

Files touched

M exo/models.py

Diff

commit 61ee67c955f0ba4731b9ef6c2722dfaa359464fd
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Oct 16 12:36:49 2024 -0700

    add nemotron-70b and nemotron-70b-bf16
---
 exo/models.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/exo/models.py b/exo/models.py
index 857317d8..728958f5 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -59,4 +59,11 @@ model_base_shards = {
   "qwen-2.5-math-72b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Qwen2.5-Math-72B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=80),
   },
+  ### nemotron
+  "nemotron-70b": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/nvidia_Llama-3.1-Nemotron-70B-Instruct-HF_4bit", start_layer=0, end_layer=0, n_layers=80),
+  },
+  "nemotron-70b-bf16": {
+    "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-bf16", start_layer=0, end_layer=0, n_layers=80),
+  },
 }

← 3e33bca7 TFLOPS on GTX 1660 - cabelo@opensuse.org · back to Exo · add nemotron-70b and nemotron-70b-bf16 to tinychat 1e4524b5 →