use NousResearch/Meta-Llama-3.1-70B-Instruct as tinygrad llama-3.1-70b model, previously using non-instruct model

dc3b2bde39a5d9b59806bc5970a8fb7fe51b2c75 · 2024-08-30 12:28:24 +0100 · Alex Cheema

Files touched

M exo/models.py

Diff

commit dc3b2bde39a5d9b59806bc5970a8fb7fe51b2c75
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Fri Aug 30 12:28:24 2024 +0100

    use NousResearch/Meta-Llama-3.1-70B-Instruct as tinygrad llama-3.1-70b model, previously using non-instruct model
---
 exo/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/models.py b/exo/models.py
index d355e88d..1c3816ac 100644
--- a/exo/models.py
+++ b/exo/models.py
@@ -8,7 +8,7 @@ model_base_shards = {
   },
   "llama-3.1-70b": {
     "MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-70B-Instruct-4bit", start_layer=0, end_layer=0, n_layers=80),
-    "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B", start_layer=0, end_layer=0, n_layers=80),
+    "TinygradDynamicShardInferenceEngine": Shard(model_id="NousResearch/Meta-Llama-3.1-70B-Instruct", start_layer=0, end_layer=0, n_layers=80),
   },
   "llama-3.1-405b": {"MLXDynamicShardInferenceEngine": Shard(model_id="mlx-community/Meta-Llama-3.1-405B-4bit", start_layer=0, end_layer=0, n_layers=126),},
   "llama-3-8b": {

← 12609cb6 integration test for udp discovery with grpc server · back to Exo · todo for speculative model 57215041 →