fix tokenizers test with restructured models

34f3c4a15577a18f76db1a9c64cbbddcf61a1b34 · 2024-11-13 17:06:11 +0400 · Alex Cheema

Files touched

M test/test_tokenizers.py

Diff

commit 34f3c4a15577a18f76db1a9c64cbbddcf61a1b34
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Nov 13 17:06:11 2024 +0400

    fix tokenizers test with restructured models
---
 test/test_tokenizers.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/test/test_tokenizers.py b/test/test_tokenizers.py
index 64538ec7..ebae91a1 100644
--- a/test/test_tokenizers.py
+++ b/test/test_tokenizers.py
@@ -1,7 +1,7 @@
 import os
 import re
 from transformers import AutoTokenizer, AutoProcessor
-from exo.models import model_base_shards
+from exo.models import model_cards
 
 
 def test_tokenizer(name, tokenizer, verbose=False):
@@ -26,7 +26,12 @@ def test_tokenizer(name, tokenizer, verbose=False):
 
 ignore = ["TriAiExperiments/SFR-Iterative-DPO-LLaMA-3-70B-R", "mlx-community/DeepSeek-Coder-V2-Lite-Instruct-4bit-mlx", "mlx-community/DeepSeek-V2.5-MLX-AQ4_1_64", "llava-hf/llava-1.5-7b-hf", "mlx-community/Qwen*", "dummy", "mlx-community/Meta-Llama-3.1-405B-Instruct-8bit"]
 ignore_pattern = re.compile(r"^(" + "|".join(model.replace("*", ".*") for model in ignore) + r")")
-models = [shard.model_id for shards in model_base_shards.values() for shard in shards.values() if not ignore_pattern.match(shard.model_id)]
+models = []
+for model_id in model_cards:
+  for engine_type, repo_id in model_cards[model_id].get("repo", {}).items():
+    if not ignore_pattern.match(repo_id):
+      models.append(repo_id)
+models = list(set(models))
 
 verbose = os.environ.get("VERBOSE", "0").lower() == "1"
 for m in models:

← 9712d696 Added a small script to compile grpc · back to Exo · disable configure_mlx.sh for now b6945224 →