fix tokenizer tests

fdd05baddb4f80fedab3e1844f54699076334ba9 · 2025-01-24 18:13:36 +0000 · Alex Cheema

Files touched

M test/test_tokenizers.py

Diff

commit fdd05baddb4f80fedab3e1844f54699076334ba9
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Fri Jan 24 18:13:36 2025 +0000

    fix tokenizer tests
---
 test/test_tokenizers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_tokenizers.py b/test/test_tokenizers.py
index 7c12902b..65b54cbe 100644
--- a/test/test_tokenizers.py
+++ b/test/test_tokenizers.py
@@ -37,5 +37,6 @@ verbose = os.environ.get("VERBOSE", "0").lower() == "1"
 for m in models:
     # TODO: figure out why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit
     # test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=False), verbose)
-    test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True), verbose)
-    test_tokenizer(m, AutoTokenizer.from_pretrained(m), verbose)
+    if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-V3-4bit"]:
+      test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True, trust_remote_code=True), verbose)
+    test_tokenizer(m, AutoTokenizer.from_pretrained(m, trust_remote_code=True), verbose)

← 59174bdc we have a lot of models so group them nicely · back to Exo · Add adaptive padding for user and assistant messages on widt 9525c0e7 →