add exception for mlx-community/DeepSeek-R1-3bit and mlx-community/DeepSeek-V3-3bit in tokenizers test

a3766f538aff74aef4690570c733121543d78c0f · 2025-01-26 20:32:48 +0000 · Alex Cheema

Files touched

M test/test_tokenizers.py

Diff

commit a3766f538aff74aef4690570c733121543d78c0f
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sun Jan 26 20:32:48 2025 +0000

    add exception for mlx-community/DeepSeek-R1-3bit and mlx-community/DeepSeek-V3-3bit in tokenizers test
---
 test/test_tokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_tokenizers.py b/test/test_tokenizers.py
index 65b54cbe..6e1b383e 100644
--- a/test/test_tokenizers.py
+++ b/test/test_tokenizers.py
@@ -37,6 +37,6 @@ verbose = os.environ.get("VERBOSE", "0").lower() == "1"
 for m in models:
     # TODO: figure out why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit
     # test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=False), verbose)
-    if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-V3-4bit"]:
+    if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-R1-3bit", "mlx-community/DeepSeek-V3-4bit", "mlx-community/DeepSeek-V3-3bit"]:
       test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True, trust_remote_code=True), verbose)
     test_tokenizer(m, AutoTokenizer.from_pretrained(m, trust_remote_code=True), verbose)

← 82ef0860 add deepseek-v3-3bit and deepseek-r1-3bit · back to Exo · rewrite ShardDownloader, simplify significantly b89495f4 →