only use_fast tokenizer for Mistral Large until this inconsistency bug is fixed #171

8cad0e18493d423f3453f70583786ccb9e193e49 · 2024-08-24 01:19:14 +0100 · Alex Cheema

Files touched

M exo/inference/tokenizers.py

Diff

commit 8cad0e18493d423f3453f70583786ccb9e193e49
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sat Aug 24 01:19:14 2024 +0100

    only use_fast tokenizer for Mistral Large until this inconsistency bug is fixed #171
---
 exo/inference/tokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/inference/tokenizers.py b/exo/inference/tokenizers.py
index a234e18d..179987b8 100644
--- a/exo/inference/tokenizers.py
+++ b/exo/inference/tokenizers.py
@@ -6,7 +6,7 @@ from exo.helpers import DEBUG
 async def resolve_tokenizer(model_id: str):
   try:
     if DEBUG >= 4: print(f"Trying AutoProcessor for {model_id}")
-    processor = AutoProcessor.from_pretrained(model_id, use_fast=True)
+    processor = AutoProcessor.from_pretrained(model_id, use_fast=True if "Mistral-Large" in model_id else False)
     if not hasattr(processor, 'eos_token_id'):
       processor.eos_token_id = getattr(processor, 'tokenizer', getattr(processor, '_tokenizer', processor)).eos_token_id
     if not hasattr(processor, 'encode'):

← 85279007 hotfix edge case where we try to render before tokenizer is · back to Exo · print traceback on topology collection error 784e6bae →