use fast AutoProcessor fixes #164 tokenizer issues with mistral-large.

0d218e244ef8bc32f6bdf4120e5b3174fee1b9f9 · 2024-08-22 14:43:08 +0100 · Alex Cheema

Files touched

M exo/inference/tokenizers.py

Diff

commit 0d218e244ef8bc32f6bdf4120e5b3174fee1b9f9
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Aug 22 14:43:08 2024 +0100

    use fast AutoProcessor fixes #164 tokenizer issues with mistral-large.
---
 exo/inference/tokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/inference/tokenizers.py b/exo/inference/tokenizers.py
index e9606798..a234e18d 100644
--- a/exo/inference/tokenizers.py
+++ b/exo/inference/tokenizers.py
@@ -6,7 +6,7 @@ from exo.helpers import DEBUG
 async def resolve_tokenizer(model_id: str):
   try:
     if DEBUG >= 4: print(f"Trying AutoProcessor for {model_id}")
-    processor = AutoProcessor.from_pretrained(model_id, use_fast=False)
+    processor = AutoProcessor.from_pretrained(model_id, use_fast=True)
     if not hasattr(processor, 'eos_token_id'):
       processor.eos_token_id = getattr(processor, 'tokenizer', getattr(processor, '_tokenizer', processor)).eos_token_id
     if not hasattr(processor, 'encode'):

← 23ae5e92 hold circleci tests for approval on non-main branches · back to Exo · tests for tokenizers. unfortunately use_fast=False and use_f e17e5f9a →