trust_remote_code=True when loading tokenizer

e138aa6a887675aa75cbd49631b2bd9672794ce9 · 2024-09-20 18:47:45 +0100 · Alex Cheema

Files touched

M exo/inference/tokenizers.py

Diff

commit e138aa6a887675aa75cbd49631b2bd9672794ce9
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Fri Sep 20 18:47:45 2024 +0100

    trust_remote_code=True when loading tokenizer
---
 exo/inference/tokenizers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/exo/inference/tokenizers.py b/exo/inference/tokenizers.py
index bf4b5a6c..c0005ed1 100644
--- a/exo/inference/tokenizers.py
+++ b/exo/inference/tokenizers.py
@@ -22,7 +22,7 @@ async def resolve_tokenizer(model_id: str):
 async def _resolve_tokenizer(model_id_or_local_path: Union[str, PathLike]):
   try:
     if DEBUG >= 4: print(f"Trying AutoProcessor for {model_id_or_local_path}")
-    processor = AutoProcessor.from_pretrained(model_id_or_local_path, use_fast=True if "Mistral-Large" in f"{model_id_or_local_path}" else False)
+    processor = AutoProcessor.from_pretrained(model_id_or_local_path, use_fast=True if "Mistral-Large" in f"{model_id_or_local_path}" else False, trust_remote_code=True)
     if not hasattr(processor, 'eos_token_id'):
       processor.eos_token_id = getattr(processor, 'tokenizer', getattr(processor, '_tokenizer', processor)).eos_token_id
     if not hasattr(processor, 'encode'):
@@ -36,7 +36,7 @@ async def _resolve_tokenizer(model_id_or_local_path: Union[str, PathLike]):
 
   try:
     if DEBUG >= 4: print(f"Trying AutoTokenizer for {model_id_or_local_path}")
-    return AutoTokenizer.from_pretrained(model_id_or_local_path)
+    return AutoTokenizer.from_pretrained(model_id_or_local_path, trust_remote_code=True)
   except Exception as e:
     if DEBUG >= 4: print(f"Failed to load tokenizer for {model_id_or_local_path}. Falling back to tinygrad tokenizer. Error: {e}")
     if DEBUG >= 4: print(traceback.format_exc())

← 9ee8a090 trigger ci · back to Exo · update readme with tips for performance for apple silicon ma cb633868 →