fix names in dummy tokenizer

185b1e375c1e8c011c429d7cb18b30dde575c7f4 · 2024-12-24 02:08:20 +0000 · Alex Cheema

Files touched

M exo/inference/tokenizers.py

Diff

commit 185b1e375c1e8c011c429d7cb18b30dde575c7f4
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Tue Dec 24 02:08:20 2024 +0000

    fix names in dummy tokenizer
---
 exo/inference/tokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/inference/tokenizers.py b/exo/inference/tokenizers.py
index 6b1439fc..4dccaf66 100644
--- a/exo/inference/tokenizers.py
+++ b/exo/inference/tokenizers.py
@@ -14,7 +14,7 @@ class DummyTokenizer:
     self.eos_token_id = 69
     self.vocab_size = 1000
 
-  def apply_chat_template(self, messages, tokenize=True, add_generation_prompt=True):
+  def apply_chat_template(self, conversation, tokenize=True, add_generation_prompt=True, tools=None, **kwargs):
     return "dummy_tokenized_prompt"
 
   def encode(self, text):

← 078b8076 fix names of qwen models · back to Exo · add exaone-3.5 LLM Model 94a5e908 →