fix token encode to use the right model

9f86737a9480d6d0b7607ee4f265e24a2fe2d07a · 2024-12-07 19:15:21 +0000 · Alex Cheema

Files touched

M exo/api/chatgpt_api.py

Diff

commit 9f86737a9480d6d0b7607ee4f265e24a2fe2d07a
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sat Dec 7 19:15:21 2024 +0000

    fix token encode to use the right model
---
 exo/api/chatgpt_api.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/exo/api/chatgpt_api.py b/exo/api/chatgpt_api.py
index 4d4c59e3..ef3d33cc 100644
--- a/exo/api/chatgpt_api.py
+++ b/exo/api/chatgpt_api.py
@@ -278,7 +278,13 @@ class ChatGPTAPI:
 
   async def handle_post_chat_token_encode(self, request):
     data = await request.json()
-    shard = build_base_shard(self.default_model, self.inference_engine_classname)
+    model = data.get("model", self.default_model)
+    if model and model.startswith("gpt-"):  # Handle gpt- model requests
+      model = self.default_model
+    if not model or model not in model_cards:
+      if DEBUG >= 1: print(f"Invalid model: {model}. Supported: {list(model_cards.keys())}. Defaulting to {self.default_model}")
+      model = self.default_model
+    shard = build_base_shard(model, self.inference_engine_classname)
     messages = [parse_message(msg) for msg in data.get("messages", [])]
     tokenizer = await resolve_tokenizer(get_repo(shard.model_id, self.inference_engine_classname))
     return web.json_response({"length": len(build_prompt(tokenizer, messages)[0])})

← 24130da4 prio mac check for interface · back to Exo · fix encode endpoint c5934348 →