disable prefix matching on prompts. causes subsequent requests to fail with cannot be broadcast. hotfix for #130

706488732fccd22e5b456091f833b0d77c46a340 · 2024-08-07 14:56:52 +0100 · Alex Cheema

Files touched

M exo/api/chatgpt_api.py

Diff

commit 706488732fccd22e5b456091f833b0d77c46a340
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Aug 7 14:56:52 2024 +0100

    disable prefix matching on prompts. causes subsequent requests to fail with cannot be broadcast. hotfix for #130
---
 exo/api/chatgpt_api.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/exo/api/chatgpt_api.py b/exo/api/chatgpt_api.py
index 87390b7f..fbb4c28e 100644
--- a/exo/api/chatgpt_api.py
+++ b/exo/api/chatgpt_api.py
@@ -305,18 +305,19 @@ class ChatGPTAPI:
     if DEBUG >= 4: print(f"Resolved tokenizer: {tokenizer}")
 
     prompt, image_str = build_prompt(tokenizer, chat_request.messages)
-    request_id = None
-    match = self.prompts.find_longest_prefix(prompt)
-    if match and len(prompt) > len(match[1].prompt):
-        if DEBUG >= 2:
-          print(f"Prompt for request starts with previous prompt {len(match[1].prompt)} of {len(prompt)}: {match[1].prompt}")
-        request_id = match[1].request_id
-        self.prompts.add(prompt, PromptSession(request_id=request_id, timestamp=int(time.time()), prompt=prompt))
-        # remove the matching prefix from the prompt
-        prompt = prompt[len(match[1].prompt):]
-    else:
-      request_id = str(uuid.uuid4())
-      self.prompts.add(prompt, PromptSession(request_id=request_id, timestamp=int(time.time()), prompt=prompt))
+    request_id = str(uuid.uuid4())
+    # request_id = None
+    # match = self.prompts.find_longest_prefix(prompt)
+    # if match and len(prompt) > len(match[1].prompt):
+    #     if DEBUG >= 2:
+    #       print(f"Prompt for request starts with previous prompt {len(match[1].prompt)} of {len(prompt)}: {match[1].prompt}")
+    #     request_id = match[1].request_id
+    #     self.prompts.add(prompt, PromptSession(request_id=request_id, timestamp=int(time.time()), prompt=prompt))
+    #     # remove the matching prefix from the prompt
+    #     prompt = prompt[len(match[1].prompt):]
+    # else:
+    #   request_id = str(uuid.uuid4())
+    #   self.prompts.add(prompt, PromptSession(request_id=request_id, timestamp=int(time.time()), prompt=prompt))
 
     callback_id = f"chatgpt-api-wait-response-{request_id}"
     callback = self.node.on_token.register(callback_id)

← 35b7042e upgrade mlx to 0.16.1 · back to Exo · cache completed download paths dd41026c →