await node process_prompt with timeoout

9223993e29d7cb8753b6b9309c7474d005d7be47 · 2024-10-03 23:17:41 +0400 · Alex Cheema

Files touched

M exo/api/chatgpt_api.py

Diff

commit 9223993e29d7cb8753b6b9309c7474d005d7be47
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Oct 3 23:17:41 2024 +0400

    await node process_prompt with timeoout
---
 exo/api/chatgpt_api.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/exo/api/chatgpt_api.py b/exo/api/chatgpt_api.py
index 1e3eef6c..787cc3dc 100644
--- a/exo/api/chatgpt_api.py
+++ b/exo/api/chatgpt_api.py
@@ -268,9 +268,10 @@ class ChatGPTAPI:
     callback = self.node.on_token.register(callback_id)
 
     if DEBUG >= 2: print(f"Sending prompt from ChatGPT api {request_id=} {shard=} {prompt=} {image_str=}")
-    asyncio.create_task(self.node.process_prompt(shard, prompt, image_str, request_id=request_id))
 
     try:
+      await asyncio.wait_for(self.node.process_prompt(shard, prompt, image_str, request_id=request_id), timeout=self.response_timeout)
+
       if DEBUG >= 2: print(f"Waiting for response to finish. timeout={self.response_timeout}s")
 
       if stream:
@@ -345,6 +346,9 @@ class ChatGPTAPI:
         return web.json_response(generate_completion(chat_request, tokenizer, prompt, request_id, tokens, stream, finish_reason, "chat.completion"))
     except asyncio.TimeoutError:
       return web.json_response({"detail": "Response generation timed out"}, status=408)
+    except Exception as e:
+      if DEBUG >= 2: traceback.print_exc()
+      return web.json_response({"detail": f"Error processing prompt (see logs with DEBUG>=2): {str(e)}"}, status=500)
     finally:
       deregistered_callback = self.node.on_token.deregister(callback_id)
       if DEBUG >= 2: print(f"Deregister {callback_id=} {deregistered_callback=}")

← b611d0a5 fix print · back to Exo · only stream results for the same request id. this allows mul 9b9f40d4 →