support for streaming from non-tail nodes on chatgpt api, addresses #20

8df2f4d85757d808089c3d91f213c828ad5a3d94 · 2024-07-17 19:05:02 -0700 · Alex Cheema

Files touched

M exo/orchestration/standard_node.py

Diff

commit 8df2f4d85757d808089c3d91f213c828ad5a3d94
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Jul 17 19:05:02 2024 -0700

    support for streaming from non-tail nodes on chatgpt api, addresses #20
---
 exo/orchestration/standard_node.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/exo/orchestration/standard_node.py b/exo/orchestration/standard_node.py
index d9c5a5ca..5873d35e 100644
--- a/exo/orchestration/standard_node.py
+++ b/exo/orchestration/standard_node.py
@@ -54,7 +54,7 @@ class StandardNode(Node):
         is_finished = is_finished or len(self.buffered_token_output[request_id][0]) >= self.max_generate_tokens
         if is_finished:
             self.buffered_token_output[request_id] = (self.buffered_token_output[request_id][0], True)
-            asyncio.create_task(self.broadcast_result(request_id, self.buffered_token_output[request_id][0], is_finished))
+        asyncio.create_task(self.broadcast_result(request_id, self.buffered_token_output[request_id][0], is_finished)) # TODO: this is n^2 communication complexity
 
         if result.size == 1:
             self.buffered_token_output[request_id][0].append(result.item())
@@ -79,7 +79,7 @@ class StandardNode(Node):
             is_finished = is_finished or len(self.buffered_token_output[request_id][0]) >= self.max_generate_tokens
             if is_finished:
                 self.buffered_token_output[request_id] = (self.buffered_token_output[request_id][0], True)
-                asyncio.create_task(self.broadcast_result(request_id, self.buffered_token_output[request_id][0], is_finished))
+            asyncio.create_task(self.broadcast_result(request_id, self.buffered_token_output[request_id][0], is_finished)) # TODO: this is n^2 communication complexity
 
             if result.size == 1:  # we got a new token out
                 self.buffered_token_output[request_id][0].append(result.item())

← 8762effa chatgpt api repsonse streaming solves #20 · back to Exo · check if inference_engine has tokenizer before printing with 1e1e11cd →