one-line output buffering

10e9f44a10abcdc11193836e53ec28e61e123e05 · 2024-11-11 16:16:24 -0800 · Nel Nibcord

Files touched

M exo/orchestration/standard_node.py

Diff

commit 10e9f44a10abcdc11193836e53ec28e61e123e05
Author: Nel Nibcord <blindcrone@tuta.io>
Date:   Mon Nov 11 16:16:24 2024 -0800

    one-line output buffering
---
 exo/orchestration/standard_node.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/exo/orchestration/standard_node.py b/exo/orchestration/standard_node.py
index 2629268f..257b386c 100644
--- a/exo/orchestration/standard_node.py
+++ b/exo/orchestration/standard_node.py
@@ -119,8 +119,7 @@ class StandardNode(Node):
     if request_id not in self.buffered_logits:
       self.buffered_logits[request_id] = ([], False)
 
-    for i in np.reshape(result, (-1, 1, result.shape[-1])):
-      self.buffered_logits[request_id][0].append(i)
+    self.buffered_logits[request_id][0] += [i for i in np.reshape(result, (-1, 1, result.shape[-1]))]
 
     if shard.is_last_layer():
       result = await self.inference_engine.sample(result)

← 52ef6ee4 Made temperature and top_p available to the inference engine · back to Exo · This doesn't need to be a tuple really bf33ffde →