throttle repo progress events and only send them out if something changed

3675804f4d88a83b0dd83033f5fe3402c336c498 · 2025-01-29 18:55:54 +0000 · Alex Cheema

Files touched

M exo/main.py

Diff

commit 3675804f4d88a83b0dd83033f5fe3402c336c498
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Jan 29 18:55:54 2025 +0000

    throttle repo progress events and only send them out if something changed
---
 exo/main.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/exo/main.py b/exo/main.py
index 5b20808b..e16daea9 100644
--- a/exo/main.py
+++ b/exo/main.py
@@ -206,14 +206,16 @@ def preemptively_load_shard(request_id: str, opaque_status: str):
       traceback.print_exc()
 node.on_opaque_status.register("preemptively_load_shard").on_next(preemptively_load_shard)
 
-last_broadcast_time = 0
+last_events: dict[str, tuple[float, RepoProgressEvent]] = {}
 def throttled_broadcast(shard: Shard, event: RepoProgressEvent):
-  global last_broadcast_time
+  global last_events
   current_time = time.time()
   if event.status == "not_started": return
-  if event.status == "complete" or current_time - last_broadcast_time >= 0.1:
-    last_broadcast_time = current_time
-    asyncio.create_task(node.broadcast_opaque_status("", json.dumps({"type": "download_progress", "node_id": node.id, "progress": event.to_dict()})))
+  last_event = last_events.get(shard.model_id)
+  if last_event and last_event[1].status == "complete" and event.status == "complete": return
+  if last_event and last_event[0] == event.status and current_time - last_event[0] < 0.2: return
+  last_events[shard.model_id] = (current_time, event)
+  asyncio.create_task(node.broadcast_opaque_status("", json.dumps({"type": "download_progress", "node_id": node.id, "progress": event.to_dict()})))
 shard_downloader.on_progress.register("broadcast").on_next(throttled_broadcast)
 
 async def run_model_cli(node: Node, model_name: str, prompt: str):

← 96f1aecb only in_progress if any given file is in_progress · back to Exo · Add 4-bit to the end of DeepSeek V3/R1 model descriptions a25e02c9 →