← back to Exo

Fix stale state.runners (#1684)

3536161f15264954e226de9a3132a7787e646273 · 2026-03-10 09:32:25 +0000 · ciaranbor

## Motivation

When a runner shuts down, there's no mechanism to remove it from
state.runners, causing stale state to accumulate.

## Changes

- apply.py: apply_runner_status_updated now handles RunnerShutdown
status by removing the runner from state instead of adding/updating it.
Removed the separate apply_runner_deleted function entirely.
- events.py: Removed the RunnerDeleted event type — runner cleanup is
now handled via RunnerStatusUpdated with a RunnerShutdown status.
- Tests: New test_apply_runner_deleted.py with 2 tests: verifying
RunnerShutdown removes the runner, and that a normal status update adds
it.

## Why It Works

Instead of a separate RunnerDeleted event and coordinating its emission
from master/placement, runner cleanup is handled naturally through the
existing RunnerStatusUpdated path — a RunnerShutdown status signals
removal. This is simpler and requires no changes to master or placement
logic.

## Test Plan

### Automated Testing

2 new unit tests covering RunnerShutdown removal and normal runner
status addition

Files touched

• M src/exo/master/tests/test_placement.py
• M src/exo/shared/apply.py
• A src/exo/shared/tests/test_apply/test_apply_runner_deleted.py
• M src/exo/shared/types/events.py

Diff

commit 3536161f15264954e226de9a3132a7787e646273
Author: ciaranbor <81697641+ciaranbor@users.noreply.github.com>
Date:   Tue Mar 10 09:32:25 2026 +0000

    Fix stale state.runners (#1684)
    
    ## Motivation
    
    When a runner shuts down, there's no mechanism to remove it from
    state.runners, causing stale state to accumulate.
    
    ## Changes
    
    - apply.py: apply_runner_status_updated now handles RunnerShutdown
    status by removing the runner from state instead of adding/updating it.
    Removed the separate apply_runner_deleted function entirely.
    - events.py: Removed the RunnerDeleted event type — runner cleanup is
    now handled via RunnerStatusUpdated with a RunnerShutdown status.
    - Tests: New test_apply_runner_deleted.py with 2 tests: verifying
    RunnerShutdown removes the runner, and that a normal status update adds
    it.
    
    ## Why It Works
    
    Instead of a separate RunnerDeleted event and coordinating its emission
    from master/placement, runner cleanup is handled naturally through the
    existing RunnerStatusUpdated path — a RunnerShutdown status signals
    removal. This is simpler and requires no changes to master or placement
    logic.
    
    ## Test Plan
    
    ### Automated Testing
    
    2 new unit tests covering RunnerShutdown removal and normal runner
    status addition
---
 src/exo/master/tests/test_placement.py             |  6 ++++-
 src/exo/shared/apply.py                            | 22 ++++++------------
 .../tests/test_apply/test_apply_runner_deleted.py  | 26 ++++++++++++++++++++++
 src/exo/shared/types/events.py                     |  5 -----
 4 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/src/exo/master/tests/test_placement.py b/src/exo/master/tests/test_placement.py
index a97438a0..390a56d2 100644
--- a/src/exo/master/tests/test_placement.py
+++ b/src/exo/master/tests/test_placement.py
@@ -14,7 +14,11 @@ from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
 from exo.shared.topology import Topology
 from exo.shared.types.commands import PlaceInstance
 from exo.shared.types.common import CommandId, NodeId
-from exo.shared.types.events import InstanceCreated, InstanceDeleted, TaskStatusUpdated
+from exo.shared.types.events import (
+    InstanceCreated,
+    InstanceDeleted,
+    TaskStatusUpdated,
+)
 from exo.shared.types.memory import Memory
 from exo.shared.types.multiaddr import Multiaddr
 from exo.shared.types.profiling import NetworkInterfaceInfo, NodeNetworkInfo
diff --git a/src/exo/shared/apply.py b/src/exo/shared/apply.py
index 94869dfe..53f3499e 100644
--- a/src/exo/shared/apply.py
+++ b/src/exo/shared/apply.py
@@ -15,7 +15,6 @@ from exo.shared.types.events import (
     NodeDownloadProgress,
     NodeGatheredInfo,
     NodeTimedOut,
-    RunnerDeleted,
     RunnerStatusUpdated,
     TaskAcknowledged,
     TaskCreated,
@@ -40,7 +39,7 @@ from exo.shared.types.tasks import Task, TaskId, TaskStatus
 from exo.shared.types.topology import Connection, RDMAConnection
 from exo.shared.types.worker.downloads import DownloadProgress
 from exo.shared.types.worker.instances import Instance, InstanceId
-from exo.shared.types.worker.runners import RunnerId, RunnerStatus
+from exo.shared.types.worker.runners import RunnerId, RunnerShutdown, RunnerStatus
 from exo.utils.info_gatherer.info_gatherer import (
     MacmonMetrics,
     MacThunderboltConnections,
@@ -78,8 +77,6 @@ def event_apply(event: Event, state: State) -> State:
             return apply_node_download_progress(event, state)
         case NodeGatheredInfo():
             return apply_node_gathered_info(event, state)
-        case RunnerDeleted():
-            return apply_runner_deleted(event, state)
         case RunnerStatusUpdated():
             return apply_runner_status_updated(event, state)
         case TaskCreated():
@@ -191,23 +188,18 @@ def apply_instance_deleted(event: InstanceDeleted, state: State) -> State:
 
 
 def apply_runner_status_updated(event: RunnerStatusUpdated, state: State) -> State:
-    new_runners: Mapping[RunnerId, RunnerStatus] = {
+    if isinstance(event.runner_status, RunnerShutdown):
+        new_runners: Mapping[RunnerId, RunnerStatus] = {
+            rid: rs for rid, rs in state.runners.items() if rid != event.runner_id
+        }
+        return state.model_copy(update={"runners": new_runners})
+    new_runners = {
         **state.runners,
         event.runner_id: event.runner_status,
     }
     return state.model_copy(update={"runners": new_runners})
 
 
-def apply_runner_deleted(event: RunnerDeleted, state: State) -> State:
-    assert event.runner_id in state.runners, (
-        "RunnerDeleted before any RunnerStatusUpdated events"
-    )
-    new_runners: Mapping[RunnerId, RunnerStatus] = {
-        rid: rs for rid, rs in state.runners.items() if rid != event.runner_id
-    }
-    return state.model_copy(update={"runners": new_runners})
-
-
 def apply_node_timed_out(event: NodeTimedOut, state: State) -> State:
     topology = copy.deepcopy(state.topology)
     topology.remove_node(event.node_id)
diff --git a/src/exo/shared/tests/test_apply/test_apply_runner_deleted.py b/src/exo/shared/tests/test_apply/test_apply_runner_deleted.py
new file mode 100644
index 00000000..57cc9b5e
--- /dev/null
+++ b/src/exo/shared/tests/test_apply/test_apply_runner_deleted.py
@@ -0,0 +1,26 @@
+from exo.shared.apply import apply_runner_status_updated
+from exo.shared.types.events import RunnerStatusUpdated
+from exo.shared.types.state import State
+from exo.shared.types.worker.runners import RunnerId, RunnerIdle, RunnerShutdown
+
+
+def test_apply_runner_shutdown_removes_runner():
+    runner_id = RunnerId()
+    state = State(runners={runner_id: RunnerIdle()})
+
+    new_state = apply_runner_status_updated(
+        RunnerStatusUpdated(runner_id=runner_id, runner_status=RunnerShutdown()), state
+    )
+
+    assert runner_id not in new_state.runners
+
+
+def test_apply_runner_status_updated_adds_runner():
+    runner_id = RunnerId()
+    state = State()
+
+    new_state = apply_runner_status_updated(
+        RunnerStatusUpdated(runner_id=runner_id, runner_status=RunnerIdle()), state
+    )
+
+    assert runner_id in new_state.runners
diff --git a/src/exo/shared/types/events.py b/src/exo/shared/types/events.py
index c7ad2574..c9c2fc3a 100644
--- a/src/exo/shared/types/events.py
+++ b/src/exo/shared/types/events.py
@@ -73,10 +73,6 @@ class RunnerStatusUpdated(BaseEvent):
     runner_status: RunnerStatus
 
 
-class RunnerDeleted(BaseEvent):
-    runner_id: RunnerId
-
-
 class NodeTimedOut(BaseEvent):
     node_id: NodeId
 
@@ -142,7 +138,6 @@ Event = (
     | InstanceCreated
     | InstanceDeleted
     | RunnerStatusUpdated
-    | RunnerDeleted
     | NodeTimedOut
     | NodeGatheredInfo
     | NodeDownloadProgress

← a6aa07ed feat(dashboard): add mobile drawer support for sidebars (#16  ·  back to Exo  ·  Add support for Nemotron sharding (#1693) 82c54dd6 →