← back to Exo
Add error handling to info gatherer monitor loops (#1422)
163cf183845b961e2828c386b05f5a4be965e8f6 · 2026-02-10 04:24:43 -0800 · Alex Cheema
## Motivation
If any of the `InfoGatherer` monitor loops throw an unexpected
exception, the entire monitoring task crashes and never recovers. This
can silently stop memory, network, or Thunderbolt data collection for
the lifetime of the process.
## Changes
Wrap the body of each `while True` monitor loop in a try/except that
logs the exception as a warning and continues to the next iteration. The
sleep at the end of each loop runs regardless, providing natural backoff
before retry.
Affected methods: `_monitor_misc`,
`_monitor_system_profiler_thunderbolt_data`, `_monitor_memory_usage`,
`_watch_system_info`, `_monitor_thunderbolt_bridge_status`.
`_monitor_macmon` already had its own error handling so was left as-is.
## Why It Works
A transient error (e.g., a subprocess failing, a permission issue) in
one iteration no longer kills the loop. The warning log provides
visibility while the monitor continues collecting data on subsequent
iterations.
## Test Plan
### Automated Testing
- `uv run basedpyright` — 0 errors
- `uv run ruff check` — passes
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: rltakashige <rl.takashige@gmail.com>
Files touched
M src/exo/utils/info_gatherer/info_gatherer.py
Diff
commit 163cf183845b961e2828c386b05f5a4be965e8f6
Author: Alex Cheema <41707476+AlexCheema@users.noreply.github.com>
Date: Tue Feb 10 04:24:43 2026 -0800
Add error handling to info gatherer monitor loops (#1422)
## Motivation
If any of the `InfoGatherer` monitor loops throw an unexpected
exception, the entire monitoring task crashes and never recovers. This
can silently stop memory, network, or Thunderbolt data collection for
the lifetime of the process.
## Changes
Wrap the body of each `while True` monitor loop in a try/except that
logs the exception as a warning and continues to the next iteration. The
sleep at the end of each loop runs regardless, providing natural backoff
before retry.
Affected methods: `_monitor_misc`,
`_monitor_system_profiler_thunderbolt_data`, `_monitor_memory_usage`,
`_watch_system_info`, `_monitor_thunderbolt_bridge_status`.
`_monitor_macmon` already had its own error handling so was left as-is.
## Why It Works
A transient error (e.g., a subprocess failing, a permission issue) in
one iteration no longer kills the loop. The warning log provides
visibility while the monitor continues collecting data on subsequent
iterations.
## Test Plan
### Automated Testing
- `uv run basedpyright` — 0 errors
- `uv run ruff check` — passes
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: rltakashige <rl.takashige@gmail.com>
---
src/exo/utils/info_gatherer/info_gatherer.py | 46 ++++++++++++++++++----------
1 file changed, 30 insertions(+), 16 deletions(-)
diff --git a/src/exo/utils/info_gatherer/info_gatherer.py b/src/exo/utils/info_gatherer/info_gatherer.py
index c8fb4e4c..9fdfcdb3 100644
--- a/src/exo/utils/info_gatherer/info_gatherer.py
+++ b/src/exo/utils/info_gatherer/info_gatherer.py
@@ -352,7 +352,10 @@ class InfoGatherer:
if self.misc_poll_interval is None:
return
while True:
- await self.info_sender.send(await MiscData.gather())
+ try:
+ await self.info_sender.send(await MiscData.gather())
+ except Exception as e:
+ logger.warning(f"Error gathering misc data: {e}")
await anyio.sleep(self.misc_poll_interval)
async def _monitor_system_profiler_thunderbolt_data(self):
@@ -363,15 +366,17 @@ class InfoGatherer:
return
while True:
- data = await ThunderboltConnectivity.gather()
- assert data is not None
-
- idents = [it for i in data if (it := i.ident(iface_map)) is not None]
- await self.info_sender.send(MacThunderboltIdentifiers(idents=idents))
+ try:
+ data = await ThunderboltConnectivity.gather()
+ assert data is not None
- conns = [it for i in data if (it := i.conn()) is not None]
- await self.info_sender.send(MacThunderboltConnections(conns=conns))
+ idents = [it for i in data if (it := i.ident(iface_map)) is not None]
+ await self.info_sender.send(MacThunderboltIdentifiers(idents=idents))
+ conns = [it for i in data if (it := i.conn()) is not None]
+ await self.info_sender.send(MacThunderboltConnections(conns=conns))
+ except Exception as e:
+ logger.warning(f"Error gathering Thunderbolt data: {e}")
await anyio.sleep(self.system_profiler_interval)
async def _monitor_memory_usage(self):
@@ -384,26 +389,35 @@ class InfoGatherer:
if self.memory_poll_rate is None:
return
while True:
- await self.info_sender.send(
- MemoryUsage.from_psutil(override_memory=override_memory)
- )
+ try:
+ await self.info_sender.send(
+ MemoryUsage.from_psutil(override_memory=override_memory)
+ )
+ except Exception as e:
+ logger.warning(f"Error gathering memory usage: {e}")
await anyio.sleep(self.memory_poll_rate)
async def _watch_system_info(self):
if self.interface_watcher_interval is None:
return
while True:
- nics = await get_network_interfaces()
- await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
+ try:
+ nics = await get_network_interfaces()
+ await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
+ except Exception as e:
+ logger.warning(f"Error gathering network interfaces: {e}")
await anyio.sleep(self.interface_watcher_interval)
async def _monitor_thunderbolt_bridge_status(self):
if self.thunderbolt_bridge_poll_interval is None:
return
while True:
- curr = await ThunderboltBridgeInfo.gather()
- if curr is not None:
- await self.info_sender.send(curr)
+ try:
+ curr = await ThunderboltBridgeInfo.gather()
+ if curr is not None:
+ await self.info_sender.send(curr)
+ except Exception as e:
+ logger.warning(f"Error gathering Thunderbolt Bridge status: {e}")
await anyio.sleep(self.thunderbolt_bridge_poll_interval)
async def _monitor_macmon(self, macmon_path: str):
← 2204f651 Yield from reachability checks (#1427)
·
back to Exo
·
cleaning up the todos (#1406) 8314a2aa →