← back to Exo

report macmon failures more aggressively (#1618)

eab3e0b45667cf9b247070e47ff99834eab762b6 · 2026-02-25 17:49:10 +0000 · Evan Quiney

macmon appears to be going silent. time it out and restart it after
3*interval (Default 3s) and report warning.

Co-authored-by: Alex Cheema <41707476+AlexCheema@users.noreply.github.com>

Files touched

• M src/exo/utils/info_gatherer/info_gatherer.py

Diff

commit eab3e0b45667cf9b247070e47ff99834eab762b6
Author: Evan Quiney <evanev7@gmail.com>
Date:   Wed Feb 25 17:49:10 2026 +0000

    report macmon failures more aggressively (#1618)
    
    macmon appears to be going silent. time it out and restart it after
    3*interval (Default 3s) and report warning.
    
    Co-authored-by: Alex Cheema <41707476+AlexCheema@users.noreply.github.com>
---
 src/exo/utils/info_gatherer/info_gatherer.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/exo/utils/info_gatherer/info_gatherer.py b/src/exo/utils/info_gatherer/info_gatherer.py
index 9cb04c06..1efb1a90 100644
--- a/src/exo/utils/info_gatherer/info_gatherer.py
+++ b/src/exo/utils/info_gatherer/info_gatherer.py
@@ -542,10 +542,13 @@ class InfoGatherer:
                     if not p.stdout:
                         logger.critical("MacMon closed stdout")
                         return
-                    async for text in TextReceiveStream(
-                        BufferedByteReceiveStream(p.stdout)
-                    ):
-                        await self.info_sender.send(MacmonMetrics.from_raw_json(text))
+                    t = TextReceiveStream(BufferedByteReceiveStream(p.stdout))
+                    while True:
+                        with anyio.fail_after(self.macmon_interval * 3):
+                            macmon_output = await t.receive()
+                            await self.info_sender.send(
+                                MacmonMetrics.from_raw_json(macmon_output)
+                            )
             except CalledProcessError as e:
                 stderr_msg = "no stderr"
                 stderr_output = cast(bytes | str | None, e.stderr)
@@ -556,8 +559,12 @@ class InfoGatherer:
                         else str(stderr_output)
                     )
                 logger.warning(
-                    f"MacMon failed with return code {e.returncode}: {stderr_msg}"
+                    f"memory monitor failed with return code {e.returncode}: {stderr_msg}"
+                )
+            except TimeoutError:
+                logger.warning(
+                    f"memory monitor silent for {self.macmon_interval * 3}s - reloading"
                 )
             except Exception as e:
-                logger.warning(f"Error in macmon monitor: {e}")
+                logger.opt(exception=e).warning("Error in memory monitor")
             await anyio.sleep(self.macmon_interval)

← c4e874e9 skip nan logprobs on tokens (#1622)  ·  back to Exo  ·  Revert "report macmon failures more aggressively (#1618)" (# ba611f9c →