← back to Exo Keepalive
auto-save: 2026-07-23T12:51:13 (1 files) — exo.log
4b340bd263d63793ba090a5641b935d8b602ea29 · 2026-07-23 12:51:33 -0700 · Steve Abrams
Files touched
Diff
commit 4b340bd263d63793ba090a5641b935d8b602ea29
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jul 23 12:51:33 2026 -0700
auto-save: 2026-07-23T12:51:13 (1 files) — exo.log
---
exo.log | 695 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 695 insertions(+)
diff --git a/exo.log b/exo.log
index e20b0be..a87c868 100644
--- a/exo.log
+++ b/exo.log
@@ -133452,3 +133452,698 @@ Installed 1 package in 33ms
[ 10:37:51.5099AM | [1mINFO [0m] [1mWaiting for other campaign to finish[0m
[ 10:37:54.5110AM | [1mINFO [0m] [1mNode elected Master - maintaining self[0m
[ 10:37:54.5112AM | [1mINFO [0m] [1mUnpausing API[0m
+[ 12:43:35.7816PM | [1mINFO [0m] [1mManually removing node 3175f9a64495523fe31555701d05903a due to inactivity[0m
+[ 12:43:43.6553PM | [33m[1mWARNING [0m] [33m[1mError gathering disk usage[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b0c74d0>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m569[0m, in [35m_monitor_disk_usage[0m
+ [35m[1mawait[0m [1mself[0m[35m[1m.[0m[1minfo_sender[0m[35m[1m.[0m[1msend[0m[1m([0m[35m[1mawait[0m [1mNodeDiskUsage[0m[35m[1m.[0m[1mgather[0m[1m([0m[1m)[0m[1m)[0m
+ [36m │ │ │ │ └ [0m[36m[1m<classmethod(<function NodeDiskUsage.gather at 0x109f1d1c0>)>[0m
+ [36m │ │ │ └ [0m[36m[1m<class 'exo.utils.info_gatherer.info_gatherer.NodeDiskUsage'>[0m
+ [36m │ │ └ [0m[36m[1m<function MemoryObjectSendStream.send at 0x107008e00>[0m
+ [36m │ └ [0m[36m[1mSender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profile=SystemPerformanceProfil...[0m
+ [36m └ [0m[36m[1mInfoGatherer(info_sender=Sender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profil...[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/streams/memory.py", line 241, in send
+ await checkpoint()
+ └ <function checkpoint at 0x106e9fe20>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/lowlevel.py", line 28, in checkpoint
+ await get_async_backend().checkpoint()
+ └ <function get_async_backend at 0x106e60f40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2341, in checkpoint
+ await sleep(0)
+ └ <function sleep at 0x1070f27a0>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 708, in sleep
+ await __sleep0()
+ └ <function __sleep0 at 0x1070f2700>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 702, in __sleep0
+ yield
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b0c74d0; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386da0>()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386da0>()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386da0>()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386da0>()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m568[0m, in [35m_monitor_disk_usage[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m5[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b0c74d0; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b313140>
+ └ <contextlib._GeneratorContextManager object at 0x10b2b5b00>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
+[ 12:43:44.2380PM | [33m[1mWARNING [0m] [33m[1mError gathering Thunderbolt data[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b0c7350>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m495[0m, in [35m_monitor_system_profiler_thunderbolt_data[0m
+ [1mdata[0m [35m[1m=[0m [35m[1mawait[0m [1mThunderboltConnectivity[0m[35m[1m.[0m[1mgather[0m[1m([0m[1m)[0m
+ [36m │ └ [0m[36m[1m<classmethod(<function ThunderboltConnectivity.gather at 0x109e4b1a0>)>[0m
+ [36m └ [0m[36m[1m<class 'exo.shared.types.thunderbolt.ThunderboltConnectivity'>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/shared/types/[0m[32m[1mthunderbolt.py[0m", line [33m77[0m, in [35mgather[0m
+ [1mproc[0m [35m[1m=[0m [35m[1mawait[0m [1manyio[0m[35m[1m.[0m[1mrun_process[0m[1m([0m
+ [36m │ └ [0m[36m[1m<function run_process at 0x106ff3b00>[0m
+ [36m └ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 108, in run_process
+ async with create_task_group() as tg:
+ │ └ <anyio._backends._asyncio.TaskGroup object at 0x10b2b7e00>
+ └ <function create_task_group at 0x106ed1760>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 785, in __aexit__
+ raise exc_val
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 119, in run_process
+ await process.wait()
+ │ └ <function Process.wait at 0x10b08f560>
+ └ Process(_process=<Process 13254>, _stdin=None, _stdout=StreamReaderWrapper(_stream=<StreamReader eof exception=ClosedResource...
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 1091, in wait
+ return await self._process.wait()
+ │ │ └ <function Process.wait at 0x1070ffa60>
+ │ └ <Process 13254>
+ └ Process(_process=<Process 13254>, _stdin=None, _stdout=StreamReaderWrapper(_stream=<StreamReader eof exception=ClosedResource...
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/subprocess.py", line 137, in wait
+ return await self._transport._wait()
+ │ │ └ <function BaseSubprocessTransport._wait at 0x107116700>
+ │ └ <_UnixSubprocessTransport closed pid=13254 returncode=0 stdout=<_UnixReadPipeTransport closed fd=55 closed> stderr=<_UnixRead...
+ └ <Process 13254>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_subprocess.py", line 255, in _wait
+ return await waiter
+ └ <Future cancelled>
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b0c7350; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b34df30>()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b34df30>()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b34df30>()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b34df30>()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m490[0m, in [35m_monitor_system_profiler_thunderbolt_data[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m30[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b0c7350; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b311340>
+ └ <contextlib._GeneratorContextManager object at 0x10b2b6c10>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
+[ 12:45:18.1970PM | [1mINFO [0m] [1mManually removing node 3175f9a64495523fe31555701d05903a due to inactivity[0m
+[ 12:45:18.9569PM | [33m[1mWARNING [0m] [33m[1mError gathering network interfaces[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b0c6e10>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m534[0m, in [35m_watch_system_info[0m
+ [1mnics[0m [35m[1m=[0m [35m[1mawait[0m [1mget_network_interfaces[0m[1m([0m[1m)[0m
+ [36m └ [0m[36m[1m<function get_network_interfaces at 0x109f1c5e0>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1msystem_info.py[0m", line [33m101[0m, in [35mget_network_interfaces[0m
+ [1minterface_types[0m [35m[1m=[0m [35m[1mawait[0m [1m_get_interface_types_from_networksetup[0m[1m([0m[1m)[0m
+ [36m └ [0m[36m[1m<function _get_interface_types_from_networksetup at 0x109f1c540>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1msystem_info.py[0m", line [33m65[0m, in [35m_get_interface_types_from_networksetup[0m
+ [1mresult[0m [35m[1m=[0m [35m[1mawait[0m [1mrun_process[0m[1m([0m[1m[[0m[36m"networksetup"[0m[1m,[0m [36m"-listallhardwareports"[0m[1m][0m[1m)[0m
+ [36m └ [0m[36m[1m<function run_process at 0x106ff3b00>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 91, in run_process
+ async with await open_process(
+ └ <function open_process at 0x107009300>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 190, in open_process
+ return await get_async_backend().open_process(
+ └ <function get_async_backend at 0x106e60f40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2588, in open_process
+ process = await asyncio.create_subprocess_exec(
+ │ └ <function create_subprocess_exec at 0x1070fff60>
+ └ <module 'asyncio' from '/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/__...
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/subprocess.py", line 224, in create_subprocess_exec
+ transport, protocol = await loop.subprocess_exec(
+ │ └ <function BaseEventLoop.subprocess_exec at 0x1070f7ba0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 1812, in subprocess_exec
+ transport = await self._make_subprocess_transport(
+ │ └ <function _UnixSelectorEventLoop._make_subprocess_transport at 0x107163420>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/unix_events.py", line 221, in _make_subprocess_transport
+ await waiter
+ └ <Future cancelled>
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b0c6e10; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle Task.task_wakeup()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle Task.task_wakeup()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle Task.task_wakeup()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle Task.task_wakeup()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m533[0m, in [35m_watch_system_info[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m10[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b0c6e10; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b310740>
+ └ <contextlib._GeneratorContextManager object at 0x10b2b78c0>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
+[ 12:45:29.3246PM | [33m[1mWARNING [0m] [33m[1mError gathering misc data[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b0c65d0>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m480[0m, in [35m_monitor_misc[0m
+ [35m[1mawait[0m [1mself[0m[35m[1m.[0m[1minfo_sender[0m[35m[1m.[0m[1msend[0m[1m([0m[35m[1mawait[0m [1mMiscData[0m[35m[1m.[0m[1mgather[0m[1m([0m[1m)[0m[1m)[0m
+ [36m │ │ │ │ └ [0m[36m[1m<classmethod(<function MiscData.gather at 0x109f1d120>)>[0m
+ [36m │ │ │ └ [0m[36m[1m<class 'exo.utils.info_gatherer.info_gatherer.MiscData'>[0m
+ [36m │ │ └ [0m[36m[1m<function MemoryObjectSendStream.send at 0x107008e00>[0m
+ [36m │ └ [0m[36m[1mSender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profile=SystemPerformanceProfil...[0m
+ [36m └ [0m[36m[1mInfoGatherer(info_sender=Sender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profil...[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m320[0m, in [35mgather[0m
+ [35m[1mreturn[0m [1mcls[0m[1m([0m[1mfriendly_name[0m[35m[1m=[0m[35m[1mawait[0m [1mget_friendly_name[0m[1m([0m[1m)[0m[1m)[0m
+ [36m │ └ [0m[36m[1m<function get_friendly_name at 0x109f1c4a0>[0m
+ [36m └ [0m[36m[1m<class 'exo.utils.info_gatherer.info_gatherer.MiscData'>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1msystem_info.py[0m", line [33m52[0m, in [35mget_friendly_name[0m
+ [1mprocess[0m [35m[1m=[0m [35m[1mawait[0m [1mrun_process[0m[1m([0m[1m[[0m[36m"scutil"[0m[1m,[0m [36m"--get"[0m[1m,[0m [36m"ComputerName"[0m[1m][0m[1m)[0m
+ [36m └ [0m[36m[1m<function run_process at 0x106ff3b00>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 91, in run_process
+ async with await open_process(
+ └ <function open_process at 0x107009300>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_subprocesses.py", line 190, in open_process
+ return await get_async_backend().open_process(
+ └ <function get_async_backend at 0x106e60f40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2588, in open_process
+ process = await asyncio.create_subprocess_exec(
+ │ └ <function create_subprocess_exec at 0x1070fff60>
+ └ <module 'asyncio' from '/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/__...
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/subprocess.py", line 224, in create_subprocess_exec
+ transport, protocol = await loop.subprocess_exec(
+ │ └ <function BaseEventLoop.subprocess_exec at 0x1070f7ba0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 1812, in subprocess_exec
+ transport = await self._make_subprocess_transport(
+ │ └ <function _UnixSelectorEventLoop._make_subprocess_transport at 0x107163420>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/unix_events.py", line 221, in _make_subprocess_transport
+ await waiter
+ └ <Future cancelled>
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b0c65d0; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle Task.task_wakeup()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle Task.task_wakeup()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle Task.task_wakeup()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle Task.task_wakeup()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m479[0m, in [35m_monitor_misc[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m10[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b0c65d0; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b312b40>
+ └ <contextlib._GeneratorContextManager object at 0x10b2b7a10>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
+[ 12:45:29.3899PM | [1mINFO [0m] [1mManually removing node 3175f9a64495523fe31555701d05903a due to inactivity[0m
+[ 12:45:29.5532PM | [33m[1mWARNING [0m] [33m[1mError gathering disk usage[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b0c77d0>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m569[0m, in [35m_monitor_disk_usage[0m
+ [35m[1mawait[0m [1mself[0m[35m[1m.[0m[1minfo_sender[0m[35m[1m.[0m[1msend[0m[1m([0m[35m[1mawait[0m [1mNodeDiskUsage[0m[35m[1m.[0m[1mgather[0m[1m([0m[1m)[0m[1m)[0m
+ [36m │ │ │ │ └ [0m[36m[1m<classmethod(<function NodeDiskUsage.gather at 0x109f1d1c0>)>[0m
+ [36m │ │ │ └ [0m[36m[1m<class 'exo.utils.info_gatherer.info_gatherer.NodeDiskUsage'>[0m
+ [36m │ │ └ [0m[36m[1m<function MemoryObjectSendStream.send at 0x107008e00>[0m
+ [36m │ └ [0m[36m[1mSender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profile=SystemPerformanceProfil...[0m
+ [36m └ [0m[36m[1mInfoGatherer(info_sender=Sender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([MacmonMetrics(system_profil...[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/streams/memory.py", line 241, in send
+ await checkpoint()
+ └ <function checkpoint at 0x106e9fe20>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/lowlevel.py", line 28, in checkpoint
+ await get_async_backend().checkpoint()
+ └ <function get_async_backend at 0x106e60f40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2341, in checkpoint
+ await sleep(0)
+ └ <function sleep at 0x1070f27a0>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 708, in sleep
+ await __sleep0()
+ └ <function __sleep0 at 0x1070f2700>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 702, in __sleep0
+ yield
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b0c77d0; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b3e9a50>()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b3e9a50>()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b3e9a50>()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b3e9a50>()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m568[0m, in [35m_monitor_disk_usage[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m5[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b0c77d0; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b313140>
+ └ <contextlib._GeneratorContextManager object at 0x10b2b6350>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
+[ 12:47:02.3058PM | [33m[1mWARNING [0m] [33m[1mError gathering static node info[0m
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 120, in fail_after
+ yield cancel_scope
+ └ <anyio._backends._asyncio.CancelScope object at 0x10b31e510>
+
+ File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m471[0m, in [35m_monitor_static_info[0m
+ [35m[1mawait[0m [1mself[0m[35m[1m.[0m[1minfo_sender[0m[35m[1m.[0m[1msend[0m[1m([0m[35m[1mawait[0m [1mStaticNodeInformation[0m[35m[1m.[0m[1mgather[0m[1m([0m[1m)[0m[1m)[0m
+ [36m │ │ │ │ └ [0m[36m[1m<classmethod(<function StaticNodeInformation.gather at 0x109f1c900>)>[0m
+ [36m │ │ │ └ [0m[36m[1m<class 'exo.utils.info_gatherer.info_gatherer.StaticNodeInformation'>[0m
+ [36m │ │ └ [0m[36m[1m<function MemoryObjectSendStream.send at 0x107008e00>[0m
+ [36m │ └ [0m[36m[1mSender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([]), open_send_channels=1, open_receive_channels=1, w...[0m
+ [36m └ [0m[36m[1mInfoGatherer(info_sender=Sender(_state=MemoryObjectStreamState(max_buffer_size=inf, buffer=deque([]), open_send_channels=1, o...[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/streams/memory.py", line 241, in send
+ await checkpoint()
+ └ <function checkpoint at 0x106e9fe20>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/lowlevel.py", line 28, in checkpoint
+ await get_async_backend().checkpoint()
+ └ <function get_async_backend at 0x106e60f40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2341, in checkpoint
+ await sleep(0)
+ └ <function sleep at 0x1070f27a0>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 708, in sleep
+ await __sleep0()
+ └ <function __sleep0 at 0x1070f2700>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/tasks.py", line 702, in __sleep0
+ yield
+
+[31m[1masyncio.exceptions.CancelledError[0m:[1m Cancelled via cancel scope 10b31e510; reason: deadline exceeded[0m
+
+
+[1mDuring handling of the above exception, another exception occurred:[0m
+
+
+[33m[1mTraceback (most recent call last):[0m
+
+ File "/Users/macstudio3/exo/.venv/bin/exo", line 10, in <module>
+ sys.exit(main())
+ │ │ └ <function main at 0x1020ec180>
+ │ └ <built-in function exit>
+ └ <module 'sys' (built-in)>
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m327[0m, in [35mmain[0m
+ [1mmain_inner[0m[1m([0m[1margs[0m[1m)[0m
+ [36m│ └ [0m[36m[1mArgs(verbosity=0, force_master=False, spawn_api=True, api_port=52415, tb_only=False, no_worker=False, no_downloads=False, off...[0m
+ [36m└ [0m[36m[1m<function main_inner at 0x10b068400>[0m
+
+ File "[32m/Users/macstudio3/exo/src/exo/[0m[32m[1mmain.py[0m", line [33m369[0m, in [35mmain_inner[0m
+ [1manyio[0m[35m[1m.[0m[1mrun[0m[1m([0m[1mnode[0m[35m[1m.[0m[1mrun[0m[1m)[0m
+ [36m│ │ │ └ [0m[36m[1m<function Node.run at 0x10b03c180>[0m
+ [36m│ │ └ [0m[36m[1mNode(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=SessionId(master_node_id='...[0m
+ [36m│ └ [0m[36m[1m<function run at 0x106e602c0>[0m
+ [36m└ [0m[36m[1m<module 'anyio' from '/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/__init__.py'>[0m
+
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_eventloop.py", line 74, in run
+ return async_backend.run(func, args, {}, backend_options)
+ │ │ │ │ └ {}
+ │ │ │ └ ()
+ │ │ └ <bound method Node.run of Node(router=<exo.routing.router.Router object at 0x10af16120>, event_router=EventRouter(session_id=...
+ │ └ <classmethod(<function AsyncIOBackend.run at 0x10b0ec7c0>)>
+ └ <class 'anyio._backends._asyncio.AsyncIOBackend'>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py", line 2325, in run
+ return runner.run(wrapper())
+ │ │ └ <function Node.run at 0x10b07bf60>
+ │ └ <function Runner.run at 0x1070fca40>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/runners.py", line 119, in run
+ return self._loop.run_until_complete(task)
+ │ │ │ └ <Task pending name='exo.main.Node.run' coro=<Node.run() running at /Users/macstudio3/exo/.venv/lib/python3.13/site-packages/a...
+ │ │ └ <function BaseEventLoop.run_until_complete at 0x1070f6340>
+ │ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ └ <asyncio.runners.Runner object at 0x10b2ec410>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 712, in run_until_complete
+ self.run_forever()
+ │ └ <function BaseEventLoop.run_forever at 0x1070f62a0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 683, in run_forever
+ self._run_once()
+ │ └ <function BaseEventLoop._run_once at 0x1070fc0e0>
+ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/base_events.py", line 2060, in _run_once
+ handle._run()
+ │ └ <function Handle._run at 0x1070a0900>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386a70>()>
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/asyncio/events.py", line 89, in _run
+ self._context.run(self._callback, *self._args)
+ │ │ │ │ │ └ <member '_args' of 'Handle' objects>
+ │ │ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386a70>()>
+ │ │ │ └ <member '_callback' of 'Handle' objects>
+ │ │ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386a70>()>
+ │ └ <member '_context' of 'Handle' objects>
+ └ <Handle <_asyncio.TaskStepMethWrapper object at 0x10b386a70>()>
+
+> File "[32m/Users/macstudio3/exo/src/exo/utils/info_gatherer/[0m[32m[1minfo_gatherer.py[0m", line [33m470[0m, in [35m_monitor_static_info[0m
+ [35m[1mwith[0m [1mfail_after[0m[1m([0m[34m[1m30[0m[1m)[0m[1m:[0m
+ [36m └ [0m[36m[1m<function fail_after at 0x106ed1580>[0m
+
+ File "/Users/macstudio3/.local/share/uv/python/cpython-3.13.14-macos-aarch64-none/lib/python3.13/contextlib.py", line 162, in __exit__
+ self.gen.throw(value)
+ │ │ │ └ CancelledError('Cancelled via cancel scope 10b31e510; reason: deadline exceeded')
+ │ │ └ <method 'throw' of 'generator' objects>
+ │ └ <generator object fail_after at 0x10b312d40>
+ └ <contextlib._GeneratorContextManager object at 0x10b46fc40>
+ File "/Users/macstudio3/exo/.venv/lib/python3.13/site-packages/anyio/_core/_tasks.py", line 123, in fail_after
+ raise TimeoutError
+
+[31m[1mTimeoutError[0m
← f324fac auto-save: 2026-07-10T10:37:42 (1 files) — exo.log
·
back to Exo Keepalive
·
auto-save: 2026-07-24T07:26:03 (1 files) — exo.log 3a1da18 →