← back to Exo
dashboard: show available disk space on downloads page
a8acb3cafb7c7ff1c06626d45b0842771cd88a79 · 2026-02-11 12:58:10 +0000 · Jake Hillion
The downloads page previously only showed the approximate space used by
downloaded models (summed from completed download sizes), but did not
show how much disk space was actually available. This made it difficult
to know if a download would succeed before pressing the button.
Added disk space tracking to the InfoGatherer that polls the models
directory partition every 30 seconds. The DiskUsage type captures total
and available space, which flows through the event system to State and
is exposed via the /state API. The dashboard now displays "X on disk /
Y available" for each node in the downloads view.
Test plan:
- CI
Files touched
M dashboard/src/lib/stores/app.svelte.tsM dashboard/src/routes/downloads/+page.svelteM src/exo/shared/apply.pyM src/exo/shared/types/profiling.pyM src/exo/shared/types/state.pyM src/exo/utils/info_gatherer/info_gatherer.py
Diff
commit a8acb3cafb7c7ff1c06626d45b0842771cd88a79
Author: Jake Hillion <jake@hillion.co.uk>
Date: Wed Feb 11 12:58:10 2026 +0000
dashboard: show available disk space on downloads page
The downloads page previously only showed the approximate space used by
downloaded models (summed from completed download sizes), but did not
show how much disk space was actually available. This made it difficult
to know if a download would succeed before pressing the button.
Added disk space tracking to the InfoGatherer that polls the models
directory partition every 30 seconds. The DiskUsage type captures total
and available space, which flows through the event system to State and
is exposed via the /state API. The dashboard now displays "X on disk /
Y available" for each node in the downloads view.
Test plan:
- CI
---
dashboard/src/lib/stores/app.svelte.ts | 10 +++++++++
dashboard/src/routes/downloads/+page.svelte | 26 ++++++++++++++++------
src/exo/shared/apply.py | 7 ++++++
src/exo/shared/types/profiling.py | 18 +++++++++++++++
src/exo/shared/types/state.py | 2 ++
src/exo/utils/info_gatherer/info_gatherer.py | 33 ++++++++++++++++++++++++++--
6 files changed, 87 insertions(+), 9 deletions(-)
diff --git a/dashboard/src/lib/stores/app.svelte.ts b/dashboard/src/lib/stores/app.svelte.ts
index a444ada2..9034a8af 100644
--- a/dashboard/src/lib/stores/app.svelte.ts
+++ b/dashboard/src/lib/stores/app.svelte.ts
@@ -524,6 +524,12 @@ class AppStore {
instances = $state<Record<string, unknown>>({});
runners = $state<Record<string, unknown>>({});
downloads = $state<Record<string, unknown[]>>({});
+ nodeDisk = $state<
+ Record<
+ string,
+ { total: { inBytes: number }; available: { inBytes: number } }
+ >
+ >({});
placementPreviews = $state<PlacementPreview[]>([]);
selectedPreviewModelId = $state<string | null>(null);
isLoadingPreviews = $state(false);
@@ -1254,6 +1260,9 @@ class AppStore {
if (data.downloads) {
this.downloads = data.downloads;
}
+ if (data.nodeDisk) {
+ this.nodeDisk = data.nodeDisk;
+ }
// Node identities (for OS version mismatch detection)
this.nodeIdentities = data.nodeIdentities ?? {};
// Thunderbolt identifiers per node
@@ -3012,6 +3021,7 @@ export const topologyData = () => appStore.topologyData;
export const instances = () => appStore.instances;
export const runners = () => appStore.runners;
export const downloads = () => appStore.downloads;
+export const nodeDisk = () => appStore.nodeDisk;
export const placementPreviews = () => appStore.placementPreviews;
export const selectedPreviewModelId = () => appStore.selectedPreviewModelId;
export const isLoadingPreviews = () => appStore.isLoadingPreviews;
diff --git a/dashboard/src/routes/downloads/+page.svelte b/dashboard/src/routes/downloads/+page.svelte
index 72fe149c..94e10612 100644
--- a/dashboard/src/routes/downloads/+page.svelte
+++ b/dashboard/src/routes/downloads/+page.svelte
@@ -3,6 +3,7 @@
import {
topologyData,
downloads,
+ nodeDisk,
type DownloadProgress,
refreshState,
lastUpdate as lastUpdateStore,
@@ -37,10 +38,13 @@
nodeId: string;
nodeName: string;
models: ModelEntry[];
+ diskAvailable?: number;
+ diskTotal?: number;
};
const data = $derived(topologyData());
const downloadsData = $derived(downloads());
+ const nodeDiskData = $derived(nodeDisk());
function getNodeLabel(nodeId: string): string {
const node = data?.nodes?.[nodeId];
@@ -327,10 +331,17 @@
];
}
+ // Get disk info for this node
+ const diskInfo = nodeDiskData?.[nodeId];
+ const diskAvailable = diskInfo?.available?.inBytes;
+ const diskTotal = diskInfo?.total?.inBytes;
+
built.push({
nodeId,
nodeName: getNodeLabel(nodeId),
models,
+ diskAvailable,
+ diskTotal,
});
}
@@ -417,6 +428,14 @@
<div class="text-xs text-exo-light-gray font-mono truncate">
{node.nodeId}
</div>
+ <div class="text-xs text-exo-light-gray font-mono mt-1">
+ {formatBytes(
+ node.models
+ .filter((m) => m.status === "completed")
+ .reduce((sum, m) => sum + m.totalBytes, 0),
+ )} models{#if node.diskAvailable != null}
+ - {formatBytes(node.diskAvailable)} free{/if}
+ </div>
</div>
<div
class="text-xs font-mono uppercase tracking-wider whitespace-nowrap shrink-0 text-right"
@@ -429,13 +448,6 @@
/ {node.models.length} models</span
>
</div>
- <div class="text-exo-light-gray normal-case tracking-normal">
- {formatBytes(
- node.models
- .filter((m) => m.status === "completed")
- .reduce((sum, m) => sum + m.totalBytes, 0),
- )} on disk
- </div>
</div>
</div>
diff --git a/src/exo/shared/apply.py b/src/exo/shared/apply.py
index 541a53d0..fd0df265 100644
--- a/src/exo/shared/apply.py
+++ b/src/exo/shared/apply.py
@@ -48,6 +48,7 @@ from exo.utils.info_gatherer.info_gatherer import (
MemoryUsage,
MiscData,
NodeConfig,
+ NodeDiskUsage,
NodeNetworkInterfaces,
RdmaCtlStatus,
StaticNodeInformation,
@@ -225,6 +226,9 @@ def apply_node_timed_out(event: NodeTimedOut, state: State) -> State:
node_memory = {
key: value for key, value in state.node_memory.items() if key != event.node_id
}
+ node_disk = {
+ key: value for key, value in state.node_disk.items() if key != event.node_id
+ }
node_system = {
key: value for key, value in state.node_system.items() if key != event.node_id
}
@@ -261,6 +265,7 @@ def apply_node_timed_out(event: NodeTimedOut, state: State) -> State:
"last_seen": last_seen,
"node_identities": node_identities,
"node_memory": node_memory,
+ "node_disk": node_disk,
"node_system": node_system,
"node_network": node_network,
"node_thunderbolt": node_thunderbolt,
@@ -294,6 +299,8 @@ def apply_node_gathered_info(event: NodeGatheredInfo, state: State) -> State:
update["node_memory"] = {**state.node_memory, event.node_id: info.memory}
case MemoryUsage():
update["node_memory"] = {**state.node_memory, event.node_id: info}
+ case NodeDiskUsage():
+ update["node_disk"] = {**state.node_disk, event.node_id: info.disk_usage}
case NodeConfig():
pass
case MiscData():
diff --git a/src/exo/shared/types/profiling.py b/src/exo/shared/types/profiling.py
index b5e4040e..ad1d48c0 100644
--- a/src/exo/shared/types/profiling.py
+++ b/src/exo/shared/types/profiling.py
@@ -1,4 +1,6 @@
+import shutil
from collections.abc import Sequence
+from pathlib import Path
from typing import Literal, Self
import psutil
@@ -38,6 +40,22 @@ class MemoryUsage(CamelCaseModel):
)
+class DiskUsage(CamelCaseModel):
+ """Disk space usage for the models directory."""
+
+ total: Memory
+ available: Memory
+
+ @classmethod
+ def from_path(cls, path: Path) -> Self:
+ """Get disk usage stats for the partition containing path."""
+ total, _used, free = shutil.disk_usage(path)
+ return cls(
+ total=Memory.from_bytes(total),
+ available=Memory.from_bytes(free),
+ )
+
+
class SystemPerformanceProfile(CamelCaseModel):
# TODO: flops_fp16: float
diff --git a/src/exo/shared/types/state.py b/src/exo/shared/types/state.py
index 38922e8e..7350cfb0 100644
--- a/src/exo/shared/types/state.py
+++ b/src/exo/shared/types/state.py
@@ -8,6 +8,7 @@ from pydantic.alias_generators import to_camel
from exo.shared.topology import Topology, TopologySnapshot
from exo.shared.types.common import NodeId
from exo.shared.types.profiling import (
+ DiskUsage,
MemoryUsage,
NodeIdentity,
NodeNetworkInfo,
@@ -50,6 +51,7 @@ class State(CamelCaseModel):
# Granular node state mappings (update independently at different frequencies)
node_identities: Mapping[NodeId, NodeIdentity] = {}
node_memory: Mapping[NodeId, MemoryUsage] = {}
+ node_disk: Mapping[NodeId, DiskUsage] = {}
node_system: Mapping[NodeId, SystemPerformanceProfile] = {}
node_network: Mapping[NodeId, NodeNetworkInfo] = {}
node_thunderbolt: Mapping[NodeId, NodeThunderboltInfo] = {}
diff --git a/src/exo/utils/info_gatherer/info_gatherer.py b/src/exo/utils/info_gatherer/info_gatherer.py
index 68dab6f9..17f07aa1 100644
--- a/src/exo/utils/info_gatherer/info_gatherer.py
+++ b/src/exo/utils/info_gatherer/info_gatherer.py
@@ -8,16 +8,17 @@ from subprocess import CalledProcessError
from typing import Self, cast
import anyio
-from anyio import create_task_group, fail_after, open_process
+from anyio import create_task_group, fail_after, open_process, to_thread
from anyio.abc import TaskGroup
from anyio.streams.buffered import BufferedByteReceiveStream
from anyio.streams.text import TextReceiveStream
from loguru import logger
from pydantic import ValidationError
-from exo.shared.constants import EXO_CONFIG_FILE
+from exo.shared.constants import EXO_CONFIG_FILE, EXO_MODELS_DIR
from exo.shared.types.memory import Memory
from exo.shared.types.profiling import (
+ DiskUsage,
MemoryUsage,
NetworkInterfaceInfo,
ThunderboltBridgeStatus,
@@ -319,6 +320,20 @@ class MiscData(TaggedModel):
return cls(friendly_name=await get_friendly_name())
+class NodeDiskUsage(TaggedModel):
+ """Disk space information for the models directory."""
+
+ disk_usage: DiskUsage
+
+ @classmethod
+ async def gather(cls) -> Self:
+ return cls(
+ disk_usage=await to_thread.run_sync(
+ lambda: DiskUsage.from_path(EXO_MODELS_DIR)
+ )
+ )
+
+
async def _gather_iface_map() -> dict[str, str] | None:
proc = await anyio.run_process(
["networksetup", "-listallhardwareports"], check=False
@@ -350,6 +365,7 @@ GatheredInfo = (
| NodeConfig
| MiscData
| StaticNodeInformation
+ | NodeDiskUsage
)
@@ -364,6 +380,7 @@ class InfoGatherer:
thunderbolt_bridge_poll_interval: float | None = 10 if IS_DARWIN else None
static_info_poll_interval: float | None = 60
rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None
+ disk_poll_interval: float | None = 30
_tg: TaskGroup = field(init=False, default_factory=create_task_group)
async def run(self):
@@ -378,6 +395,7 @@ class InfoGatherer:
tg.start_soon(self._monitor_memory_usage)
tg.start_soon(self._monitor_misc)
tg.start_soon(self._monitor_static_info)
+ tg.start_soon(self._monitor_disk_usage)
nc = await NodeConfig.gather()
if nc is not None:
@@ -481,6 +499,17 @@ class InfoGatherer:
logger.warning(f"Error gathering RDMA ctl status: {e}")
await anyio.sleep(self.rdma_ctl_poll_interval)
+ async def _monitor_disk_usage(self):
+ if self.disk_poll_interval is None:
+ return
+ while True:
+ try:
+ with fail_after(5):
+ await self.info_sender.send(await NodeDiskUsage.gather())
+ except Exception as e:
+ logger.warning(f"Error gathering disk usage: {e}")
+ await anyio.sleep(self.disk_poll_interval)
+
async def _monitor_macmon(self, macmon_path: str):
if self.macmon_interval is None:
return
← a0721dbe feat: warn when cluster nodes have mismatched macOS versions
·
back to Exo
·
Make info gatherer monitors resilient with retry loops and t 98773437 →