← back to Exo
downloads: add read_only flag to DownloadCompleted for EXO_MODELS_PATH
ab9273e7232a156f86f62e7f67354cff75843ea1 · 2026-02-20 19:30:13 +0000 · Jake Hillion
Models in EXO_MODELS_PATH are pre-downloaded into read-only directories
and must not be deleted. The DownloadCoordinator had no awareness of
these paths, so they never appeared as completed downloads in cluster
state, and the bench harness could attempt to delete them when freeing
disk space.
Added a `read_only: bool` field to `DownloadCompleted` (default False).
The DownloadCoordinator now checks `resolve_model_in_path` in
`_start_download`, proactively scans EXO_MODELS_PATH in
`_emit_existing_download_progress` to emit DownloadCompleted events for
all pre-downloaded models (overriding DownloadPending from the regular
scan), and refuses deletion of read-only models. The bench harness
filters out read-only models from deletion candidates.
Test plan:
- Ran with EXO_MODELS_PATH. Available models now show as downloaded in
the UI. There isn't good UI for the fact they can't be deleted, but it
should work with exo_bench.
Files touched
M bench/harness.pyM src/exo/download/coordinator.pyM src/exo/shared/types/worker/downloads.pyM src/exo/worker/main.py
Diff
commit ab9273e7232a156f86f62e7f67354cff75843ea1
Author: Jake Hillion <jake@hillion.co.uk>
Date: Fri Feb 20 19:30:13 2026 +0000
downloads: add read_only flag to DownloadCompleted for EXO_MODELS_PATH
Models in EXO_MODELS_PATH are pre-downloaded into read-only directories
and must not be deleted. The DownloadCoordinator had no awareness of
these paths, so they never appeared as completed downloads in cluster
state, and the bench harness could attempt to delete them when freeing
disk space.
Added a `read_only: bool` field to `DownloadCompleted` (default False).
The DownloadCoordinator now checks `resolve_model_in_path` in
`_start_download`, proactively scans EXO_MODELS_PATH in
`_emit_existing_download_progress` to emit DownloadCompleted events for
all pre-downloaded models (overriding DownloadPending from the regular
scan), and refuses deletion of read-only models. The bench harness
filters out read-only models from deletion candidates.
Test plan:
- Ran with EXO_MODELS_PATH. Available models now show as downloaded in
the UI. There isn't good UI for the fact they can't be deleted, but it
should work with exo_bench.
---
bench/harness.py | 3 +-
src/exo/download/coordinator.py | 68 ++++++++++++++++++++++++++++++--
src/exo/shared/types/worker/downloads.py | 1 +
src/exo/worker/main.py | 1 +
4 files changed, 69 insertions(+), 4 deletions(-)
diff --git a/bench/harness.py b/bench/harness.py
index 263c8e91..95f133f7 100644
--- a/bench/harness.py
+++ b/bench/harness.py
@@ -365,7 +365,7 @@ def run_planning_phase(
f"have {avail // (1024**3)}GB. Use --danger-delete-downloads to free space."
)
- # Delete from smallest to largest
+ # Delete from smallest to largest (skip read-only models from EXO_MODELS_PATH)
completed = [
(
unwrap_instance(p["DownloadCompleted"]["shardMetadata"])["modelCard"][
@@ -375,6 +375,7 @@ def run_planning_phase(
)
for p in node_downloads
if "DownloadCompleted" in p
+ and not p["DownloadCompleted"].get("readOnly", False)
]
for del_model, size in sorted(completed, key=lambda x: x[1]):
logger.info(f"Deleting {del_model} from {node_id} ({size // (1024**2)}MB)")
diff --git a/src/exo/download/coordinator.py b/src/exo/download/coordinator.py
index dd6337cc..5b6026be 100644
--- a/src/exo/download/coordinator.py
+++ b/src/exo/download/coordinator.py
@@ -12,10 +12,11 @@ from exo.download.download_utils import (
RepoDownloadProgress,
delete_model,
map_repo_download_progress_to_download_progress_data,
+ resolve_model_in_path,
)
from exo.download.shard_downloader import ShardDownloader
-from exo.shared.constants import EXO_MODELS_DIR
-from exo.shared.models.model_cards import ModelId
+from exo.shared.constants import EXO_MODELS_DIR, EXO_MODELS_PATH
+from exo.shared.models.model_cards import ModelId, get_model_cards
from exo.shared.types.commands import (
CancelDownload,
DeleteDownload,
@@ -38,7 +39,7 @@ from exo.shared.types.worker.downloads import (
DownloadPending,
DownloadProgress,
)
-from exo.shared.types.worker.shards import ShardMetadata
+from exo.shared.types.worker.shards import PipelineShardMetadata, ShardMetadata
from exo.utils.channels import Receiver, Sender, channel
@@ -214,6 +215,25 @@ class DownloadCoordinator:
)
return
+ # Check EXO_MODELS_PATH for pre-downloaded models
+ found_path = resolve_model_in_path(model_id)
+ if found_path is not None:
+ logger.info(
+ f"DownloadCoordinator: Model {model_id} found in EXO_MODELS_PATH at {found_path}"
+ )
+ completed = DownloadCompleted(
+ shard_metadata=shard,
+ node_id=self.node_id,
+ total=shard.model_card.storage_size,
+ model_directory=str(found_path),
+ read_only=True,
+ )
+ self.download_status[model_id] = completed
+ await self.event_sender.send(
+ NodeDownloadProgress(download_progress=completed)
+ )
+ return
+
# Emit pending status
progress = DownloadPending(
shard_metadata=shard,
@@ -298,6 +318,15 @@ class DownloadCoordinator:
self.active_downloads[model_id] = task
async def _delete_download(self, model_id: ModelId) -> None:
+ # Protect read-only models (from EXO_MODELS_PATH) from deletion
+ if model_id in self.download_status:
+ current = self.download_status[model_id]
+ if isinstance(current, DownloadCompleted) and current.read_only:
+ logger.warning(
+ f"Refusing to delete read-only model {model_id} (from EXO_MODELS_PATH)"
+ )
+ return
+
# Cancel if active
if model_id in self.active_downloads:
logger.info(f"Cancelling active download for {model_id} before deletion")
@@ -397,6 +426,39 @@ class DownloadCoordinator:
await self.event_sender.send(
NodeDownloadProgress(download_progress=status)
)
+ # Scan EXO_MODELS_PATH for pre-downloaded models
+ if EXO_MODELS_PATH is not None:
+ for card in await get_model_cards():
+ mid = card.model_id
+ if mid in self.active_downloads:
+ continue
+ if isinstance(
+ self.download_status.get(mid),
+ (DownloadCompleted, DownloadOngoing, DownloadFailed),
+ ):
+ continue
+ found = resolve_model_in_path(mid)
+ if found is not None:
+ path_shard = PipelineShardMetadata(
+ model_card=card,
+ device_rank=0,
+ world_size=1,
+ start_layer=0,
+ end_layer=card.n_layers,
+ n_layers=card.n_layers,
+ )
+ path_completed: DownloadProgress = DownloadCompleted(
+ node_id=self.node_id,
+ shard_metadata=path_shard,
+ total=card.storage_size,
+ model_directory=str(found),
+ read_only=True,
+ )
+ self.download_status[mid] = path_completed
+ await self.event_sender.send(
+ NodeDownloadProgress(download_progress=path_completed)
+ )
+
logger.debug(
"DownloadCoordinator: Done emitting existing download progress."
)
diff --git a/src/exo/shared/types/worker/downloads.py b/src/exo/shared/types/worker/downloads.py
index d3203a25..29540fe1 100644
--- a/src/exo/shared/types/worker/downloads.py
+++ b/src/exo/shared/types/worker/downloads.py
@@ -36,6 +36,7 @@ class DownloadPending(BaseDownloadProgress):
class DownloadCompleted(BaseDownloadProgress):
total: Memory
+ read_only: bool = False
class DownloadFailed(BaseDownloadProgress):
diff --git a/src/exo/worker/main.py b/src/exo/worker/main.py
index b80716c5..3d3d30a1 100644
--- a/src/exo/worker/main.py
+++ b/src/exo/worker/main.py
@@ -227,6 +227,7 @@ class Worker:
shard_metadata=shard,
model_directory=str(found_path),
total=shard.model_card.storage_size,
+ read_only=True,
)
)
)
← 71e48c0f model-cards: add missing metadata for Qwen3 Coder Next varia
·
back to Exo
·
fix: change RDMA AVAILABLE to RDMA NOT ENABLED warning (#158 1780e4ad →