← back to Exo
Ciaran/dashboard download bug (#1755)
63f57fc19380058c1adfe24bd12716014695ff1f · 2026-03-19 14:47:45 +0000 · ciaranbor
## Motivation
Download progress in the dashboard was broken: mainly treating all
download statuses as ongoing
## Changes
- Backend (apply.py): Deduplicate download progress events by model_id
instead of full shard_metadata, preventing duplicate entries per node
- Dashboard (+page.svelte): Extract shared collectDownloadStatus()
helper that both getModelDownloadStatus and getInstanceDownloadStatus
use, eliminating ~100 lines of duplicated logic. Adds proper handling
for
DownloadCompleted/DownloadFailed events, uses a Map to deduplicate
per-node entries, and introduces a typed NodeDownloadStatus with
explicit status states (downloading/completed/partial/pending)
- ModelCard: Replace single aggregate progress bar with per-node
download bars, each color-coded by status. Instance preview now scopes
download status to participating nodes only
## Why It Works
- Deduplicating by model_id in apply.py ensures each node has exactly
one download entry per model
- The perNodeMap in the frontend keeps only the latest event per node,
preventing duplicate bars
- Handling DownloadCompleted allows the UI to show finished downloads
instead of dropping them
- Scoping instance previews to assigned nodes avoids showing irrelevant
download progress
Files touched
M dashboard/src/lib/components/ModelCard.svelteM dashboard/src/routes/+page.svelteM src/exo/shared/apply.py
Diff
commit 63f57fc19380058c1adfe24bd12716014695ff1f
Author: ciaranbor <81697641+ciaranbor@users.noreply.github.com>
Date: Thu Mar 19 14:47:45 2026 +0000
Ciaran/dashboard download bug (#1755)
## Motivation
Download progress in the dashboard was broken: mainly treating all
download statuses as ongoing
## Changes
- Backend (apply.py): Deduplicate download progress events by model_id
instead of full shard_metadata, preventing duplicate entries per node
- Dashboard (+page.svelte): Extract shared collectDownloadStatus()
helper that both getModelDownloadStatus and getInstanceDownloadStatus
use, eliminating ~100 lines of duplicated logic. Adds proper handling
for
DownloadCompleted/DownloadFailed events, uses a Map to deduplicate
per-node entries, and introduces a typed NodeDownloadStatus with
explicit status states (downloading/completed/partial/pending)
- ModelCard: Replace single aggregate progress bar with per-node
download bars, each color-coded by status. Instance preview now scopes
download status to participating nodes only
## Why It Works
- Deduplicating by model_id in apply.py ensures each node has exactly
one download entry per model
- The perNodeMap in the frontend keeps only the latest event per node,
preventing duplicate bars
- Handling DownloadCompleted allows the UI to show finished downloads
instead of dropping them
- Scoping instance previews to assigned nodes avoids showing irrelevant
download progress
---
dashboard/src/lib/components/ModelCard.svelte | 82 +++--
dashboard/src/routes/+page.svelte | 420 ++++++++++++--------------
src/exo/shared/apply.py | 8 +-
3 files changed, 244 insertions(+), 266 deletions(-)
diff --git a/dashboard/src/lib/components/ModelCard.svelte b/dashboard/src/lib/components/ModelCard.svelte
index b432b7a8..84287158 100644
--- a/dashboard/src/lib/components/ModelCard.svelte
+++ b/dashboard/src/lib/components/ModelCard.svelte
@@ -16,7 +16,9 @@
perNode?: Array<{
nodeId: string;
nodeName: string;
- progress: DownloadProgress;
+ status: "completed" | "partial" | "pending" | "downloading";
+ percentage: number;
+ progress: DownloadProgress | null;
}>;
} | null;
nodes?: Record<string, NodeInfo>;
@@ -145,10 +147,7 @@
return `${s}s`;
}
- const isDownloading = $derived(downloadStatus?.isDownloading ?? false);
- const progress = $derived(downloadStatus?.progress);
- const percentage = $derived(progress?.percentage ?? 0);
- let expandedNodes = $state<Set<string>>(new Set());
+ const perNode = $derived(downloadStatus?.perNode ?? []);
function toggleNodeDetails(nodeId: string): void {
const next = new Set(expandedNodes);
@@ -587,23 +586,49 @@
</span>
</div>
- <!-- Download Status -->
- {#if isDownloading && progress}
+ <!-- Download Status (per-node) -->
+ {#if perNode.length > 0}
<div class="mb-2 space-y-1">
- <div class="flex items-center justify-between text-xs font-mono">
- <span class="text-blue-400 tracking-wider uppercase">Downloading</span
- >
- <span class="text-white/60"
- >{percentage.toFixed(1)}% · {formatSpeed(progress.speed)}
- · {formatEta(progress.etaMs)}</span
- >
- </div>
- <div class="h-1 bg-exo-medium-gray/30 rounded overflow-hidden">
- <div
- class="h-full bg-blue-500/70 transition-all duration-300"
- style="width: {percentage}%"
- ></div>
+ <div
+ class="text-[10px] font-mono text-white/20 tracking-widest uppercase"
+ >
+ Download progress
</div>
+ {#each perNode as node}
+ <div class="flex items-center gap-2 text-xs font-mono">
+ <span class="text-white/40 w-20 truncate" title={node.nodeId}
+ >{node.nodeName}</span
+ >
+ <div
+ class="flex-1 h-1 bg-exo-medium-gray/30 rounded overflow-hidden"
+ >
+ <div
+ class="h-full transition-all duration-300 {node.status ===
+ 'downloading'
+ ? 'bg-blue-500/70'
+ : node.status === 'completed'
+ ? 'bg-exo-yellow/40'
+ : 'bg-white/20'}"
+ style="width: {node.percentage}%"
+ ></div>
+ </div>
+ <span
+ class="text-right {node.status === 'completed'
+ ? 'text-exo-yellow/60'
+ : node.status === 'downloading'
+ ? 'text-blue-400/60'
+ : 'text-white/30'}"
+ >
+ {#if node.status === "downloading" && node.progress}
+ {Math.round(node.percentage)}% {formatSpeed(
+ node.progress.speed,
+ )}
+ {:else}
+ {node.percentage > 0 ? `${Math.round(node.percentage)}%` : "0%"}
+ {/if}
+ </span>
+ </div>
+ {/each}
</div>
{/if}
@@ -662,15 +687,7 @@
{@const allConnections =
isDebugMode && usedNodes.length > 1
? (() => {
- const conns: Array<{
- ip: string;
- iface: string | null;
- from: string;
- to: string;
- midX: number;
- midY: number;
- arrow: string;
- }> = [];
+ const conns: Array = [];
for (let i = 0; i < usedNodes.length; i++) {
for (let j = i + 1; j < usedNodes.length; j++) {
const n1 = usedNodes[i];
@@ -682,7 +699,12 @@
const toPos = nodePositions[c.to];
const arrow =
fromPos && toPos ? getArrow(fromPos, toPos) : "→";
- conns.push({ ...c, midX, midY, arrow });
+ conns.push({
+ ...c,
+ midX,
+ midY,
+ arrow,
+ });
}
}
}
diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte
index 262a8338..cc403fef 100644
--- a/dashboard/src/routes/+page.svelte
+++ b/dashboard/src/routes/+page.svelte
@@ -1535,34 +1535,44 @@
}
// Helper to get download status for a model (checks all downloads for matching model ID)
- function getModelDownloadStatus(modelId: string): {
+ type NodeDownloadStatus = {
+ nodeId: string;
+ nodeName: string;
+ status: "completed" | "partial" | "pending" | "downloading";
+ percentage: number;
+ progress: DownloadProgress | null;
+ };
+
+ // Shared helper: collect per-node download status for a model across a set of nodes.
+ // Handles deduplication, entry parsing, and aggregation in one place.
+ function collectDownloadStatus(
+ modelId: string,
+ nodeIds?: string[],
+ ): {
isDownloading: boolean;
progress: DownloadProgress | null;
- perNode: Array<{
- nodeId: string;
- nodeName: string;
- progress: DownloadProgress;
- }>;
+ perNode: NodeDownloadStatus[];
+ failedError: string | null;
} {
+ const empty = {
+ isDownloading: false,
+ progress: null,
+ perNode: [] as NodeDownloadStatus[],
+ failedError: null,
+ };
+
if (!downloadsData || Object.keys(downloadsData).length === 0) {
- return { isDownloading: false, progress: null, perNode: [] };
+ return empty;
}
- let totalBytes = 0;
- let downloadedBytes = 0;
- let totalSpeed = 0;
- let completedFiles = 0;
- let totalFiles = 0;
- let isDownloading = false;
- const allFiles: DownloadProgress["files"] = [];
- const perNode: Array<{
- nodeId: string;
- nodeName: string;
- progress: DownloadProgress;
- }> = [];
+ // Deduplicate by nodeId — a node can have multiple entries for the same model
+ // (e.g. PipelineShardMetadata + TensorShardMetadata). Keep the last entry,
+ // which is the most recently applied event.
+ const perNodeMap = new Map<string, NodeDownloadStatus>();
- // Check all nodes for downloads matching this model
+ const nodeIdSet = nodeIds ? new Set(nodeIds) : null;
for (const [nodeId, nodeDownloads] of Object.entries(downloadsData)) {
+ if (nodeIdSet && !nodeIdSet.has(nodeId)) continue;
if (!Array.isArray(nodeDownloads)) continue;
for (const downloadWrapped of nodeDownloads) {
@@ -1575,29 +1585,45 @@
const downloadPayload = (downloadWrapped as Record<string, unknown>)[
downloadKind
] as Record<string, unknown>;
+ if (!downloadPayload) continue;
+
+ const downloadModelId = extractModelIdFromDownload(downloadPayload);
+ if (!downloadModelId || downloadModelId !== modelId) continue;
+
+ // DownloadFailed — return with any data collected so far
+ if (downloadKind === "DownloadFailed") {
+ return {
+ isDownloading: false,
+ progress: null,
+ perNode: Array.from(perNodeMap.values()),
+ failedError:
+ (downloadPayload.errorMessage as string) ||
+ (downloadPayload.error_message as string) ||
+ "Download failed",
+ };
+ }
if (
downloadKind !== "DownloadOngoing" &&
- downloadKind !== "DownloadPending"
+ downloadKind !== "DownloadPending" &&
+ downloadKind !== "DownloadCompleted"
)
continue;
- if (!downloadPayload) continue;
- const downloadModelId = extractModelIdFromDownload(downloadPayload);
+ const nodeName =
+ data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
- // Match if the model ID contains or equals the requested model
- // (handles cases like "mlx-community/Meta-Llama..." matching)
- if (
- !downloadModelId ||
- !downloadModelId.includes(modelId.split("/").pop() || modelId)
- ) {
- // Try exact match or partial match
- if (downloadModelId !== modelId) continue;
+ if (downloadKind === "DownloadCompleted") {
+ perNodeMap.set(nodeId, {
+ nodeId,
+ nodeName,
+ status: "completed",
+ percentage: 100,
+ progress: null,
+ });
+ continue;
}
- // For DownloadPending with partial bytes (paused/resumed downloads),
- // synthesize a progress object from the top-level downloaded/total fields
- let progress: DownloadProgress | null;
if (downloadKind === "DownloadPending") {
const pendingDownloaded = getBytes(
downloadPayload.downloaded ??
@@ -1610,44 +1636,67 @@
downloadPayload.totalBytes,
);
if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
- isDownloading = true;
- progress = {
- totalBytes: pendingTotal,
- downloadedBytes: pendingDownloaded,
- speed: 0,
- etaMs: 0,
- percentage:
- pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
- completedFiles: 0,
- totalFiles: 0,
- files: [],
- };
- } else {
- isDownloading = true;
- progress = parseDownloadProgress(downloadPayload);
+ const pct =
+ pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0;
+ perNodeMap.set(nodeId, {
+ nodeId,
+ nodeName,
+ status: pendingDownloaded > 0 ? "partial" : "pending",
+ percentage: pct,
+ progress: null,
+ });
+ continue;
}
- if (progress) {
- // Sum all values across nodes - each node downloads independently
- totalBytes += progress.totalBytes;
- downloadedBytes += progress.downloadedBytes;
- totalSpeed += progress.speed;
- completedFiles += progress.completedFiles;
- totalFiles += progress.totalFiles;
- allFiles.push(...progress.files);
-
- const nodeName =
- data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
- perNode.push({ nodeId, nodeName, progress });
- }
+ // DownloadOngoing
+ const progress = parseDownloadProgress(downloadPayload);
+ if (
+ !progress ||
+ (progress.downloadedBytes <= 0 && progress.totalBytes <= 0)
+ )
+ continue;
+
+ perNodeMap.set(nodeId, {
+ nodeId,
+ nodeName,
+ status: "downloading",
+ percentage: progress.percentage,
+ progress,
+ });
+ }
+ }
+
+ // Aggregate from deduplicated per-node entries
+ const perNode = Array.from(perNodeMap.values());
+ let totalBytes = 0;
+ let downloadedBytes = 0;
+ let totalSpeed = 0;
+ let completedFiles = 0;
+ let totalFiles = 0;
+ let isDownloading = false;
+ const allFiles: DownloadProgress["files"] = [];
+
+ for (const node of perNode) {
+ if (node.status === "downloading" && node.progress) {
+ isDownloading = true;
+ totalBytes += node.progress.totalBytes;
+ downloadedBytes += node.progress.downloadedBytes;
+ totalSpeed += node.progress.speed;
+ completedFiles += node.progress.completedFiles;
+ totalFiles += node.progress.totalFiles;
+ allFiles.push(...node.progress.files);
}
}
if (!isDownloading) {
- return { isDownloading: false, progress: null, perNode: [] };
+ return {
+ isDownloading: false,
+ progress: null,
+ perNode,
+ failedError: null,
+ };
}
- // ETA = total remaining bytes / total speed across all nodes
const remainingBytes = totalBytes - downloadedBytes;
const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
@@ -1664,9 +1713,21 @@
files: allFiles,
},
perNode,
+ failedError: null,
};
}
+ function getModelDownloadStatus(
+ modelId: string,
+ nodeIds?: string[],
+ ): {
+ isDownloading: boolean;
+ progress: DownloadProgress | null;
+ perNode: NodeDownloadStatus[];
+ } {
+ return collectDownloadStatus(modelId, nodeIds);
+ }
+
// Helper to get download status for an instance
function getInstanceDownloadStatus(
instanceId: string,
@@ -1677,26 +1738,9 @@
errorMessage: string | null;
progress: DownloadProgress | null;
statusText: string;
- perNode: Array<{
- nodeId: string;
- nodeName: string;
- progress: DownloadProgress;
- }>;
+ perNode: NodeDownloadStatus[];
} {
- if (!downloadsData || Object.keys(downloadsData).length === 0) {
- // No download data yet — defer to runner status instead of assuming RUNNING
- const statusInfo = deriveInstanceStatus(instanceWrapped);
- return {
- isDownloading: false,
- isFailed: false,
- errorMessage: null,
- progress: null,
- statusText: statusInfo.statusText,
- perNode: [],
- };
- }
-
- // Unwrap the instance
+ // Unwrap the instance to get shard assignments
const [instanceTag, instance] = getTagged(instanceWrapped);
if (!instance || typeof instance !== "object") {
return {
@@ -1716,132 +1760,45 @@
modelId?: string;
};
};
- const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
- const runnerToShard = inst.shardAssignments?.runnerToShard || {};
const instanceModelId = inst.shardAssignments?.modelId;
- // Build reverse mapping: runnerId -> nodeId
+ if (!instanceModelId) {
+ const statusInfo = deriveInstanceStatus(instanceWrapped);
+ return {
+ isDownloading: false,
+ isFailed: statusInfo.statusText === "FAILED",
+ errorMessage: null,
+ progress: null,
+ statusText: statusInfo.statusText,
+ perNode: [],
+ };
+ }
+
+ // Get node IDs assigned to this instance
+ const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
+ const runnerToShard = inst.shardAssignments?.runnerToShard || {};
const runnerToNode: Record<string, string> = {};
for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
runnerToNode[runnerId] = nodeId;
}
+ const instanceNodeIds = Object.keys(runnerToShard)
+ .map((runnerId) => runnerToNode[runnerId])
+ .filter(Boolean);
- let totalBytes = 0;
- let downloadedBytes = 0;
- let totalSpeed = 0;
- let completedFiles = 0;
- let totalFiles = 0;
- let isDownloading = false;
- const allFiles: DownloadProgress["files"] = [];
- const perNode: Array<{
- nodeId: string;
- nodeName: string;
- progress: DownloadProgress;
- }> = [];
-
- // Check downloads for nodes that are part of this instance
- for (const runnerId of Object.keys(runnerToShard)) {
- const nodeId = runnerToNode[runnerId];
- if (!nodeId) continue;
-
- const nodeDownloads = downloadsData[nodeId];
- if (!Array.isArray(nodeDownloads)) continue;
-
- for (const downloadWrapped of nodeDownloads) {
- if (!downloadWrapped || typeof downloadWrapped !== "object") continue;
-
- const keys = Object.keys(downloadWrapped as Record<string, unknown>);
- if (keys.length !== 1) continue;
-
- const downloadKind = keys[0];
- const downloadPayload = (downloadWrapped as Record<string, unknown>)[
- downloadKind
- ] as Record<string, unknown>;
+ const result = collectDownloadStatus(instanceModelId, instanceNodeIds);
- // Handle DownloadFailed - return immediately with error info
- if (downloadKind === "DownloadFailed") {
- const downloadModelId = extractModelIdFromDownload(downloadPayload);
- if (
- instanceModelId &&
- downloadModelId &&
- downloadModelId === instanceModelId
- ) {
- return {
- isDownloading: false,
- isFailed: true,
- errorMessage:
- (downloadPayload.errorMessage as string) || "Download failed",
- progress: null,
- statusText: "FAILED",
- perNode: [],
- };
- }
- }
-
- if (
- downloadKind !== "DownloadOngoing" &&
- downloadKind !== "DownloadPending"
- )
- continue;
- if (!downloadPayload) continue;
-
- // Check if this download is for this instance's model
- const downloadModelId = extractModelIdFromDownload(downloadPayload);
- if (
- instanceModelId &&
- downloadModelId &&
- downloadModelId === instanceModelId
- ) {
- // For DownloadPending with partial bytes, synthesize progress
- let progress: DownloadProgress | null;
- if (downloadKind === "DownloadPending") {
- const pendingDownloaded = getBytes(
- downloadPayload.downloaded ??
- downloadPayload.downloaded_bytes ??
- downloadPayload.downloadedBytes,
- );
- const pendingTotal = getBytes(
- downloadPayload.total ??
- downloadPayload.total_bytes ??
- downloadPayload.totalBytes,
- );
- if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
- isDownloading = true;
- progress = {
- totalBytes: pendingTotal,
- downloadedBytes: pendingDownloaded,
- speed: 0,
- etaMs: 0,
- percentage:
- pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
- completedFiles: 0,
- totalFiles: 0,
- files: [],
- };
- } else {
- isDownloading = true;
- progress = parseDownloadProgress(downloadPayload);
- }
-
- if (progress) {
- // Sum all values across nodes - each node downloads independently
- totalBytes += progress.totalBytes;
- downloadedBytes += progress.downloadedBytes;
- totalSpeed += progress.speed;
- completedFiles += progress.completedFiles;
- totalFiles += progress.totalFiles;
- allFiles.push(...progress.files);
-
- const nodeName =
- data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
- perNode.push({ nodeId, nodeName, progress });
- }
- }
- }
+ if (result.failedError) {
+ return {
+ isDownloading: false,
+ isFailed: true,
+ errorMessage: result.failedError,
+ progress: null,
+ statusText: "FAILED",
+ perNode: [],
+ };
}
- if (!isDownloading) {
- // Check runner status for other states
+ if (!result.isDownloading) {
const statusInfo = deriveInstanceStatus(instanceWrapped);
return {
isDownloading: false,
@@ -1849,30 +1806,17 @@
errorMessage: null,
progress: null,
statusText: statusInfo.statusText,
- perNode: [],
+ perNode: result.perNode,
};
}
- // ETA = total remaining bytes / total speed across all nodes
- const remainingBytes = totalBytes - downloadedBytes;
- const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
-
return {
isDownloading: true,
isFailed: false,
errorMessage: null,
- progress: {
- totalBytes,
- downloadedBytes,
- speed: totalSpeed,
- etaMs,
- percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
- completedFiles,
- totalFiles,
- files: allFiles,
- },
+ progress: result.progress,
statusText: "DOWNLOADING",
- perNode,
+ perNode: result.perNode,
};
}
@@ -5369,10 +5313,10 @@
<div
class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
>
- {#each downloadInfo.perNode as nodeProg}
+ {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
{@const nodePercent = Math.min(
100,
- Math.max(0, nodeProg.progress.percentage),
+ Math.max(0, nodeProg.percentage),
)}
{@const isExpanded =
instanceDownloadExpandedNodes.has(
@@ -5428,15 +5372,17 @@
>
<span
>{formatBytes(
- nodeProg.progress.downloadedBytes,
+ nodeProg.progress?.downloadedBytes ??
+ 0,
)} / {formatBytes(
- nodeProg.progress.totalBytes,
+ nodeProg.progress?.totalBytes ?? 0,
)}</span
>
<span
- >{formatSpeed(nodeProg.progress.speed)} •
- ETA {formatEta(
- nodeProg.progress.etaMs,
+ >{formatSpeed(
+ nodeProg.progress?.speed ?? 0,
+ )} • ETA {formatEta(
+ nodeProg.progress?.etaMs ?? 0,
)}</span
>
</div>
@@ -5444,14 +5390,14 @@
{#if isExpanded}
<div class="mt-2 space-y-1.5">
- {#if nodeProg.progress.files.length === 0}
+ {#if nodeProg.progress?.files ?? [].length === 0}
<div
class="text-[11px] font-mono text-exo-light-gray/70"
>
No file details reported.
</div>
{:else}
- {#each nodeProg.progress.files as f}
+ {#each nodeProg.progress?.files ?? [] as f}
{@const filePercent = Math.min(
100,
Math.max(0, f.percentage ?? 0),
@@ -5927,12 +5873,15 @@
)}
{@const allPreviews = filteredPreviews()}
{#if selectedModel && allPreviews.length > 0}
- {@const downloadStatus = getModelDownloadStatus(
- selectedModel.id,
- )}
{@const tags = modelTags()[selectedModel.id] || []}
<div class="space-y-3">
{#each allPreviews as apiPreview, i}
+ {@const downloadStatus = getModelDownloadStatus(
+ selectedModel.id,
+ apiPreview.memory_delta_by_node
+ ? Object.keys(apiPreview.memory_delta_by_node)
+ : undefined,
+ )}
<div
role="group"
onmouseenter={() => {
@@ -6503,10 +6452,10 @@
<div
class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
>
- {#each downloadInfo.perNode as nodeProg}
+ {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
{@const nodePercent = Math.min(
100,
- Math.max(0, nodeProg.progress.percentage),
+ Math.max(0, nodeProg.percentage),
)}
{@const isExpanded =
instanceDownloadExpandedNodes.has(
@@ -6565,16 +6514,17 @@
>
<span
>{formatBytes(
- nodeProg.progress.downloadedBytes,
+ nodeProg.progress
+ ?.downloadedBytes ?? 0,
)} / {formatBytes(
- nodeProg.progress.totalBytes,
+ nodeProg.progress?.totalBytes ?? 0,
)}</span
>
<span
>{formatSpeed(
- nodeProg.progress.speed,
+ nodeProg.progress?.speed ?? 0,
)} • ETA {formatEta(
- nodeProg.progress.etaMs,
+ nodeProg.progress?.etaMs ?? 0,
)}</span
>
</div>
@@ -6582,14 +6532,14 @@
{#if isExpanded}
<div class="mt-2 space-y-1.5">
- {#if nodeProg.progress.files.length === 0}
+ {#if nodeProg.progress?.files ?? [].length === 0}
<div
class="text-[11px] font-mono text-exo-light-gray/70"
>
No file details reported.
</div>
{:else}
- {#each nodeProg.progress.files as f}
+ {#each nodeProg.progress?.files ?? [] as f}
{@const filePercent = Math.min(
100,
Math.max(0, f.percentage ?? 0),
diff --git a/src/exo/shared/apply.py b/src/exo/shared/apply.py
index 53f3499e..38132a32 100644
--- a/src/exo/shared/apply.py
+++ b/src/exo/shared/apply.py
@@ -115,7 +115,13 @@ def apply_node_download_progress(event: NodeDownloadProgress, state: State) -> S
replaced = False
for i, existing_dp in enumerate(current):
- if existing_dp.shard_metadata == dp.shard_metadata:
+ # TODO(ciaran): deduplicate by model_id for now. Will need to use
+ # shard_metadata again when pipeline and tensor downloads differ.
+ # For now this is fine
+ if (
+ existing_dp.shard_metadata.model_card.model_id
+ == dp.shard_metadata.model_card.model_id
+ ):
current[i] = dp
replaced = True
break
← a6519ba0 Update mflux to 0.16.9 (#1751)
·
back to Exo
·
teeny refactor (#1753) 07598a3a →