handle -gzip suffix in etag for integrity check fixes #633

ee095766d94c20badf2655ab80c7b66793c9e1b6 · 2025-02-25 22:08:15 +0000 · Alex Cheema

Files touched

M exo/download/new_shard_download.py

Diff

commit ee095766d94c20badf2655ab80c7b66793c9e1b6
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Tue Feb 25 22:08:15 2025 +0000

    handle -gzip suffix in etag for integrity check fixes #633
---
 exo/download/new_shard_download.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/exo/download/new_shard_download.py b/exo/download/new_shard_download.py
index 59c661c5..02e9f7bf 100644
--- a/exo/download/new_shard_download.py
+++ b/exo/download/new_shard_download.py
@@ -141,7 +141,8 @@ async def download_file_with_retry(repo_id: str, revision: str, path: str, targe
 async def _download_file(repo_id: str, revision: str, path: str, target_dir: Path, on_progress: Callable[[int, int], None] = lambda _, __: None) -> Path:
   if await aios.path.exists(target_dir/path): return target_dir/path
   await aios.makedirs((target_dir/path).parent, exist_ok=True)
-  length, remote_hash = await file_meta(repo_id, revision, path)
+  length, etag = await file_meta(repo_id, revision, path)
+  remote_hash = etag[:-5] if etag.endswith("-gzip") else etag
   partial_path = target_dir/f"{path}.partial"
   resume_byte_pos = (await aios.stat(partial_path)).st_size if (await aios.path.exists(partial_path)) else None
   if resume_byte_pos != length:

← f9a1e534 update notice in README · back to Exo · bump grpcio and grpcio-tools to 1.70.0 36a6389a →