increase download chunks to 1MB

ae770db4f3184ed551b81138c77d590ac6ebeffe · 2025-01-27 05:37:50 +0000 · Alex Cheema

Files touched

M exo/download/new_shard_download.py

Diff

commit ae770db4f3184ed551b81138c77d590ac6ebeffe
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Mon Jan 27 05:37:50 2025 +0000

    increase download chunks to 1MB
---
 exo/download/new_shard_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/download/new_shard_download.py b/exo/download/new_shard_download.py
index b4dbea41..cfc22d38 100644
--- a/exo/download/new_shard_download.py
+++ b/exo/download/new_shard_download.py
@@ -94,7 +94,7 @@ async def download_file(session: aiohttp.ClientSession, repo_id: str, revision:
     length = int(r.headers.get('content-length', 0))
     n_read = 0
     async with aiofiles.tempfile.NamedTemporaryFile(dir=target_dir, delete=False) as temp_file:
-      while chunk := await r.content.read(16384): on_progress(n_read := n_read + await temp_file.write(chunk), length)
+      while chunk := await r.content.read(1024 * 1024): on_progress(n_read := n_read + await temp_file.write(chunk), length)
       await aios.rename(temp_file.name, target_dir/path)
     return target_dir/path

← 82f75d0c increase hf download http timeout 15 mins for large download · back to Exo · increase file download timeout to 30min 4748bb7d →