remove mx.set_cache_limit

8a6da58404c74345e9bfd05dbf8f4663662b82c7 · 2025-12-30 16:52:16 +0000 · Alex Cheema

Files touched

M src/exo/worker/engines/mlx/utils_mlx.py

Diff

commit 8a6da58404c74345e9bfd05dbf8f4663662b82c7
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Tue Dec 30 16:52:16 2025 +0000

    remove mx.set_cache_limit
---
 src/exo/worker/engines/mlx/utils_mlx.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/exo/worker/engines/mlx/utils_mlx.py b/src/exo/worker/engines/mlx/utils_mlx.py
index 1aa9b827..8d5d3a16 100644
--- a/src/exo/worker/engines/mlx/utils_mlx.py
+++ b/src/exo/worker/engines/mlx/utils_mlx.py
@@ -395,11 +395,5 @@ def set_wired_limit_for_model(model_size: Memory):
             "MB. This can be slow. See the documentation for possible work-arounds: "
             "https://github.com/ml-explore/mlx-lm/tree/main#large-models"
         )
-    kv_bytes = int(0.02 * model_bytes)
-    target_cache = int(1.10 * (model_bytes + kv_bytes))
-    target_cache = min(target_cache, max_rec_size)
-    mx.set_cache_limit(target_cache)
     mx.set_wired_limit(max_rec_size)
-    logger.info(
-        f"Wired limit set to {max_rec_size}. Cache limit set to {target_cache}."
-    )
+    logger.info(f"Wired limit set to {max_rec_size}.")

← 16e2bfd3 log EXO_LIBP2P_NAMESPACE on start · back to Exo · set KV_CACHE_BITS to None to disable quantized kv cache 31d4cd84 →