set KV_CACHE_BITS to None to disable quantized kv cache

31d4cd84097dfa04aa2da3eb4e5efd67a7cdd42c · 2025-12-30 16:49:32 +0000 · Alex Cheema

Files touched

M src/exo/worker/engines/mlx/constants.py

Diff

commit 31d4cd84097dfa04aa2da3eb4e5efd67a7cdd42c
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Tue Dec 30 16:49:32 2025 +0000

    set KV_CACHE_BITS to None to disable quantized kv cache
---
 src/exo/worker/engines/mlx/constants.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/exo/worker/engines/mlx/constants.py b/src/exo/worker/engines/mlx/constants.py
index 0c17c97d..2544e9be 100644
--- a/src/exo/worker/engines/mlx/constants.py
+++ b/src/exo/worker/engines/mlx/constants.py
@@ -9,7 +9,7 @@ MAX_KV_SIZE: int | None = 3200
 KEEP_KV_SIZE: int | None = 1600
 QUANTIZE_MODEL_MODE: str | None = "affine"
 CACHE_GROUP_SIZE: int = 64
-KV_CACHE_BITS: int | None = 8
+KV_CACHE_BITS: int | None = None
 
 # TODO: We should really make this opt-in, but Kimi requires trust_remote_code=True
 TRUST_REMOTE_CODE: bool = True

← 8a6da584 remove mx.set_cache_limit · back to Exo · prioritise mlx jaccl coordinator ip (en0 -> en1 -> non-TB5 - 35be4c55 →