Add more model cards (#1970)

fd707de30b42db4211d15da96b9052e1dc280ed1 · 2026-04-23 15:28:40 +0100 · rltakashige
Files touched

A resources/inference_model_cards/mlx-community--GLM-5.1-DQ4plus-q8.toml
A resources/inference_model_cards/mlx-community--GLM-5.1-MXFP4-Q8.toml
A resources/inference_model_cards/mlx-community--GLM-5.1.toml
A resources/inference_model_cards/mlx-community--Kimi-K2.6-mlx-DQ3_K_M-q8.toml
A resources/inference_model_cards/mlx-community--Qwen3.6-27B-4bit.toml
A resources/inference_model_cards/mlx-community--Qwen3.6-27B-8bit.toml
A resources/inference_model_cards/mlx-community--Qwen3.6-27B-bf16.toml
M resources/inference_model_cards/moonshotai--Kimi-K2.6.toml
M src/exo/download/download_utils.py
Diff

commit fd707de30b42db4211d15da96b9052e1dc280ed1
Author: rltakashige <rl.takashige@gmail.com>
Date:   Thu Apr 23 15:28:40 2026 +0100

    Add more model cards (#1970)
---
 .../mlx-community--GLM-5.1-DQ4plus-q8.toml         | 21 +++++++++++++
 .../mlx-community--GLM-5.1-MXFP4-Q8.toml           | 21 +++++++++++++
 .../mlx-community--GLM-5.1.toml                    | 21 +++++++++++++
 .../mlx-community--Kimi-K2.6-mlx-DQ3_K_M-q8.toml   | 33 ++++++++++++++++++++
 .../mlx-community--Qwen3.6-27B-4bit.toml           | 35 ++++++++++++++++++++++
 .../mlx-community--Qwen3.6-27B-8bit.toml           | 35 ++++++++++++++++++++++
 .../mlx-community--Qwen3.6-27B-bf16.toml           | 35 ++++++++++++++++++++++
 .../moonshotai--Kimi-K2.6.toml                     |  1 +
 src/exo/download/download_utils.py                 | 27 ++++++++++++++++-
 9 files changed, 228 insertions(+), 1 deletion(-)

diff --git a/resources/inference_model_cards/mlx-community--GLM-5.1-DQ4plus-q8.toml b/resources/inference_model_cards/mlx-community--GLM-5.1-DQ4plus-q8.toml
new file mode 100644
index 00000000..3912a7b7
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--GLM-5.1-DQ4plus-q8.toml
@@ -0,0 +1,21 @@
+model_id = "mlx-community/GLM-5.1-DQ4plus-q8"
+n_layers = 78
+hidden_size = 6144
+num_key_value_heads = 64
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "glm"
+quantization = "8bit"
+base_model = "GLM-5.1"
+capabilities = ["text", "thinking"]
+
+context_length = 202752
+
+[storage_size]
+in_bytes = 465173655552
+
+# Source: https://huggingface.co/zai-org/GLM-5.1
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
diff --git a/resources/inference_model_cards/mlx-community--GLM-5.1-MXFP4-Q8.toml b/resources/inference_model_cards/mlx-community--GLM-5.1-MXFP4-Q8.toml
new file mode 100644
index 00000000..fc644788
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--GLM-5.1-MXFP4-Q8.toml
@@ -0,0 +1,21 @@
+model_id = "mlx-community/GLM-5.1-MXFP4-Q8"
+n_layers = 78
+hidden_size = 6144
+num_key_value_heads = 64
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "glm"
+quantization = "MXFP4-Q8"
+base_model = "GLM-5.1"
+capabilities = ["text", "thinking"]
+
+context_length = 202752
+
+[storage_size]
+in_bytes = 405480321024
+
+# Source: https://huggingface.co/zai-org/GLM-5.1
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
diff --git a/resources/inference_model_cards/mlx-community--GLM-5.1.toml b/resources/inference_model_cards/mlx-community--GLM-5.1.toml
new file mode 100644
index 00000000..dec2122e
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--GLM-5.1.toml
@@ -0,0 +1,21 @@
+model_id = "mlx-community/GLM-5.1"
+n_layers = 78
+hidden_size = 6144
+num_key_value_heads = 64
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "glm"
+quantization = "bf16"
+base_model = "GLM-5.1"
+capabilities = ["text", "thinking"]
+
+context_length = 202752
+
+[storage_size]
+in_bytes = 1487822475264
+
+# Source: https://huggingface.co/zai-org/GLM-5.1
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
diff --git a/resources/inference_model_cards/mlx-community--Kimi-K2.6-mlx-DQ3_K_M-q8.toml b/resources/inference_model_cards/mlx-community--Kimi-K2.6-mlx-DQ3_K_M-q8.toml
new file mode 100644
index 00000000..4bc0b97c
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--Kimi-K2.6-mlx-DQ3_K_M-q8.toml
@@ -0,0 +1,33 @@
+model_id = "mlx-community/Kimi-K2.6-mlx-DQ3_K_M-q8"
+n_layers = 61
+hidden_size = 7168
+num_key_value_heads = 64
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "kimi"
+quantization = "3bit"
+base_model = "Kimi K2.6"
+capabilities = ["text", "thinking", "thinking_toggle", "vision"]
+
+context_length = 262144
+
+[storage_size]
+in_bytes = 470628683776
+
+[vision]
+image_token_id = 163605
+model_type = "kimi_vl"
+weights_repo = "exolabs/Kimi-K2.6-vision"
+processor_repo = "moonshotai/Kimi-K2.6"
+
+# Source: https://huggingface.co/moonshotai/Kimi-K2.6
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+min_p = 0.01
+
+# Source: https://huggingface.co/moonshotai/Kimi-K2.6
+[sampling_defaults.non_thinking]
+temperature = 0.6
+top_p = 0.95
+min_p = 0.01
diff --git a/resources/inference_model_cards/mlx-community--Qwen3.6-27B-4bit.toml b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-4bit.toml
new file mode 100644
index 00000000..d8188395
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-4bit.toml
@@ -0,0 +1,35 @@
+model_id = "mlx-community/Qwen3.6-27B-4bit"
+n_layers = 64
+hidden_size = 5120
+num_key_value_heads = 4
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "qwen"
+quantization = "4bit"
+base_model = "Qwen3.6 27B"
+capabilities = ["text", "thinking", "thinking_toggle", "vision"]
+
+context_length = 262144
+
+[storage_size]
+in_bytes = 16054262240
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
diff --git a/resources/inference_model_cards/mlx-community--Qwen3.6-27B-8bit.toml b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-8bit.toml
new file mode 100644
index 00000000..91d5ec9e
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-8bit.toml
@@ -0,0 +1,35 @@
+model_id = "mlx-community/Qwen3.6-27B-8bit"
+n_layers = 64
+hidden_size = 5120
+num_key_value_heads = 4
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "qwen"
+quantization = "8bit"
+base_model = "Qwen3.6 27B"
+capabilities = ["text", "thinking", "thinking_toggle", "vision"]
+
+context_length = 262144
+
+[storage_size]
+in_bytes = 29500938720
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
diff --git a/resources/inference_model_cards/mlx-community--Qwen3.6-27B-bf16.toml b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-bf16.toml
new file mode 100644
index 00000000..f688f3a6
--- /dev/null
+++ b/resources/inference_model_cards/mlx-community--Qwen3.6-27B-bf16.toml
@@ -0,0 +1,35 @@
+model_id = "mlx-community/Qwen3.6-27B-bf16"
+n_layers = 64
+hidden_size = 5120
+num_key_value_heads = 4
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "qwen"
+quantization = "bf16"
+base_model = "Qwen3.6 27B"
+capabilities = ["text", "thinking", "thinking_toggle", "vision"]
+
+context_length = 262144
+
+[storage_size]
+in_bytes = 54713457120
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.6-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
diff --git a/resources/inference_model_cards/moonshotai--Kimi-K2.6.toml b/resources/inference_model_cards/moonshotai--Kimi-K2.6.toml
index d18c73c1..8d9d19b7 100644
--- a/resources/inference_model_cards/moonshotai--Kimi-K2.6.toml
+++ b/resources/inference_model_cards/moonshotai--Kimi-K2.6.toml
@@ -17,6 +17,7 @@ in_bytes = 595148192736
 [vision]
 image_token_id = 163605
 model_type = "kimi_vl"
+weights_repo = "exolabs/Kimi-K2.6-vision"
 processor_repo = "moonshotai/Kimi-K2.6"
 
 # Source: https://huggingface.co/moonshotai/Kimi-K2.6
diff --git a/src/exo/download/download_utils.py b/src/exo/download/download_utils.py
index 5aedfb20..bab70a1c 100644
--- a/src/exo/download/download_utils.py
+++ b/src/exo/download/download_utils.py
@@ -167,6 +167,29 @@ def select_download_dir(required_bytes: int) -> Path:
     )
 
 
+async def select_download_dir_for_shard(
+    model_id: ModelId,
+    filtered_file_list: list[FileListEntry],
+    total_size: int,
+) -> Path:
+    for candidate_dir in EXO_MODELS_DIRS:
+        if not candidate_dir.exists():
+            continue
+        sub = candidate_dir / model_id.normalize()
+        if not await aios.path.isdir(sub):
+            continue
+        existing_bytes = 0
+        for file_entry in filtered_file_list:
+            existing_bytes += await get_downloaded_size(sub / file_entry.path)
+        remaining = max(total_size - existing_bytes, 0)
+        try:
+            if shutil.disk_usage(candidate_dir).free >= remaining:
+                return candidate_dir
+        except OSError:
+            continue
+    return select_download_dir(total_size)
+
+
 async def resolve_model_dir(model_id: ModelId) -> Path:
     """Return the directory for a model's files, creating it if needed.
 
@@ -852,7 +875,9 @@ async def download_shard(
             else EXO_DEFAULT_MODELS_DIR / model_id.normalize()
         )
     else:
-        models_dir = select_download_dir(total_size)
+        models_dir = await select_download_dir_for_shard(
+            model_id, filtered_file_list, total_size
+        )
         target_dir = models_dir / model_id.normalize()
         await aios.makedirs(target_dir, exist_ok=True)
     file_progress: dict[str, RepoFileDownloadProgress] = {}
← 45248c5c chore(app): hardcode bug report presigned-URL endpoint (#197 · back to Exo · fix: map presence_penalty and frequency_penalty from ChatCom 48a922fd →