← back to Exo

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-8bit.toml

23 lines

model_id = "mlx-community/DeepSeek-V3.2-8bit"
n_layers = 61
hidden_size = 7168
num_key_value_heads = 128
supports_tensor = true
tasks = ["TextGeneration"]
family = "deepseek"
quantization = "8bit"
base_model = "DeepSeek V3.2"
capabilities = ["text", "thinking", "thinking_toggle"]
reasoning_dialect = "tool_conditional"

context_length = 131072
backends = ["MlxMetal", "MlxCuda", "MlxCpu"]
[storage_size]
in_bytes = 755957120916

# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/generation_config.json
# Source: https://docs.vllm.ai/projects/recipes/en/latest/DeepSeek/DeepSeek-V3_2.html
[sampling_defaults]
temperature = 1.0
top_p = 0.95