← back to Exo

resources/inference_model_cards/mlx-community--DeepSeek-V3.1-8bit.toml

23 lines

model_id = "mlx-community/DeepSeek-V3.1-8bit"
n_layers = 61
hidden_size = 7168
num_key_value_heads = 128
supports_tensor = true
tasks = ["TextGeneration"]
family = "deepseek"
quantization = "8bit"
base_model = "DeepSeek V3.1"
capabilities = ["text", "thinking", "thinking_toggle"]
reasoning_dialect = "post_last_user"

context_length = 131072
backends = ["MlxMetal", "MlxCuda", "MlxCpu"]
[storage_size]
in_bytes = 765577920512

# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/main/generation_config.json
# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/discussions/19
[sampling_defaults]
temperature = 0.6
top_p = 0.95