← back to Exo
feat: add custom HuggingFace model support (#1368)
2063278906d96e9017c48d53d54d9e1631b6cda9 · 2026-02-04 05:06:15 -0800 · Alex Cheema
## Motivation
Users should be able to run any HuggingFace model, not just the ones we
ship TOML cards for. Continues the aim of #1191 with a minimal
implementation on top of the current TOML model card system.
Custom cards are saved to `~/.exo/custom_model_cards/` rather than the
bundled `resources/inference_model_cards/` because `RESOURCES_DIR` is
read-only in PyInstaller bundles (`sys._MEIPASS`). This also fixes
`fetch_from_hf` which was saving cards to the wrong path (`resources/`
root instead of `resources/inference_model_cards/`).
## Changes
- Add `EXO_CUSTOM_MODEL_CARDS_DIR` constant
(`~/.exo/custom_model_cards/`)
- Update `model_cards.py`: add custom dir to search path, fix
`save_to_custom_dir`, add `delete_custom_card`/`is_custom_card`
- Add `POST /models/add` and `DELETE /models/custom/{model_id}` API
endpoints
- Add `is_custom` field to `ModelListModel` API response
- Dashboard: add custom model input form in dropdown, delete button for
custom models, show actual API errors, auto-select newly added model
## Why It Works
Two separate directories for model cards: the bundled read-only
`resources/inference_model_cards/` for built-in cards, and user-writable
`~/.exo/custom_model_cards/` for custom cards. Both are scanned when
listing models. This works in all environments including PyInstaller
bundles where `RESOURCES_DIR` points to `sys._MEIPASS`.
## Test Plan
### Manual Testing
- Add a custom model via the dropdown (e.g.
`mlx-community/Llama-3.2-1B-Instruct-4bit`)
- Verify it appears in the model list with the delete (x) button
- Delete it and verify it disappears
- Try adding an invalid model ID and verify the actual error is shown
### Automated Testing
- `uv run basedpyright` — 0 errors
- `uv run ruff check` — passes
- `uv run pytest src/` — passes
- `cd dashboard && npm run build` — builds
---------
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Files touched
M dashboard/src/routes/+page.svelteM src/exo/master/api.pyM src/exo/shared/constants.pyM src/exo/shared/models/model_cards.pyM src/exo/shared/types/api.py
Diff
commit 2063278906d96e9017c48d53d54d9e1631b6cda9
Author: Alex Cheema <41707476+AlexCheema@users.noreply.github.com>
Date: Wed Feb 4 05:06:15 2026 -0800
feat: add custom HuggingFace model support (#1368)
## Motivation
Users should be able to run any HuggingFace model, not just the ones we
ship TOML cards for. Continues the aim of #1191 with a minimal
implementation on top of the current TOML model card system.
Custom cards are saved to `~/.exo/custom_model_cards/` rather than the
bundled `resources/inference_model_cards/` because `RESOURCES_DIR` is
read-only in PyInstaller bundles (`sys._MEIPASS`). This also fixes
`fetch_from_hf` which was saving cards to the wrong path (`resources/`
root instead of `resources/inference_model_cards/`).
## Changes
- Add `EXO_CUSTOM_MODEL_CARDS_DIR` constant
(`~/.exo/custom_model_cards/`)
- Update `model_cards.py`: add custom dir to search path, fix
`save_to_custom_dir`, add `delete_custom_card`/`is_custom_card`
- Add `POST /models/add` and `DELETE /models/custom/{model_id}` API
endpoints
- Add `is_custom` field to `ModelListModel` API response
- Dashboard: add custom model input form in dropdown, delete button for
custom models, show actual API errors, auto-select newly added model
## Why It Works
Two separate directories for model cards: the bundled read-only
`resources/inference_model_cards/` for built-in cards, and user-writable
`~/.exo/custom_model_cards/` for custom cards. Both are scanned when
listing models. This works in all environments including PyInstaller
bundles where `RESOURCES_DIR` points to `sys._MEIPASS`.
## Test Plan
### Manual Testing
- Add a custom model via the dropdown (e.g.
`mlx-community/Llama-3.2-1B-Instruct-4bit`)
- Verify it appears in the model list with the delete (x) button
- Delete it and verify it disappears
- Try adding an invalid model ID and verify the actual error is shown
### Automated Testing
- `uv run basedpyright` — 0 errors
- `uv run ruff check` — passes
- `uv run pytest src/` — passes
- `cd dashboard && npm run build` — builds
---------
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
---
dashboard/src/routes/+page.svelte | 126 ++++++++++++++++++++++++++++++++---
src/exo/master/api.py | 36 ++++++++++
src/exo/shared/constants.py | 2 +
src/exo/shared/models/model_cards.py | 36 ++++++++--
src/exo/shared/types/api.py | 5 ++
5 files changed, 191 insertions(+), 14 deletions(-)
diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte
index 183b6547..6ecce7cc 100644
--- a/dashboard/src/routes/+page.svelte
+++ b/dashboard/src/routes/+page.svelte
@@ -100,6 +100,7 @@
storage_size_megabytes?: number;
tasks?: string[];
hugging_face_id?: string;
+ is_custom?: boolean;
}>
>([]);
@@ -215,6 +216,11 @@
let isModelDropdownOpen = $state(false);
let modelDropdownSearch = $state("");
+ // Custom model add state
+ let customModelInput = $state("");
+ let isAddingCustomModel = $state(false);
+ let customModelError = $state("");
+
// Slider dragging state
let isDraggingSlider = $state(false);
let sliderTrackElement: HTMLDivElement | null = $state(null);
@@ -530,6 +536,57 @@
}
}
+ async function addCustomModel() {
+ const modelId = customModelInput.trim();
+ if (!modelId) return;
+
+ isAddingCustomModel = true;
+ customModelError = "";
+
+ try {
+ const response = await fetch("/models/add", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ model_id: modelId }),
+ });
+
+ if (!response.ok) {
+ try {
+ const err = await response.json();
+ customModelError =
+ err.detail || `Failed to add model (${response.status})`;
+ } catch {
+ customModelError = `Failed to add model (${response.status}: ${response.statusText})`;
+ }
+ return;
+ }
+
+ const added = await response.json();
+ customModelInput = "";
+ await fetchModels();
+ selectPreviewModel(added.id);
+ isModelDropdownOpen = false;
+ } catch {
+ customModelError = "Network error";
+ } finally {
+ isAddingCustomModel = false;
+ }
+ }
+
+ async function deleteCustomModel(modelId: string) {
+ try {
+ const response = await fetch(
+ `/models/custom/${encodeURIComponent(modelId)}`,
+ { method: "DELETE" },
+ );
+ if (response.ok) {
+ await fetchModels();
+ }
+ } catch {
+ console.error("Failed to delete custom model");
+ }
+ }
+
async function launchInstance(
modelId: string,
specificPreview?: PlacementPreview | null,
@@ -2475,7 +2532,7 @@
>
<!-- Search within dropdown -->
<div
- class="sticky top-0 bg-exo-dark-gray border-b border-exo-medium-gray/30 p-2"
+ class="sticky top-0 bg-exo-dark-gray border-b border-exo-medium-gray/30 p-2 space-y-2"
>
<input
type="text"
@@ -2483,6 +2540,34 @@
bind:value={modelDropdownSearch}
class="w-full bg-exo-dark-gray/60 border border-exo-medium-gray/30 rounded px-2 py-1.5 text-xs font-mono text-white/80 placeholder:text-white/40 focus:outline-none focus:border-exo-yellow/50"
/>
+ <!-- Add custom model -->
+ <form
+ onsubmit={(e) => {
+ e.preventDefault();
+ addCustomModel();
+ }}
+ class="flex gap-1.5"
+ >
+ <input
+ type="text"
+ placeholder="org/model-name"
+ bind:value={customModelInput}
+ class="flex-1 bg-exo-dark-gray/60 border border-exo-medium-gray/30 rounded px-2 py-1.5 text-xs font-mono text-white/80 placeholder:text-white/40 focus:outline-none focus:border-exo-yellow/50"
+ />
+ <button
+ type="submit"
+ disabled={!customModelInput.trim() ||
+ isAddingCustomModel}
+ class="px-2 py-1.5 text-xs font-mono bg-exo-yellow/20 text-exo-yellow border border-exo-yellow/30 rounded hover:bg-exo-yellow/30 disabled:opacity-40 disabled:cursor-not-allowed"
+ >
+ {isAddingCustomModel ? "..." : "+ ADD"}
+ </button>
+ </form>
+ {#if customModelError}
+ <div class="text-xs text-red-400 font-mono">
+ {customModelError}
+ </div>
+ {/if}
</div>
<!-- Options -->
@@ -2557,14 +2642,37 @@
{/if}
<span class="truncate">{model.name || model.id}</span>
</span>
- <span
- class="flex-shrink-0 text-xs {modelCanFit
- ? 'text-white/50'
- : 'text-red-400/60'}"
- >
- {sizeGB >= 1
- ? sizeGB.toFixed(0)
- : sizeGB.toFixed(1)}GB
+ <span class="flex items-center gap-1.5 flex-shrink-0">
+ <span
+ class="text-xs {modelCanFit
+ ? 'text-white/50'
+ : 'text-red-400/60'}"
+ >
+ {sizeGB >= 1
+ ? sizeGB.toFixed(0)
+ : sizeGB.toFixed(1)}GB
+ </span>
+ {#if model.is_custom}
+ <button
+ type="button"
+ onclick={(e) => {
+ e.stopPropagation();
+ deleteCustomModel(model.id);
+ }}
+ class="text-white/30 hover:text-red-400 transition-colors p-0.5"
+ title="Remove custom model"
+ >
+ <svg
+ class="w-3 h-3"
+ fill="none"
+ viewBox="0 0 24 24"
+ stroke="currentColor"
+ stroke-width="2"
+ >
+ <path d="M18 6L6 18M6 6l12 12" />
+ </svg>
+ </button>
+ {/if}
</span>
</button>
{:else}
diff --git a/src/exo/master/api.py b/src/exo/master/api.py
index 8cf86725..01e145b4 100644
--- a/src/exo/master/api.py
+++ b/src/exo/master/api.py
@@ -50,10 +50,13 @@ from exo.shared.logging import InterceptLogger
from exo.shared.models.model_cards import (
ModelCard,
ModelId,
+ delete_custom_card,
get_model_cards,
+ is_custom_card,
)
from exo.shared.tracing import TraceEvent, compute_stats, export_trace, load_trace_file
from exo.shared.types.api import (
+ AddCustomModelParams,
AdvancedImageParams,
BenchChatCompletionRequest,
BenchChatCompletionResponse,
@@ -257,6 +260,8 @@ class API:
self.app.delete("/instance/{instance_id}")(self.delete_instance)
self.app.get("/models")(self.get_models)
self.app.get("/v1/models")(self.get_models)
+ self.app.post("/models/add")(self.add_custom_model)
+ self.app.delete("/models/custom/{model_id:path}")(self.delete_custom_model)
self.app.post("/v1/chat/completions", response_model=None)(
self.chat_completions
)
@@ -1216,11 +1221,42 @@ class API:
storage_size_megabytes=int(card.storage_size.in_mb),
supports_tensor=card.supports_tensor,
tasks=[task.value for task in card.tasks],
+ is_custom=is_custom_card(card.model_id),
)
for card in await get_model_cards()
]
)
+ async def add_custom_model(self, payload: AddCustomModelParams) -> ModelListModel:
+ """Fetch a model from HuggingFace and save as a custom model card."""
+ try:
+ card = await ModelCard.fetch_from_hf(payload.model_id)
+ except Exception as exc:
+ raise HTTPException(
+ status_code=400, detail=f"Failed to fetch model: {exc}"
+ ) from exc
+
+ return ModelListModel(
+ id=card.model_id,
+ hugging_face_id=card.model_id,
+ name=card.model_id.short(),
+ description="",
+ tags=[],
+ storage_size_megabytes=int(card.storage_size.in_mb),
+ supports_tensor=card.supports_tensor,
+ tasks=[task.value for task in card.tasks],
+ is_custom=True,
+ )
+
+ async def delete_custom_model(self, model_id: ModelId) -> JSONResponse:
+ """Delete a user-added custom model card."""
+ deleted = await delete_custom_card(model_id)
+ if not deleted:
+ raise HTTPException(status_code=404, detail="Custom model card not found")
+ return JSONResponse(
+ {"message": "Model card deleted", "model_id": str(model_id)}
+ )
+
async def run(self):
cfg = Config()
cfg.bind = f"0.0.0.0:{self.port}"
diff --git a/src/exo/shared/constants.py b/src/exo/shared/constants.py
index 39438cbd..b09d6b2d 100644
--- a/src/exo/shared/constants.py
+++ b/src/exo/shared/constants.py
@@ -58,6 +58,8 @@ LIBP2P_COMMANDS_TOPIC = "commands"
EXO_MAX_CHUNK_SIZE = 512 * 1024
+EXO_CUSTOM_MODEL_CARDS_DIR = EXO_DATA_HOME / "custom_model_cards"
+
EXO_IMAGE_CACHE_DIR = EXO_CACHE_HOME / "images"
EXO_TRACING_CACHE_DIR = EXO_CACHE_HOME / "traces"
diff --git a/src/exo/shared/models/model_cards.py b/src/exo/shared/models/model_cards.py
index 58d84849..47b54d38 100644
--- a/src/exo/shared/models/model_cards.py
+++ b/src/exo/shared/models/model_cards.py
@@ -18,14 +18,19 @@ from pydantic import (
)
from tomlkit.exceptions import TOMLKitError
-from exo.shared.constants import EXO_ENABLE_IMAGE_MODELS, RESOURCES_DIR
+from exo.shared.constants import (
+ EXO_CUSTOM_MODEL_CARDS_DIR,
+ EXO_ENABLE_IMAGE_MODELS,
+ RESOURCES_DIR,
+)
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.utils.pydantic_ext import CamelCaseModel
# kinda ugly...
# TODO: load search path from config.toml
-_csp = [Path(RESOURCES_DIR) / "inference_model_cards"]
+_custom_cards_dir = Path(str(EXO_CUSTOM_MODEL_CARDS_DIR))
+_csp = [Path(RESOURCES_DIR) / "inference_model_cards", _custom_cards_dir]
if EXO_ENABLE_IMAGE_MODELS:
_csp.append(Path(RESOURCES_DIR) / "image_model_cards")
@@ -85,8 +90,9 @@ class ModelCard(CamelCaseModel):
data = tomlkit.dumps(py) # pyright: ignore[reportUnknownMemberType]
await f.write(data)
- async def save_to_default_path(self):
- await self.save(Path(RESOURCES_DIR) / (self.model_id.normalize() + ".toml"))
+ async def save_to_custom_dir(self) -> None:
+ await aios.makedirs(str(_custom_cards_dir), exist_ok=True)
+ await self.save(_custom_cards_dir / (self.model_id.normalize() + ".toml"))
@staticmethod
async def load_from_path(path: Path) -> "ModelCard":
@@ -120,11 +126,31 @@ class ModelCard(CamelCaseModel):
supports_tensor=config_data.supports_tensor,
tasks=[ModelTask.TextGeneration],
)
- await mc.save_to_default_path()
+ await mc.save_to_custom_dir()
_card_cache[model_id] = mc
return mc
+async def delete_custom_card(model_id: ModelId) -> bool:
+ """Delete a user-added custom model card. Returns True if deleted."""
+ card_path = _custom_cards_dir / (ModelId(model_id).normalize() + ".toml")
+ if await card_path.exists():
+ await card_path.unlink()
+ _card_cache.pop(model_id, None)
+ return True
+ return False
+
+
+def is_custom_card(model_id: ModelId) -> bool:
+ """Check if a model card exists in the custom cards directory."""
+ import os
+
+ card_path = Path(str(EXO_CUSTOM_MODEL_CARDS_DIR)) / (
+ ModelId(model_id).normalize() + ".toml"
+ )
+ return os.path.isfile(str(card_path))
+
+
# TODO: quantizing and dynamically creating model cards
def _generate_image_model_quant_variants( # pyright: ignore[reportUnusedFunction]
base_name: str,
diff --git a/src/exo/shared/types/api.py b/src/exo/shared/types/api.py
index 40dbb288..e5710014 100644
--- a/src/exo/shared/types/api.py
+++ b/src/exo/shared/types/api.py
@@ -42,6 +42,7 @@ class ModelListModel(BaseModel):
storage_size_megabytes: int = Field(default=0)
supports_tensor: bool = Field(default=False)
tasks: list[str] = Field(default=[])
+ is_custom: bool = Field(default=False)
class ModelList(BaseModel):
@@ -201,6 +202,10 @@ class BenchChatCompletionRequest(ChatCompletionRequest):
pass
+class AddCustomModelParams(BaseModel):
+ model_id: ModelId
+
+
class PlaceInstanceParams(BaseModel):
model_id: ModelId
sharding: Sharding = Sharding.Pipeline
← a0f4f363 Reduce reliance on internet (#1363)
·
back to Exo
·
feat: add model picker modal with grouped models and HF Hub 41ed7afb →