some finishing touches to get this working e2e

20241e32907831e74745724655c342f94cbff1dd · 2025-07-28 13:07:29 +0100 · Alex Cheema
Files touched

M master/api.py
M master/main.py
M master/placement.py
M shared/types/api.py
Diff

commit 20241e32907831e74745724655c342f94cbff1dd
Author: Alex Cheema <41707476+AlexCheema@users.noreply.github.com>
Date:   Mon Jul 28 13:07:29 2025 +0100

    some finishing touches to get this working e2e
---
 master/api.py       | 25 +++++++++++++++++++++++++
 master/main.py      |  2 +-
 master/placement.py |  7 ++++++-
 shared/types/api.py | 21 +++++++++++++++++++--
 4 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/master/api.py b/master/api.py
index 5b4ea986..69efb21b 100644
--- a/master/api.py
+++ b/master/api.py
@@ -5,6 +5,7 @@ from typing import Callable, List, Sequence, final
 
 import uvicorn
 from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
 
 from shared.db.sqlite.connector import AsyncSQLiteEventStorage
@@ -16,6 +17,8 @@ from shared.types.api import (
     CreateInstanceResponse,
     CreateInstanceTaskParams,
     DeleteInstanceResponse,
+    ModelList,
+    ModelListModel,
     StreamingChoiceResponse,
 )
 from shared.types.common import CommandId
@@ -64,12 +67,22 @@ async def resolve_model_meta(model_id: str) -> ModelMetadata:
 class API:
     def __init__(self, command_buffer: List[Command], global_events: AsyncSQLiteEventStorage, get_state: Callable[[], State]) -> None:
         self._app = FastAPI()
+        self._setup_cors()
         self._setup_routes()
 
         self.command_buffer = command_buffer
         self.global_events = global_events
         self.get_state = get_state
 
+    def _setup_cors(self) -> None:
+        self._app.add_middleware(
+            CORSMiddleware,
+            allow_origins=["*"],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
     def _setup_routes(self) -> None:
         # self._app.get("/topology/control_plane")(self.get_control_plane_topology)
         # self._app.get("/topology/data_plane")(self.get_data_plane_topology)
@@ -77,6 +90,8 @@ class API:
         self._app.post("/instance")(self.create_instance)
         self._app.get("/instance/{instance_id}")(self.get_instance)
         self._app.delete("/instance/{instance_id}")(self.delete_instance)
+        self._app.get("/models")(self.get_models)
+        self._app.get("/v1/models")(self.get_models)
         # self._app.get("/model/{model_id}/metadata")(self.get_model_data)
         # self._app.post("/model/{model_id}/instances")(self.get_instances_by_model)
         self._app.post("/v1/chat/completions")(self.chat_completions)
@@ -196,6 +211,16 @@ class API:
             media_type="text/plain"
         )
 
+    async def get_models(self) -> ModelList:
+        """Returns list of available models."""
+        return ModelList(data=[
+            ModelListModel(
+                id=card.short_id,
+                hugging_face_id=card.model_id,
+                name=card.name,
+                description=card.description,
+                tags=card.tags) for card in MODEL_CARDS.values()])
+
 
 
 def start_fastapi_server(
diff --git a/master/main.py b/master/main.py
index 6417b9c4..45224d66 100644
--- a/master/main.py
+++ b/master/main.py
@@ -127,7 +127,7 @@ class Master:
             print(f"applying event: {event_from_log}")
             self.state = apply(self.state, event_from_log)
 
-        self.logger.info(f"state: {self.state.model_dump_json()}")
+        self.logger.info(f"state: {self.state}")
 
     async def run(self):
         self.state = await self._get_state_snapshot()
diff --git a/master/placement.py b/master/placement.py
index e502f5d3..da15c650 100644
--- a/master/placement.py
+++ b/master/placement.py
@@ -1,3 +1,4 @@
+import random
 from collections.abc import Mapping
 from copy import deepcopy
 from functools import singledispatch
@@ -17,6 +18,9 @@ from shared.types.worker.common import InstanceId
 from shared.types.worker.instances import Instance, InstanceStatus
 
 
+def random_ephemeral_port() -> int:
+    return random.randint(49152, 65535)
+
 @singledispatch
 def get_instance_placements(
     command: CreateInstanceCommand,
@@ -52,7 +56,8 @@ def get_instance_placements(
         shard_assignments=shard_assignments,
         hosts=[Host(
             ip=host.ip,
-            port=host.port,
+            # NOTE: it's fine to have non-deterministic ports here since this is in a command decision
+            port=random_ephemeral_port(),
         ) for host in hosts]
     )
     return target_instances
diff --git a/shared/types/api.py b/shared/types/api.py
index 98d99468..cdf9913e 100644
--- a/shared/types/api.py
+++ b/shared/types/api.py
@@ -1,6 +1,7 @@
-from typing import Any, Literal
+import time
+from typing import Any, List, Literal
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from shared.openai_compat import FinishReason
 from shared.types.common import CommandId
@@ -8,6 +9,22 @@ from shared.types.models import ModelMetadata
 from shared.types.worker.instances import InstanceId
 
 
+class ModelListModel(BaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "exo"
+    # openwebui fields
+    hugging_face_id: str = Field(default="")
+    name: str = Field(default="")
+    description: str = Field(default="")
+    context_length: int = Field(default=0)
+    tags: List[str] = Field(default=[])
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: List[ModelListModel]
+
 class ChatCompletionMessage(BaseModel):
     role: Literal["system", "user", "assistant", "developer", "tool", "function"]
     content: str | None = None
← 176d077c Fix IPv4 serialisation for topology · back to Exo · fix ci linter dbd0bdc3 →