← back to Exo
New Runner!
21acd3794ac5865263f33d0b638e940bab7de1db · 2025-07-10 16:34:35 +0100 · Matt Beton
Files touched
M .gitignoreM pyproject.tomlA shared/mlx/auto_parallel.pyA shared/mlx/utils_mlx.pyM shared/openai.pyM shared/types/events/events.pyM shared/types/states/master.pyM shared/types/tasks/common.pyM shared/types/worker/commands_runner.pyM shared/types/worker/downloads.pyM shared/types/worker/mlx.pyM shared/types/worker/runners.pyM shared/types/worker/shards.pyM uv.lockM worker/pyproject.tomlA worker/runner/communication.pyA worker/runner/conftest.pyA worker/runner/runner.pyA worker/runner/runner_supervisor.pyA worker/runner/test_serdes.pyA worker/runner/test_supervisor.pyA worker/runner/utils.py
Diff
commit 21acd3794ac5865263f33d0b638e940bab7de1db
Author: Matt Beton <matthew.beton@gmail.com>
Date: Thu Jul 10 16:34:35 2025 +0100
New Runner!
---
.gitignore | 4 +-
pyproject.toml | 5 +
shared/mlx/auto_parallel.py | 93 +++++++
shared/mlx/utils_mlx.py | 119 +++++++++
shared/openai.py | 15 +-
shared/types/events/events.py | 2 +-
shared/types/states/master.py | 6 +-
shared/types/tasks/common.py | 55 ++++-
shared/types/worker/commands_runner.py | 10 +-
shared/types/worker/downloads.py | 4 +-
shared/types/worker/mlx.py | 6 +-
shared/types/worker/runners.py | 16 +-
shared/types/worker/shards.py | 16 +-
uv.lock | 428 ++++++++++++++++++++++++++++-----
worker/pyproject.toml | 11 +-
worker/runner/communication.py | 75 ++++++
worker/runner/conftest.py | 107 +++++++++
worker/runner/runner.py | 151 ++++++++++++
worker/runner/runner_supervisor.py | 175 ++++++++++++++
worker/runner/test_serdes.py | 33 +++
worker/runner/test_supervisor.py | 190 +++++++++++++++
worker/runner/utils.py | 8 +
22 files changed, 1428 insertions(+), 101 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4f2f08c0..8275d34c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
*/__pycache__
-__pycache__
\ No newline at end of file
+__pycache__
+
+hosts_*.json
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 73dca1bf..77ed9d0e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,7 @@ dev = [
"basedpyright>=1.29.4",
"maturin>=1.9.0",
"pytest>=8.4.0",
+ "pytest-asyncio>=1.0.0",
"ruff>=0.11.13",
]
@@ -105,3 +106,7 @@ extend-exclude = ["shared/protobufs/**"]
[tool.ruff.lint]
extend-select = ["I", "N", "B", "A", "PIE", "SIM"]
+
+[tool.pytest.ini_options]
+pythonpath = "."
+asyncio_mode = "auto"
\ No newline at end of file
diff --git a/shared/mlx/auto_parallel.py b/shared/mlx/auto_parallel.py
new file mode 100644
index 00000000..987933bf
--- /dev/null
+++ b/shared/mlx/auto_parallel.py
@@ -0,0 +1,93 @@
+from typing import Protocol, cast, override
+
+import mlx.core as mx
+import mlx.nn as nn
+
+from shared.types.worker.shards import PipelineShardMeta
+
+
+class IdentityLayer(nn.Module):
+ @override
+ def __call__(self, x: mx.array, *args: object, **kwargs: object) -> mx.array:
+ return x
+
+class _LayerCallable(Protocol):
+ """Structural type that any compatible layer must satisfy.
+
+ We require a single positional input of type ``mx.array`` and an
+ ``mx.array`` output, while permitting arbitrary *args / **kwargs so this
+ protocol matches the vast majority of `mlx.nn.Module` subclasses.
+ """
+
+ def __call__(self, x: mx.array, *args: object, **kwargs: object) -> mx.array: ...
+
+class PipelineFirstLayer(nn.Module):
+ def __init__(self, original_layer: _LayerCallable, r: int, s: int):
+ super().__init__()
+ self.original_layer: _LayerCallable = original_layer
+ self.r: int = r
+ self.s: int = s
+
+ @override
+ def __call__(self, x: mx.array, *args: object, **kwargs: object) -> mx.array:
+ if self.r != 0:
+ x = mx.distributed.recv_like(x, (self.r - 1))
+ return self.original_layer(x, *args, **kwargs)
+
+class PipelineLastLayer(nn.Module):
+ def __init__(self, original_layer: _LayerCallable, r: int, s: int):
+ super().__init__()
+ self.original_layer: _LayerCallable = original_layer
+ self.r: int = r
+ self.s: int = s
+
+ @override
+ def __call__(self, x: mx.array, *args: object, **kwargs: object) -> mx.array:
+ output: mx.array = self.original_layer(x, *args, **kwargs)
+ if self.r != self.s - 1:
+ output = mx.distributed.send(output, (self.r + 1) % self.s)
+ output = mx.distributed.all_gather(output)[-output.shape[0]:] # pyright: ignore[reportUnknownMemberType]
+ return output
+
+def inner_model(model: nn.Module) -> nn.Module:
+ inner = getattr(model, 'model', None)
+ if isinstance(inner, nn.Module):
+ return inner
+
+ inner = getattr(model, 'transformer', None)
+ if isinstance(inner, nn.Module):
+ return inner
+
+ raise ValueError("Model must either have a 'model' or 'transformer' attribute")
+
+# def auto_parallel(model: nn.Module, rank: int, size: int, start_layer: int, end_layer: int) -> nn.Module:
+def auto_parallel(model: nn.Module, model_shard_meta: PipelineShardMeta) -> nn.Module:
+ """
+ Automatically parallelize a model across multiple devices.
+
+ Args:
+ model: The model to parallelize (must have a 'layers' or 'h' property)
+ model_shard_meta: The metadata for the model shard
+
+ Returns:
+ The parallelized model
+ """
+
+ inner_model_instance: nn.Module = inner_model(model)
+
+ # Handle both model.layers and model.h cases
+ layers: list[_LayerCallable]
+ if hasattr(inner_model_instance, 'layers'):
+ layers = cast(list[_LayerCallable], inner_model_instance.layers)
+ else:
+ layers = cast(list[_LayerCallable], inner_model_instance.h)
+
+ layers[:model_shard_meta.start_layer] = [IdentityLayer() for _ in range(model_shard_meta.start_layer)]
+ layers[model_shard_meta.end_layer:] = [IdentityLayer() for _ in range(len(layers) - model_shard_meta.end_layer)]
+ layers[model_shard_meta.start_layer] = PipelineFirstLayer(layers[model_shard_meta.start_layer], model_shard_meta.device_rank, model_shard_meta.world_size)
+ layers[model_shard_meta.end_layer - 1] = PipelineLastLayer(layers[model_shard_meta.end_layer - 1], model_shard_meta.device_rank, model_shard_meta.world_size)
+
+ # At this point `layers` *must* be a concrete list.
+ assert isinstance(layers, list), "Expected a list of layers after auto-parallel initialisation"
+
+ return model
\ No newline at end of file
diff --git a/shared/mlx/utils_mlx.py b/shared/mlx/utils_mlx.py
new file mode 100644
index 00000000..397593d3
--- /dev/null
+++ b/shared/mlx/utils_mlx.py
@@ -0,0 +1,119 @@
+# type: ignore
+
+
+import asyncio
+import concurrent.futures
+import os
+from asyncio import AbstractEventLoop
+from typing import Callable
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx_lm.sample_utils import make_sampler
+from mlx_lm.tokenizer_utils import TokenizerWrapper, load_tokenizer
+from mlx_lm.utils import load_model
+from pydantic import RootModel
+
+from shared.mlx.auto_parallel import auto_parallel
+from shared.types.tasks.common import ChatCompletionParams
+from shared.types.worker.mlx import Host
+from shared.types.worker.shards import ShardMeta
+from worker.runner.communication import runner_print
+
+
+def mx_barrier():
+ mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.default_stream(mx.Device(mx.cpu)))) # type: ignore
+
+class HostList(RootModel[list[str]]):
+
+ @classmethod
+ def from_hosts(cls, hosts: list[Host]) -> "HostList":
+ return cls(root=[str(host) for host in hosts])
+
+def mlx_distributed_init(rank: int, hosts: list[Host]) -> mx.distributed.Group:
+ """
+ Initialize the MLX distributed (runs in thread pool)
+ """
+ runner_print(f"Starting initialization for rank {rank}")
+
+ # Setup distributed environment
+ hostfile = f"./hosts_{rank}.json" # TODO: this needs to be unique?
+ hosts_json = HostList.from_hosts(hosts).model_dump_json()
+
+ runner_print(f'rank {rank} hostfile: {hostfile} hosts: {hosts_json}')
+
+ with open(hostfile, "w") as f:
+ _ = f.write(hosts_json)
+
+ os.environ["MLX_HOSTFILE"] = hostfile
+ os.environ["MLX_RANK"] = str(rank)
+ os.environ["MLX_RING_VERBOSE"] = "1"
+
+ # Initialize distributed
+ group = mx.distributed.init(backend="ring", strict=True)
+ runner_print(f"Rank {rank} mlx distributed initialization complete")
+
+ return group
+
+def initialize_mlx(
+ model_shard_meta: ShardMeta,
+ hosts: list[Host],
+) -> tuple[nn.Module, TokenizerWrapper, Callable[[mx.array], mx.array]]:
+ """
+ Initialize the MLX model, tokenizer, and sampler. Runs in the MLX thread.
+ """
+ mx.random.seed(42)
+ if len(hosts) > 1:
+ mlx_distributed_init(model_shard_meta.device_rank, hosts)
+ sampler: Callable[[mx.array], mx.array] = make_sampler(temp=0.7)
+
+ model, tokenizer = shard_and_load(model_shard_meta)
+
+ return model, tokenizer, sampler
+
+def shard_and_load(model_shard_meta: ShardMeta) -> tuple[nn.Module, TokenizerWrapper]:
+ runner_print(f'loading model from {model_shard_meta.model_path}')
+
+ model, config = load_model(model_shard_meta.model_path, lazy=True, strict=False)
+
+ tokenizer = load_tokenizer(model_shard_meta.model_path)
+ assert isinstance(tokenizer, TokenizerWrapper)
+ model = auto_parallel(model, model_shard_meta)
+
+ # Synchronize processes before generation to avoid timeout
+ mx_barrier()
+
+ return model, tokenizer
+
+
+async def apply_chat_template(
+ mlx_executor: concurrent.futures.ThreadPoolExecutor,
+ tokenizer: TokenizerWrapper,
+ chat_task: ChatCompletionParams,
+) -> str:
+ loop: AbstractEventLoop = asyncio.get_running_loop()
+
+ # Now we can properly access the messages
+ messages = chat_task.messages
+ messages_dicts = [msg.model_dump() for msg in messages]
+
+ # Filter out None values, keeping only 'role' and 'content' keys
+ formatted_messages = []
+ for message in messages_dicts:
+ filtered_message = {k: v for k, v in message.items() if v is not None}
+ # Verify we have exactly the expected keys
+ assert set(filtered_message.keys()) == {'role', 'content'}, f"Expected only 'role' and 'content' keys, got: {filtered_message.keys()}"
+ formatted_messages.append(filtered_message)
+
+ messages_dicts = formatted_messages
+
+ prompt: str = await loop.run_in_executor(
+ executor=mlx_executor,
+ func=lambda: tokenizer.apply_chat_template(
+ messages_dicts,
+ tokenize=False,
+ add_generation_prompt=True,
+ )
+ )
+
+ return prompt
\ No newline at end of file
diff --git a/shared/openai.py b/shared/openai.py
index 0a0a546f..db367ad3 100644
--- a/shared/openai.py
+++ b/shared/openai.py
@@ -1,20 +1,21 @@
from typing import TYPE_CHECKING, Literal, TypeAlias, get_type_hints
+FinishReason: TypeAlias = Literal[
+ "stop", "length", "tool_calls", "content_filter", "function_call"
+]
+
if TYPE_CHECKING:
import openai.types as openai_types
import openai.types.chat as openai_chat
types = openai_types
chat = openai_chat
+
+ assert (
+ get_type_hints(chat.chat_completion_chunk.Choice)["finish_reason"] == FinishReason
+ ), "Upstream changed Choice.finish_reason; update FinishReason alias."
else:
types = None
chat = None
-FinishReason: TypeAlias = Literal[
- "stop", "length", "tool_calls", "content_filter", "function_call"
-]
-assert (
- get_type_hints(chat.chat_completion_chunk.Choice)["finish_reason"] == FinishReason
-), "Upstream changed Choice.finish_reason; update FinishReason alias."
-
__all__ = ["types", "chat", "FinishReason"]
diff --git a/shared/types/events/events.py b/shared/types/events/events.py
index 1f6422c8..fdb70d23 100644
--- a/shared/types/events/events.py
+++ b/shared/types/events/events.py
@@ -58,7 +58,7 @@ class TimerData(BaseModel):
class TaskCreated[TaskTypeT: TaskType](TaskEvent):
event_type: TaskEventTypes = TaskEventTypes.TaskCreated
task_id: TaskId
- task_data: TaskData[TaskTypeT]
+ task_data: TaskData
task_state: TaskState[Literal[TaskStatusIncompleteType.Pending], TaskTypeT]
on_instance: InstanceId
diff --git a/shared/types/states/master.py b/shared/types/states/master.py
index e1233b11..15721254 100644
--- a/shared/types/states/master.py
+++ b/shared/types/states/master.py
@@ -24,7 +24,7 @@ from shared.types.networking.topology import (
)
from shared.types.profiling.common import NodePerformanceProfile
from shared.types.states.shared import SharedState
-from shared.types.tasks.common import TaskData, TaskType
+from shared.types.tasks.common import TaskData
from shared.types.worker.instances import InstanceData, InstanceId
@@ -65,8 +65,8 @@ class ControlPlaneNetworkState(State[EventCategories.ControlPlaneEventTypes]):
class MasterState(SharedState):
data_plane_network_state: DataPlaneNetworkState
control_plane_network_state: ControlPlaneNetworkState
- job_inbox: Queue[TaskData[TaskType]]
- job_outbox: Queue[TaskData[TaskType]]
+ job_inbox: Queue[TaskData]
+ job_outbox: Queue[TaskData]
cache_policy: CachePolicy[CachePolicyType]
diff --git a/shared/types/tasks/common.py b/shared/types/tasks/common.py
index 7e58c35f..fd4c6a0f 100644
--- a/shared/types/tasks/common.py
+++ b/shared/types/tasks/common.py
@@ -1,8 +1,7 @@
from collections.abc import Mapping
from enum import Enum
-from typing import Annotated, Generic, Literal, TypeVar, Union
+from typing import Annotated, Any, Generic, Literal, TypeVar, Union
-import openai.types.chat as openai
from pydantic import BaseModel, Field, TypeAdapter
from shared.types.common import NewUUID
@@ -21,21 +20,61 @@ class TaskType(str, Enum):
TaskTypeT = TypeVar("TaskTypeT", bound=TaskType, covariant=True)
-class TaskData(BaseModel, Generic[TaskTypeT]): ...
+class BaseTaskData(BaseModel, Generic[TaskTypeT]):
+ task_type: TaskTypeT
-class ChatCompletionNonStreamingTask(TaskData[TaskType.ChatCompletionNonStreaming]):
+# Custom message types that mirror OpenAI's but are designed for serialization
+class ChatCompletionMessage(BaseModel):
+ role: Literal["system", "user", "assistant", "developer", "tool", "function"]
+ content: str | None = None
+ name: str | None = None
+ tool_calls: list[dict[str, Any]] | None = None
+ tool_call_id: str | None = None
+ function_call: dict[str, Any] | None = None
+
+
+class ChatCompletionParams(BaseModel):
+ model: str
+ messages: list[ChatCompletionMessage]
+ frequency_penalty: float | None = None
+ logit_bias: dict[str, int] | None = None
+ logprobs: bool | None = None
+ top_logprobs: int | None = None
+ max_tokens: int | None = None
+ n: int | None = None
+ presence_penalty: float | None = None
+ response_format: dict[str, Any] | None = None
+ seed: int | None = None
+ stop: str | list[str] | None = None
+ stream: bool = False
+ temperature: float | None = None
+ top_p: float | None = None
+ tools: list[dict[str, Any]] | None = None
+ tool_choice: str | dict[str, Any] | None = None
+ parallel_tool_calls: bool | None = None
+ user: str | None = None
+
+class ChatCompletionNonStreamingTask(BaseTaskData[TaskType.ChatCompletionNonStreaming]):
task_type: Literal[TaskType.ChatCompletionNonStreaming] = (
TaskType.ChatCompletionNonStreaming
)
- task_data: openai.completion_create_params.CompletionCreateParams
+ task_data: ChatCompletionParams
-class ChatCompletionStreamingTask(TaskData[TaskType.ChatCompletionStreaming]):
+class ChatCompletionStreamingTask(BaseTaskData[TaskType.ChatCompletionStreaming]):
task_type: Literal[TaskType.ChatCompletionStreaming] = (
TaskType.ChatCompletionStreaming
)
- task_data: openai.completion_create_params.CompletionCreateParams
+ task_data: ChatCompletionParams
+
+
+TaskData = Annotated[
+ ChatCompletionNonStreamingTask | ChatCompletionStreamingTask,
+ Field(discriminator="task_type"),
+]
+
+TaskDataValidator: TypeAdapter[TaskData] = TypeAdapter(TaskData)
class TaskStatusIncompleteType(str, Enum):
@@ -96,7 +135,7 @@ class TaskState[TaskStatusTypeT: TaskStatusType, TaskTypeT: TaskType](BaseModel)
class BaseTask[TaskTypeT: TaskType, TaskStatusTypeT: TaskStatusType](BaseModel):
task_type: TaskTypeT
- task_data: TaskData[TaskTypeT]
+ task_data: TaskData
task_state: TaskState[TaskStatusTypeT, TaskTypeT]
on_instance: InstanceId
diff --git a/shared/types/worker/commands_runner.py b/shared/types/worker/commands_runner.py
index 7f636588..184805a4 100644
--- a/shared/types/worker/commands_runner.py
+++ b/shared/types/worker/commands_runner.py
@@ -4,9 +4,11 @@ from typing import Annotated, Generic, Literal, TypeVar
from pydantic import BaseModel, Field, TypeAdapter
from shared.openai import FinishReason
-from shared.types.api import ChatTask
+
+# Accept TaskData so the runner can handle both streaming and non-streaming chat tasks.
+from shared.types.tasks.common import TaskData
from shared.types.worker.mlx import Host
-from shared.types.worker.shards import PartitionStrategy, ShardMetadata
+from shared.types.worker.shards import ShardMeta
## Messages passed TO the runner
@@ -26,7 +28,7 @@ class BaseRunnerMessage(BaseModel, Generic[MT]):
class SetupMessage(BaseRunnerMessage[MessageType.Setup]):
type: Literal[MessageType.Setup] = Field(default=MessageType.Setup, frozen=True)
- model_shard_meta: ShardMetadata[PartitionStrategy]
+ model_shard_meta: ShardMeta
hosts: list[Host]
@@ -34,7 +36,7 @@ class ChatTaskMessage(BaseRunnerMessage[MessageType.ChatTask]):
type: Literal[MessageType.ChatTask] = Field(
default=MessageType.ChatTask, frozen=True
)
- task: ChatTask
+ task: TaskData
class ExitMessage(BaseRunnerMessage[MessageType.Exit]):
diff --git a/shared/types/worker/downloads.py b/shared/types/worker/downloads.py
index c88b2d57..9e768d14 100644
--- a/shared/types/worker/downloads.py
+++ b/shared/types/worker/downloads.py
@@ -15,7 +15,7 @@ from pydantic import BaseModel, Field, PositiveInt
from shared.types.common import NodeId
from shared.types.models.common import ModelId
from shared.types.models.sources import ModelSource
-from shared.types.worker.shards import PartitionStrategy, ShardMetadata
+from shared.types.worker.shards import BaseShardMeta, PartitionStrategy
class DownloadProgressData(BaseModel):
@@ -80,6 +80,6 @@ DownloadEffectHandler = Callable[
def download_shard(
model_id: ModelId,
model_source: ModelSource,
- shard_meta: ShardMetadata[PartitionStrategy],
+ shard_meta: BaseShardMeta[PartitionStrategy],
effect_handlers: Sequence[DownloadEffectHandler],
) -> None: ...
diff --git a/shared/types/worker/mlx.py b/shared/types/worker/mlx.py
index 496ef369..d53a14de 100644
--- a/shared/types/worker/mlx.py
+++ b/shared/types/worker/mlx.py
@@ -7,7 +7,11 @@ class Host(BaseModel):
port: int
@field_validator("port")
- def check_port(self, v: int) -> int:
+ @classmethod
+ def check_port(cls, v: int) -> int:
if not (0 <= v <= 65535):
raise ValueError("Port must be between 0 and 65535")
return v
+
+ def __str__(self) -> str:
+ return f"{self.host}:{self.port}"
diff --git a/shared/types/worker/runners.py b/shared/types/worker/runners.py
index c7528094..239fadf0 100644
--- a/shared/types/worker/runners.py
+++ b/shared/types/worker/runners.py
@@ -8,7 +8,19 @@ from shared.types.common import NodeId
from shared.types.models.common import ModelId
from shared.types.worker.common import RunnerId
from shared.types.worker.downloads import BaseDownloadProgress, DownloadStatus
-from shared.types.worker.shards import PartitionStrategy, ShardMetadata
+from shared.types.worker.shards import BaseShardMeta, PartitionStrategy
+
+
+class RunnerError(Exception):
+ error_type: str
+ error_message: str
+ traceback: str
+
+ def __init__(self, error_type: str, error_message: str, traceback: str):
+ self.error_type = error_type
+ self.error_message = error_message
+ self.traceback = traceback
+ super().__init__(f"{error_type}: {error_message}\n{traceback}")
class RunnerStateType(str, Enum):
@@ -60,7 +72,7 @@ PartitionStrategyT = TypeVar(name="PartitionStrategyT", bound=PartitionStrategy)
class ShardAssignments(BaseModel):
model_id: ModelId
- runner_to_shard: Mapping[RunnerId, ShardMetadata[PartitionStrategy]]
+ runner_to_shard: Mapping[RunnerId, BaseShardMeta[PartitionStrategy]]
node_to_runner: Mapping[NodeId, Sequence[RunnerId]]
@model_validator(mode="after")
diff --git a/shared/types/worker/shards.py b/shared/types/worker/shards.py
index 5b33457d..8ed6f1c6 100644
--- a/shared/types/worker/shards.py
+++ b/shared/types/worker/shards.py
@@ -1,7 +1,7 @@
from enum import Enum
from typing import Annotated, Generic, Literal, TypeVar
-from pydantic import BaseModel, DirectoryPath, Field, TypeAdapter
+from pydantic import BaseModel, ConfigDict, DirectoryPath, Field, TypeAdapter
from shared.types.common import NodeId
from shared.types.models.common import ModelId
@@ -14,7 +14,7 @@ class PartitionStrategy(str, Enum):
PartitionStrategyT = TypeVar(name="PartitionStrategyT", bound=PartitionStrategy)
-class ShardMetadata(BaseModel, Generic[PartitionStrategyT]):
+class BaseShardMeta(BaseModel, Generic[PartitionStrategyT]):
"""
Defines a specific shard of the model that is ready to be run on a device.
Replaces previous `Shard` object.
@@ -26,20 +26,20 @@ class ShardMetadata(BaseModel, Generic[PartitionStrategyT]):
model_path: DirectoryPath
-class PipelineShardMeta(ShardMetadata[PartitionStrategy.pipeline]):
+class PipelineShardMeta(BaseShardMeta[Literal[PartitionStrategy.pipeline]]):
"""
Pipeline parallelism shard meta.
"""
+ model_config = ConfigDict(use_enum_values=False)
- partition_strategy: Literal[PartitionStrategy.pipeline] = Field(
- default=PartitionStrategy.pipeline, frozen=True
- )
+ partition_strategy: Literal[PartitionStrategy.pipeline] = PartitionStrategy.pipeline
start_layer: Annotated[int, Field(ge=0)]
end_layer: Annotated[int, Field(ge=0)]
_ShardMeta = Annotated[PipelineShardMeta, Field(discriminator="partition_strategy")]
-ShardMetaAdapter: TypeAdapter[ShardMetadata[PartitionStrategy]] = TypeAdapter(
+ShardMeta = _ShardMeta # Public alias for the discriminated union
+ShardMetaAdapter: TypeAdapter[BaseShardMeta[PartitionStrategy]] = TypeAdapter(
_ShardMeta
)
@@ -51,4 +51,4 @@ class ShardPlacement(BaseModel, Generic[PartitionStrategyT]):
"""
model_id: ModelId
- shard_assignments: dict[NodeId, ShardMetadata[PartitionStrategyT]]
+ shard_assignments: dict[NodeId, BaseShardMeta[PartitionStrategyT]]
diff --git a/uv.lock b/uv.lock
index d08efbb3..a8021600 100644
--- a/uv.lock
+++ b/uv.lock
@@ -44,23 +44,43 @@ wheels = [
[[package]]
name = "basedpyright"
-version = "1.29.4"
+version = "1.30.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nodejs-wheel-binaries", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/80/fb/bd92196a07e3b4ccee4ff2761a26a05bff77d4da089b67b4b1a547868099/basedpyright-1.29.4.tar.gz", hash = "sha256:2df1976f8591eedf4b4ce8f9d123f43e810cc8cb7cc83c53eec0e2f8044073d0", size = 21961481, upload-time = "2025-06-11T22:25:55.173Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1d/d8/a2c9dfa97de316fe228c978bc4677cadb4dc44971d52db026405b8e58377/basedpyright-1.30.0.tar.gz", hash = "sha256:45f5c94b92a8cb9506998c6d29129becd5a2118f14fdbc0df289b96d6a8ff8bc", size = 22059435, upload-time = "2025-07-09T12:12:58.642Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/d5/dc/180fe721a2574fb3aad4051adcca196ac2d18adaf75122f5eeb47436cca2/basedpyright-1.29.4-py3-none-any.whl", hash = "sha256:e087513979972f83010639c6c1a1c13dd3b1d24ee45f8ecff747962cc2063d6f", size = 11476859, upload-time = "2025-06-11T22:25:52.01Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/62/65a06c403ac5e7fc0e11b5ab7617a584786a9606c4a19b7269dcc3c61eb3/basedpyright-1.30.0-py3-none-any.whl", hash = "sha256:782afca88f88a24429a82d900a77deafe88ac88af256774ee304528dd93344f2", size = 11537772, upload-time = "2025-07-09T12:12:54.568Z" },
]
[[package]]
name = "certifi"
-version = "2025.6.15"
+version = "2025.7.9"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/73/f7/f14b46d4bcd21092d7d3ccef689615220d8a08fb25e564b65d20738e672e/certifi-2025.6.15.tar.gz", hash = "sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b", size = 158753, upload-time = "2025-06-15T02:45:51.329Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/de/8a/c729b6b60c66a38f590c4e774decc4b2ec7b0576be8f1aa984a53ffa812a/certifi-2025.7.9.tar.gz", hash = "sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079", size = 160386, upload-time = "2025-07-09T02:13:58.874Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/84/ae/320161bd181fc06471eed047ecce67b693fd7515b16d495d8932db763426/certifi-2025.6.15-py3-none-any.whl", hash = "sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057", size = 157650, upload-time = "2025-06-15T02:45:49.977Z" },
+ { url = "https://files.pythonhosted.org/packages/66/f3/80a3f974c8b535d394ff960a11ac20368e06b736da395b551a49ce950cce/certifi-2025.7.9-py3-none-any.whl", hash = "sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39", size = 159230, upload-time = "2025-07-09T02:13:57.007Z" },
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" },
+ { url = "https://files.pythonhosted.org/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" },
+ { url = "https://files.pythonhosted.org/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/57/1b090ff183d13cef485dfbe272e2fe57622a76694061353c59da52c9a659/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", size = 146231, upload-time = "2025-05-02T08:33:02.081Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", size = 148243, upload-time = "2025-05-02T08:33:04.063Z" },
+ { url = "https://files.pythonhosted.org/packages/c0/0f/9abe9bd191629c33e69e47c6ef45ef99773320e9ad8e9cb08b8ab4a8d4cb/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", size = 150442, upload-time = "2025-05-02T08:33:06.418Z" },
+ { url = "https://files.pythonhosted.org/packages/67/7c/a123bbcedca91d5916c056407f89a7f5e8fdfce12ba825d7d6b9954a1a3c/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", size = 145147, upload-time = "2025-05-02T08:33:08.183Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/fe/1ac556fa4899d967b83e9893788e86b6af4d83e4726511eaaad035e36595/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", size = 153057, upload-time = "2025-05-02T08:33:09.986Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/ff/acfc0b0a70b19e3e54febdd5301a98b72fa07635e56f24f60502e954c461/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", size = 156454, upload-time = "2025-05-02T08:33:11.814Z" },
+ { url = "https://files.pythonhosted.org/packages/92/08/95b458ce9c740d0645feb0e96cea1f5ec946ea9c580a94adfe0b617f3573/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", size = 154174, upload-time = "2025-05-02T08:33:13.707Z" },
+ { url = "https://files.pythonhosted.org/packages/78/be/8392efc43487ac051eee6c36d5fbd63032d78f7728cb37aebcc98191f1ff/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", size = 149166, upload-time = "2025-05-02T08:33:15.458Z" },
+ { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" },
]
[[package]]
@@ -91,6 +111,7 @@ dev = [
{ name = "basedpyright", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "maturin", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pytest-asyncio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
@@ -107,6 +128,7 @@ dev = [
{ name = "basedpyright", specifier = ">=1.29.4" },
{ name = "maturin", specifier = ">=1.9.0" },
{ name = "pytest", specifier = ">=8.4.0" },
+ { name = "pytest-asyncio", specifier = ">=1.0.0" },
{ name = "ruff", specifier = ">=0.11.13" },
]
@@ -166,10 +188,34 @@ version = "0.1.0"
source = { editable = "worker" }
dependencies = [
{ name = "exo-shared", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "mlx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "mlx-lm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
[package.metadata]
-requires-dist = [{ name = "exo-shared", editable = "shared" }]
+requires-dist = [
+ { name = "exo-shared", editable = "shared" },
+ { name = "mlx", specifier = ">=0.26.1" },
+ { name = "mlx-lm", specifier = ">=0.25.3" },
+]
+
+[[package]]
+name = "filelock"
+version = "3.18.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075, upload-time = "2025-03-14T07:11:40.47Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" },
+]
+
+[[package]]
+name = "fsspec"
+version = "2025.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/f7/27f15d41f0ed38e8fcc488584b57e902b331da7f7c6dcda53721b15838fc/fsspec-2025.5.1.tar.gz", hash = "sha256:2e55e47a540b91843b755e83ded97c6e897fa0942b11490113f09e9c443c2475", size = 303033, upload-time = "2025-05-24T12:03:23.792Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052, upload-time = "2025-05-24T12:03:21.66Z" },
+]
[[package]]
name = "h11"
@@ -180,6 +226,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
]
+[[package]]
+name = "hf-xet"
+version = "1.1.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ed/d4/7685999e85945ed0d7f0762b686ae7015035390de1161dcea9d5276c134c/hf_xet-1.1.5.tar.gz", hash = "sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694", size = 495969, upload-time = "2025-06-20T21:48:38.007Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/00/89/a1119eebe2836cb25758e7661d6410d3eae982e2b5e974bcc4d250be9012/hf_xet-1.1.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23", size = 2687929, upload-time = "2025-06-20T21:48:32.284Z" },
+ { url = "https://files.pythonhosted.org/packages/de/5f/2c78e28f309396e71ec8e4e9304a6483dcbc36172b5cea8f291994163425/hf_xet-1.1.5-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8", size = 2556338, upload-time = "2025-06-20T21:48:30.079Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/2f/6cad7b5fe86b7652579346cb7f85156c11761df26435651cbba89376cd2c/hf_xet-1.1.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1", size = 3102894, upload-time = "2025-06-20T21:48:28.114Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/54/0fcf2b619720a26fbb6cc941e89f2472a522cd963a776c089b189559447f/hf_xet-1.1.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dbba1660e5d810bd0ea77c511a99e9242d920790d0e63c0e4673ed36c4022d18", size = 3002134, upload-time = "2025-06-20T21:48:25.906Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/92/1d351ac6cef7c4ba8c85744d37ffbfac2d53d0a6c04d2cabeba614640a78/hf_xet-1.1.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab34c4c3104133c495785d5d8bba3b1efc99de52c02e759cf711a91fd39d3a14", size = 3171009, upload-time = "2025-06-20T21:48:33.987Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/65/4b2ddb0e3e983f2508528eb4501288ae2f84963586fbdfae596836d5e57a/hf_xet-1.1.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a", size = 3279245, upload-time = "2025-06-20T21:48:36.051Z" },
+]
+
[[package]]
name = "httpcore"
version = "1.0.9"
@@ -208,6 +268,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
]
+[[package]]
+name = "huggingface-hub"
+version = "0.33.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "fsspec", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "hf-xet", marker = "(platform_machine == 'aarch64' and sys_platform == 'darwin') or (platform_machine == 'amd64' and sys_platform == 'darwin') or (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'amd64' and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+ { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fa/42/8a95c5632080ae312c0498744b2b852195e10b05a20b1be11c5141092f4c/huggingface_hub-0.33.2.tar.gz", hash = "sha256:84221defaec8fa09c090390cd68c78b88e3c4c2b7befba68d3dc5aacbc3c2c5f", size = 426637, upload-time = "2025-07-02T06:26:05.156Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/44/f4/5f3f22e762ad1965f01122b42dae5bf0e009286e2dba601ce1d0dba72424/huggingface_hub-0.33.2-py3-none-any.whl", hash = "sha256:3749498bfa91e8cde2ddc2c1db92c79981f40e66434c20133b39e5928ac9bcc5", size = 515373, upload-time = "2025-07-02T06:26:03.072Z" },
+]
+
[[package]]
name = "idna"
version = "3.10"
@@ -226,6 +305,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
]
+[[package]]
+name = "jinja2"
+version = "3.1.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "markupsafe", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
+]
+
[[package]]
name = "jiter"
version = "0.10.0"
@@ -270,21 +361,45 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" },
]
+[[package]]
+name = "markupsafe"
+version = "3.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274, upload-time = "2024-10-18T15:21:24.577Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352, upload-time = "2024-10-18T15:21:25.382Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122, upload-time = "2024-10-18T15:21:26.199Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085, upload-time = "2024-10-18T15:21:27.029Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978, upload-time = "2024-10-18T15:21:27.846Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208, upload-time = "2024-10-18T15:21:28.744Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357, upload-time = "2024-10-18T15:21:29.545Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344, upload-time = "2024-10-18T15:21:30.366Z" },
+ { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510, upload-time = "2024-10-18T15:21:33.625Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486, upload-time = "2024-10-18T15:21:34.611Z" },
+ { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480, upload-time = "2024-10-18T15:21:35.398Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914, upload-time = "2024-10-18T15:21:36.231Z" },
+ { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796, upload-time = "2024-10-18T15:21:37.073Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473, upload-time = "2024-10-18T15:21:37.932Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114, upload-time = "2024-10-18T15:21:39.799Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098, upload-time = "2024-10-18T15:21:40.813Z" },
+]
+
[[package]]
name = "maturin"
-version = "1.9.0"
+version = "1.9.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/2a/3a/117a238e055c7d9de5a27619e09f2762830f3ea227f69e110d86e2ec5bd9/maturin-1.9.0.tar.gz", hash = "sha256:ccb9cb87f8df88d1bab8f49efe3fc77f0abb0639ea4b4ebf4f35549200d16b9e", size = 209543, upload-time = "2025-06-23T14:36:05.768Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/f7/73cf2ae0d6db943a627d28c09f5368735fce6b8b2ad1e1f6bcda2632c80a/maturin-1.9.1.tar.gz", hash = "sha256:97b52fb19d20c1fdc70e4efdc05d79853a4c9c0051030c93a793cd5181dc4ccd", size = 209757, upload-time = "2025-07-08T04:54:43.877Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/b7/3f/3063ce9ace8fe33e02cc05209551a5a0d0af9b7990b14e063876ff149e82/maturin-1.9.0-py3-none-linux_armv6l.whl", hash = "sha256:18d77e395f62a0227697098526be6becb3ceea34a79f338b1b716fb96e42a1b2", size = 8130784, upload-time = "2025-06-23T14:35:35.813Z" },
- { url = "https://files.pythonhosted.org/packages/97/52/cb5491ad290002186af3bcb4768f7bb5c6c8d6917cf0a98b945533cd8c04/maturin-1.9.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:33f046f52327b68c28203efe5ecc4fd1952b4d1fe34e65853092e3347a6a6fa0", size = 16082407, upload-time = "2025-06-23T14:35:39.584Z" },
- { url = "https://files.pythonhosted.org/packages/e1/9c/c6fd50c23875fc741651b2fedfffdf4f671cb74c46e66f365d1f9b861daf/maturin-1.9.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6b075f82dc87fa70d583b1fe909ac5e96f36ec2043721acb82f9d6757e860459", size = 8405709, upload-time = "2025-06-23T14:35:42.248Z" },
- { url = "https://files.pythonhosted.org/packages/c6/44/bf61ff9d3f0db8c5a868da55e7827e5fb1a82642705384bcc85bc9a1918f/maturin-1.9.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:c99003470cb37388a31152af4b00492c5db8d767f689a64f45eb5830adc6f3f4", size = 8152167, upload-time = "2025-06-23T14:35:45.013Z" },
- { url = "https://files.pythonhosted.org/packages/8e/99/634aa686a41f899b39300c28ecca756974609e65e80e7a1b7a77765bd070/maturin-1.9.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:35a506c3139d6847edd160f99fd0da7c7b2bbb4d53e0fef995479eed3a92ac37", size = 8808959, upload-time = "2025-06-23T14:35:47.099Z" },
- { url = "https://files.pythonhosted.org/packages/98/4d/4cfa79bad83d2722c47c058f0b527ac5f27c852845b9e79aca95e4fe09c5/maturin-1.9.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a48d8917e60875a06ef36568c2c4a926b6e2681616a251cc50cbf0a5c8aa7428", size = 7911691, upload-time = "2025-06-23T14:35:49.768Z" },
- { url = "https://files.pythonhosted.org/packages/4d/8b/a9410f5ebccad93f86539ab2f77a7aabb9dd05396f9238125c946dc0798c/maturin-1.9.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:5a7a829b03415b7fcaaabeafb520a92cd32b6dd9e8d12e34c7cd7689d404e6a3", size = 7990238, upload-time = "2025-06-23T14:35:51.8Z" },
- { url = "https://files.pythonhosted.org/packages/13/8c/9dd88d5a30717a01793f81ad561b4e77316e0e6154f73e8b072b9ad3378e/maturin-1.9.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:3aa8de021f91bd41918f4afd1b285e84e1b858e354b1de01597bb97a1b9820e1", size = 10134367, upload-time = "2025-06-23T14:35:54.288Z" },
- { url = "https://files.pythonhosted.org/packages/35/34/bb85f46570b4ff2e7bf0dfb8c7408855df811f15d0c1a22896a4699ac0ac/maturin-1.9.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:289d0c2925a8c8ba3ce058e7b691b1c274fd06e36a915232f4e07fa62266f9b6", size = 9001993, upload-time = "2025-06-23T14:35:56.692Z" },
+ { url = "https://files.pythonhosted.org/packages/48/f2/de43e8954092bd957fbdfbc5b978bf8be40f27aec1a4ebd65e57cfb3ec8a/maturin-1.9.1-py3-none-linux_armv6l.whl", hash = "sha256:fe8f59f9e387fb19635eab6b7381ef718e5dc7a328218e6da604c91f206cbb72", size = 8270244, upload-time = "2025-07-08T04:54:17.962Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/72/36966375c2c2bb2d66df4fa756cfcd54175773719b98d4b26a6b4d1f0bfc/maturin-1.9.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6a9c9d176f6df3a8ec1a4c9c72c8a49674ed13668a03c9ead5fab983bbeeb624", size = 16053959, upload-time = "2025-07-08T04:54:21.153Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/40/4e0da87e563333ff1605fef15bed5858c2a41c0c0404e47f20086f214473/maturin-1.9.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e14eedbc4369dda1347ce9ddc183ade7c513d9975b7ea2b9c9e4211fb74f597a", size = 8407170, upload-time = "2025-07-08T04:54:23.351Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/27/4b29614964c10370effcdfcf34ec57126c9a4b921b7a2c42a94ae3a59cb0/maturin-1.9.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:2f05f07bc887e010c44d32a088aea4f36a2104e301f51f408481e4e9759471a7", size = 8258775, upload-time = "2025-07-08T04:54:25.596Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/5b/b15ad53e1e6733d8798ce903d25d9e05aa3083b2544f1a6f863ea01dd50d/maturin-1.9.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:e7eb54db3aace213420cd545b24a149842e8d6b1fcec046d0346f299d8adfc34", size = 8787295, upload-time = "2025-07-08T04:54:27.154Z" },
+ { url = "https://files.pythonhosted.org/packages/72/d8/b97f4767786eae63bb6b700b342766bcea88da98796bfee290bcddd99fd8/maturin-1.9.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:9d037a37b8ef005eebdea61eaf0e3053ebcad3b740162932fbc120db5fdf5653", size = 8053283, upload-time = "2025-07-08T04:54:28.953Z" },
+ { url = "https://files.pythonhosted.org/packages/95/45/770fc005bceac81f5905c96f37c36f65fa9c3da3f4aa8d4e4d2a883aa967/maturin-1.9.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7c26fb60d80e6a72a8790202bb14dbef956b831044f55d1ce4e2c2e915eb6124", size = 8127120, upload-time = "2025-07-08T04:54:30.779Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/a6/be684b4fce58f8b3a9d3b701c23961d5fe0e1710ed484e2216441997e74f/maturin-1.9.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:e0a2c546c123ed97d1ee0c9cc80a912d9174913643c737c12adf4bce46603bb3", size = 10569627, upload-time = "2025-07-08T04:54:32.54Z" },
+ { url = "https://files.pythonhosted.org/packages/24/ad/7f8a9d8a1b79c2ed6291aaaa22147c98efee729b23df2803c319dd658049/maturin-1.9.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5dde6fbcc36a1173fe74e6629fee36e89df76236247b64b23055f1f820bdf35", size = 8934678, upload-time = "2025-07-08T04:54:34.529Z" },
]
[[package]]
@@ -298,32 +413,73 @@ wheels = [
[[package]]
name = "mlx"
-version = "0.26.1"
+version = "0.26.3"
source = { registry = "https://pypi.org/simple" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a2/a7/871c451fe81274d37022a62f825c1dcd22b30e1f8bd2241f91d9f508c9b9/mlx-0.26.1-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:ccd8662abad0f1340326412d6051c116fcb5c923c4d2a25ba1277ae65ab140dd", size = 32396333, upload-time = "2025-06-04T01:02:29.963Z" },
- { url = "https://files.pythonhosted.org/packages/82/77/720bea5a67934b50372dfd5043864458f103743edcc7c30049e788ea3762/mlx-0.26.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0c113dd7c7ac13af6e39f0132d33a8dc78928e858ba8d18f8c89f8bfa694a358", size = 31871172, upload-time = "2025-06-04T01:03:05.075Z" },
- { url = "https://files.pythonhosted.org/packages/15/4f/83f67bc4fe012dffffd2d96d2767b83fee9b2d7d185611d554ac659cfa4d/mlx-0.26.1-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:2ec37131dbb06c0be78ce56b1731ddab6e56183012e7b83bea79b5329ef7d695", size = 31871791, upload-time = "2025-06-04T01:03:15.384Z" },
- { url = "https://files.pythonhosted.org/packages/4f/fb/4123952002fd91f096ba07ce797b6bb6a32cc7a89c988565e261559f77dd/mlx-0.26.1-cp313-cp313-manylinux_2_31_x86_64.whl", hash = "sha256:db96a53466d8efc6cf2a2918b2d4e29cbf9f25174c838fb3c380c8717a40752f", size = 10120515, upload-time = "2025-06-06T23:07:38.428Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/4a/252ea27179c3733d099d5fef51cf1a3ae4da5ba0cf78f031b631b02bd380/mlx-0.26.3-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:6895cdfbfc79225cc6e6a9ef06c2175124afe16ff5cdba9fa540bbb3450b4fc9", size = 33955210, upload-time = "2025-07-08T21:31:33.549Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/ab/ebcd556b470b776c4f97abdc2f7418921dd49a1d69418f733ce2a9e427f2/mlx-0.26.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f800afe89512581e4a56f29382d3baed70b52708f32fcc213574bdddac725642", size = 33342472, upload-time = "2025-07-08T21:30:33.94Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/87/15d98f0354f2a2022c5606a17f10cee62f558f98ec1308a49b50d838da44/mlx-0.26.3-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:84e2aa1414463d4fd21a18339eda37a52725d7df7e8496a1dfb49feb57898097", size = 33343866, upload-time = "2025-07-08T21:31:32.251Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/6e/b64d31616cabc24073e6f8b1250ca5bb0c930e275cc8c1e4a5d039b5bbb1/mlx-0.26.3-cp313-cp313-manylinux_2_31_x86_64.whl", hash = "sha256:c435d90d367be56173f7c98abbf658f3d61e5bf64a801094e0c0c239db5a1498", size = 10072491, upload-time = "2025-07-08T21:34:00.447Z" },
+]
+
+[[package]]
+name = "mlx-lm"
+version = "0.25.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "jinja2", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "mlx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "transformers", extra = ["sentencepiece"], marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ec/bc/0c3f69a8ff78fc8152985be99b2f83dc7e902b9b96ff5260c6a4958c10f1/mlx_lm-0.25.3.tar.gz", hash = "sha256:40ea0a2849abd804a40a3e388627ae5327918a8656287022610150fd453a2242", size = 154221, upload-time = "2025-07-01T03:04:07.056Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/58/ce/3484a973943572461765977231e3b9b68876a8d7e16c3e6110b81c180a89/mlx_lm-0.25.3-py3-none-any.whl", hash = "sha256:56a84f1ae4a3581b13c84c4d8edaa6704b971b40090b725dfc3b719b522ccc2b", size = 203913, upload-time = "2025-07-01T03:04:05.928Z" },
]
[[package]]
name = "nodejs-wheel-binaries"
-version = "22.16.0"
+version = "22.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/86/8962d1d24ff480f4dd31871f42c8e0d8e2c851cd558a07ee689261d310ab/nodejs_wheel_binaries-22.17.0.tar.gz", hash = "sha256:529142012fb8fd20817ef70e2ef456274df4f49933292e312c8bbc7285af6408", size = 8068, upload-time = "2025-06-29T20:24:25.002Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5d/53/b942c6da4ff6f87a315033f6ff6fed8fd3c22047d7ff5802badaa5dfc2c2/nodejs_wheel_binaries-22.17.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:6545a6f6d2f736d9c9e2eaad7e599b6b5b2d8fd4cbd2a1df0807cbcf51b9d39b", size = 51003554, upload-time = "2025-06-29T20:23:47.042Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/b7/7184a9ad2364912da22f2fe021dc4a3301721131ef7759aeb4a1f19db0b4/nodejs_wheel_binaries-22.17.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:4bea5b994dd87c20f8260031ea69a97c3d282e2d4472cc8908636a313a830d00", size = 51936848, upload-time = "2025-06-29T20:23:52.064Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/7a/0ea425147b8110b8fd65a6c21cfd3bd130cdec7766604361429ef870d799/nodejs_wheel_binaries-22.17.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:885508615274a22499dd5314759c1cf96ba72de03e6485d73b3e5475e7f12662", size = 57925230, upload-time = "2025-06-29T20:23:56.81Z" },
+ { url = "https://files.pythonhosted.org/packages/23/5f/10a3f2ac08a839d065d9ccfd6d9df66bc46e100eaf87a8a5cf149eb3fb8e/nodejs_wheel_binaries-22.17.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f38ce034a602bcab534d55cbe0390521e73e5dcffdd1c4b34354b932172af2", size = 58457829, upload-time = "2025-06-29T20:24:01.945Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/a4/d2ca331e16eef0974eb53702df603c54f77b2a7e2007523ecdbf6cf61162/nodejs_wheel_binaries-22.17.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5eed087855b644c87001fe04036213193963ccd65e7f89949e9dbe28e7743d9b", size = 59778054, upload-time = "2025-06-29T20:24:07.14Z" },
+ { url = "https://files.pythonhosted.org/packages/be/2b/04e0e7f7305fe2ba30fd4610bfb432516e0f65379fe6c2902f4b7b1ad436/nodejs_wheel_binaries-22.17.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:715f413c81500f0770ea8936ef1fc2529b900da8054cbf6da67cec3ee308dc76", size = 60830079, upload-time = "2025-06-29T20:24:12.21Z" },
+]
+
+[[package]]
+name = "numpy"
+version = "2.3.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0f/c6/66f36b7b0d528660dfb4a59cb9b8dd6a3f4c0a3939cd49c404a775ea4a63/nodejs_wheel_binaries-22.16.0.tar.gz", hash = "sha256:d695832f026df3a0cf9a089d222225939de9d1b67f8f0a353b79f015aabbe7e2", size = 8061, upload-time = "2025-05-22T07:27:52.149Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/19/d7c972dfe90a353dbd3efbbe1d14a5951de80c99c9dc1b93cd998d51dc0f/numpy-2.3.1.tar.gz", hash = "sha256:1ec9ae20a4226da374362cca3c62cd753faf2f951440b0e3b98e93c235441d2b", size = 20390372, upload-time = "2025-06-21T12:28:33.469Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/d7/dc/417a5c5f99e53a5d2b3be122506312731eb90fb9630c248e327e2e38cc6b/nodejs_wheel_binaries-22.16.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:986b715a96ed703f8ce0c15712f76fc42895cf09067d72b6ef29e8b334eccf64", size = 50957501, upload-time = "2025-05-22T07:27:20.132Z" },
- { url = "https://files.pythonhosted.org/packages/0e/dd/d6ce48209ed15f5d1fccb29eeaa111f962557123eaf4fd03a7316c42734c/nodejs_wheel_binaries-22.16.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:4ae3cf22138891cb44c3ee952862a257ce082b098b29024d7175684a9a77b0c0", size = 51891634, upload-time = "2025-05-22T07:27:24.029Z" },
- { url = "https://files.pythonhosted.org/packages/80/fa/a07e622fd87717eec3e5cff41575f85ad62717e8698884d28ca809266ca1/nodejs_wheel_binaries-22.16.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71f2de4dc0b64ae43e146897ce811f80ac4f9acfbae6ccf814226282bf4ef174", size = 57857862, upload-time = "2025-05-22T07:27:27.933Z" },
- { url = "https://files.pythonhosted.org/packages/1f/80/52736f9570a93f8e6b7942981dc9770eca2bc7aa1d200c1d54198374a6ca/nodejs_wheel_binaries-22.16.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbfccbcd558d2f142ccf66d8c3a098022bf4436db9525b5b8d32169ce185d99e", size = 58395868, upload-time = "2025-05-22T07:27:32.088Z" },
- { url = "https://files.pythonhosted.org/packages/0f/0e/53616a5ed8fc1fbe9e48bf132862da5a9abf5cc7f8483dab1722ec257187/nodejs_wheel_binaries-22.16.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:447ad796850eb52ca20356ad39b2d296ed8fef3f214921f84a1ccdad49f2eba1", size = 59712469, upload-time = "2025-05-22T07:27:37.193Z" },
- { url = "https://files.pythonhosted.org/packages/4a/cd/e2b5083df581fc1d08eb93feb6f8fbd3d56b113cef9b59d8e0fb7d4dd4f3/nodejs_wheel_binaries-22.16.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7f526ca6a132b0caf633566a2a78c6985fe92857e7bfdb37380f76205a10b808", size = 60763005, upload-time = "2025-05-22T07:27:41.39Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/bd/35ad97006d8abff8631293f8ea6adf07b0108ce6fec68da3c3fcca1197f2/numpy-2.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25a1992b0a3fdcdaec9f552ef10d8103186f5397ab45e2d25f8ac51b1a6b97e8", size = 20889381, upload-time = "2025-06-21T12:19:04.103Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/4f/df5923874d8095b6062495b39729178eef4a922119cee32a12ee1bd4664c/numpy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7dea630156d39b02a63c18f508f85010230409db5b2927ba59c8ba4ab3e8272e", size = 14152726, upload-time = "2025-06-21T12:19:25.599Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/0f/a1f269b125806212a876f7efb049b06c6f8772cf0121139f97774cd95626/numpy-2.3.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:bada6058dd886061f10ea15f230ccf7dfff40572e99fef440a4a857c8728c9c0", size = 5105145, upload-time = "2025-06-21T12:19:34.782Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/63/a7f7fd5f375b0361682f6ffbf686787e82b7bbd561268e4f30afad2bb3c0/numpy-2.3.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:a894f3816eb17b29e4783e5873f92faf55b710c2519e5c351767c51f79d8526d", size = 6639409, upload-time = "2025-06-21T12:19:45.228Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/0d/1854a4121af895aab383f4aa233748f1df4671ef331d898e32426756a8a6/numpy-2.3.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:18703df6c4a4fee55fd3d6e5a253d01c5d33a295409b03fda0c86b3ca2ff41a1", size = 14257630, upload-time = "2025-06-21T12:20:06.544Z" },
+ { url = "https://files.pythonhosted.org/packages/50/30/af1b277b443f2fb08acf1c55ce9d68ee540043f158630d62cef012750f9f/numpy-2.3.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5902660491bd7a48b2ec16c23ccb9124b8abfd9583c5fdfa123fe6b421e03de1", size = 16627546, upload-time = "2025-06-21T12:20:31.002Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/ec/3b68220c277e463095342d254c61be8144c31208db18d3fd8ef02712bcd6/numpy-2.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:36890eb9e9d2081137bd78d29050ba63b8dab95dff7912eadf1185e80074b2a0", size = 15562538, upload-time = "2025-06-21T12:20:54.322Z" },
+ { url = "https://files.pythonhosted.org/packages/77/2b/4014f2bcc4404484021c74d4c5ee8eb3de7e3f7ac75f06672f8dcf85140a/numpy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a780033466159c2270531e2b8ac063704592a0bc62ec4a1b991c7c40705eb0e8", size = 18360327, upload-time = "2025-06-21T12:21:21.053Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/19/a029cd335cf72f79d2644dcfc22d90f09caa86265cbbde3b5702ccef6890/numpy-2.3.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:b0b5397374f32ec0649dd98c652a1798192042e715df918c20672c62fb52d4b8", size = 20987593, upload-time = "2025-06-21T12:21:51.664Z" },
+ { url = "https://files.pythonhosted.org/packages/25/91/8ea8894406209107d9ce19b66314194675d31761fe2cb3c84fe2eeae2f37/numpy-2.3.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c5bdf2015ccfcee8253fb8be695516ac4457c743473a43290fd36eba6a1777eb", size = 14300523, upload-time = "2025-06-21T12:22:13.583Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/7f/06187b0066eefc9e7ce77d5f2ddb4e314a55220ad62dd0bfc9f2c44bac14/numpy-2.3.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d70f20df7f08b90a2062c1f07737dd340adccf2068d0f1b9b3d56e2038979fee", size = 5227993, upload-time = "2025-06-21T12:22:22.53Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/ec/a926c293c605fa75e9cfb09f1e4840098ed46d2edaa6e2152ee35dc01ed3/numpy-2.3.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:2fb86b7e58f9ac50e1e9dd1290154107e47d1eef23a0ae9145ded06ea606f992", size = 6736652, upload-time = "2025-06-21T12:22:33.629Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/62/d68e52fb6fde5586650d4c0ce0b05ff3a48ad4df4ffd1b8866479d1d671d/numpy-2.3.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:23ab05b2d241f76cb883ce8b9a93a680752fbfcbd51c50eff0b88b979e471d8c", size = 14331561, upload-time = "2025-06-21T12:22:55.056Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/ec/b74d3f2430960044bdad6900d9f5edc2dc0fb8bf5a0be0f65287bf2cbe27/numpy-2.3.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ce2ce9e5de4703a673e705183f64fd5da5bf36e7beddcb63a25ee2286e71ca48", size = 16693349, upload-time = "2025-06-21T12:23:20.53Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/15/def96774b9d7eb198ddadfcbd20281b20ebb510580419197e225f5c55c3e/numpy-2.3.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c4913079974eeb5c16ccfd2b1f09354b8fed7e0d6f2cab933104a09a6419b1ee", size = 15642053, upload-time = "2025-06-21T12:23:43.697Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/57/c3203974762a759540c6ae71d0ea2341c1fa41d84e4971a8e76d7141678a/numpy-2.3.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:010ce9b4f00d5c036053ca684c77441f2f2c934fd23bee058b4d6f196efd8280", size = 18434184, upload-time = "2025-06-21T12:24:10.708Z" },
]
[[package]]
name = "openai"
-version = "1.93.0"
+version = "1.93.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -335,9 +491,9 @@ dependencies = [
{ name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/e4/d7/e91c6a9cf71726420cddf539852ee4c29176ebb716a702d9118d0409fd8e/openai-1.93.0.tar.gz", hash = "sha256:988f31ade95e1ff0585af11cc5a64510225e4f5cd392698c675d0a9265b8e337", size = 486573, upload-time = "2025-06-27T21:21:39.421Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e0/66/fadc0cad6a229c6a85c3aa5f222a786ec4d9bf14c2a004f80ffa21dbaf21/openai-1.93.3.tar.gz", hash = "sha256:488b76399238c694af7e4e30c58170ea55e6f65038ab27dbe95b5077a00f8af8", size = 487595, upload-time = "2025-07-09T14:08:27.789Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/64/46/a10d9df4673df56f71201d129ba1cb19eaff3366d08c8664d61a7df52e65/openai-1.93.0-py3-none-any.whl", hash = "sha256:3d746fe5498f0dd72e0d9ab706f26c91c0f646bf7459e5629af8ba7c9dbdf090", size = 755038, upload-time = "2025-06-27T21:21:37.532Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/b9/0df6351b25c6bd494c534d2a8191dc9460fb5bb09c88b1427775d49fde05/openai-1.93.3-py3-none-any.whl", hash = "sha256:41aaa7594c7d141b46eed0a58dcd75d20edcc809fdd2c931ecbb4957dc98a892", size = 755132, upload-time = "2025-07-09T14:08:25.533Z" },
]
[[package]]
@@ -420,16 +576,16 @@ wheels = [
[[package]]
name = "pygments"
-version = "2.19.1"
+version = "2.19.2"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581, upload-time = "2025-01-06T17:26:30.443Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
]
[[package]]
name = "pytest"
-version = "8.4.0"
+version = "8.4.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "iniconfig", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -437,9 +593,72 @@ dependencies = [
{ name = "pluggy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "pygments", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/fb/aa/405082ce2749be5398045152251ac69c0f3578c7077efc53431303af97ce/pytest-8.4.0.tar.gz", hash = "sha256:14d920b48472ea0dbf68e45b96cd1ffda4705f33307dcc86c676c1b5104838a6", size = 1515232, upload-time = "2025-06-02T17:36:30.03Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2f/de/afa024cbe022b1b318a3d224125aa24939e99b4ff6f22e0ba639a2eaee47/pytest-8.4.0-py3-none-any.whl", hash = "sha256:f40f825768ad76c0977cbacdf1fd37c6f7a468e460ea6a0636078f8972d4517e", size = 363797, upload-time = "2025-06-02T17:36:27.859Z" },
+ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" },
+]
+
+[[package]]
+name = "pytest-asyncio"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d0/d4/14f53324cb1a6381bef29d698987625d80052bb33932d8e7cbf9b337b17c/pytest_asyncio-1.0.0.tar.gz", hash = "sha256:d15463d13f4456e1ead2594520216b225a16f781e144f8fdf6c5bb4667c48b3f", size = 46960, upload-time = "2025-05-26T04:54:40.484Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/30/05/ce271016e351fddc8399e546f6e23761967ee09c8c568bbfbecb0c150171/pytest_asyncio-1.0.0-py3-none-any.whl", hash = "sha256:4f024da9f1ef945e680dc68610b52550e36590a67fd31bb3b4943979a1f90ef3", size = 15976, upload-time = "2025-05-26T04:54:39.035Z" },
+]
+
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },
+ { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },
+ { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },
+]
+
+[[package]]
+name = "regex"
+version = "2024.11.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/5f/bd69653fbfb76cf8604468d3b4ec4c403197144c7bfe0e6a5fc9e02a07cb/regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519", size = 399494, upload-time = "2024-11-06T20:12:31.635Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/90/73/bcb0e36614601016552fa9344544a3a2ae1809dc1401b100eab02e772e1f/regex-2024.11.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a6ba92c0bcdf96cbf43a12c717eae4bc98325ca3730f6b130ffa2e3c3c723d84", size = 483525, upload-time = "2024-11-06T20:10:45.19Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/3f/f1a082a46b31e25291d830b369b6b0c5576a6f7fb89d3053a354c24b8a83/regex-2024.11.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:525eab0b789891ac3be914d36893bdf972d483fe66551f79d3e27146191a37d4", size = 288324, upload-time = "2024-11-06T20:10:47.177Z" },
+ { url = "https://files.pythonhosted.org/packages/09/c9/4e68181a4a652fb3ef5099e077faf4fd2a694ea6e0f806a7737aff9e758a/regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:086a27a0b4ca227941700e0b31425e7a28ef1ae8e5e05a33826e17e47fbfdba0", size = 284617, upload-time = "2024-11-06T20:10:49.312Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/fd/37868b75eaf63843165f1d2122ca6cb94bfc0271e4428cf58c0616786dce/regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde01f35767c4a7899b7eb6e823b125a64de314a8ee9791367c9a34d56af18d0", size = 795023, upload-time = "2024-11-06T20:10:51.102Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/7c/d4cd9c528502a3dedb5c13c146e7a7a539a3853dc20209c8e75d9ba9d1b2/regex-2024.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b583904576650166b3d920d2bcce13971f6f9e9a396c673187f49811b2769dc7", size = 833072, upload-time = "2024-11-06T20:10:52.926Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/db/46f563a08f969159c5a0f0e722260568425363bea43bb7ae370becb66a67/regex-2024.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4de13f06a0d54fa0d5ab1b7138bfa0d883220965a29616e3ea61b35d5f5fc7", size = 823130, upload-time = "2024-11-06T20:10:54.828Z" },
+ { url = "https://files.pythonhosted.org/packages/db/60/1eeca2074f5b87df394fccaa432ae3fc06c9c9bfa97c5051aed70e6e00c2/regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cde6e9f2580eb1665965ce9bf17ff4952f34f5b126beb509fee8f4e994f143c", size = 796857, upload-time = "2024-11-06T20:10:56.634Z" },
+ { url = "https://files.pythonhosted.org/packages/10/db/ac718a08fcee981554d2f7bb8402f1faa7e868c1345c16ab1ebec54b0d7b/regex-2024.11.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d7f453dca13f40a02b79636a339c5b62b670141e63efd511d3f8f73fba162b3", size = 784006, upload-time = "2024-11-06T20:10:59.369Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/41/7da3fe70216cea93144bf12da2b87367590bcf07db97604edeea55dac9ad/regex-2024.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59dfe1ed21aea057a65c6b586afd2a945de04fc7db3de0a6e3ed5397ad491b07", size = 781650, upload-time = "2024-11-06T20:11:02.042Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/d5/880921ee4eec393a4752e6ab9f0fe28009435417c3102fc413f3fe81c4e5/regex-2024.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b97c1e0bd37c5cd7902e65f410779d39eeda155800b65fc4d04cc432efa9bc6e", size = 789545, upload-time = "2024-11-06T20:11:03.933Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/96/53770115e507081122beca8899ab7f5ae28ae790bfcc82b5e38976df6a77/regex-2024.11.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f9d1e379028e0fc2ae3654bac3cbbef81bf3fd571272a42d56c24007979bafb6", size = 853045, upload-time = "2024-11-06T20:11:06.497Z" },
+ { url = "https://files.pythonhosted.org/packages/31/d3/1372add5251cc2d44b451bd94f43b2ec78e15a6e82bff6a290ef9fd8f00a/regex-2024.11.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13291b39131e2d002a7940fb176e120bec5145f3aeb7621be6534e46251912c4", size = 860182, upload-time = "2024-11-06T20:11:09.06Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/e3/c446a64984ea9f69982ba1a69d4658d5014bc7a0ea468a07e1a1265db6e2/regex-2024.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f51f88c126370dcec4908576c5a627220da6c09d0bff31cfa89f2523843316d", size = 787733, upload-time = "2024-11-06T20:11:11.256Z" },
+]
+
+[[package]]
+name = "requests"
+version = "2.32.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "charset-normalizer", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "idna", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" },
]
[[package]]
@@ -457,26 +676,52 @@ wheels = [
[[package]]
name = "ruff"
-version = "0.11.13"
+version = "0.12.2"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ed/da/9c6f995903b4d9474b39da91d2d626659af3ff1eeb43e9ae7c119349dba6/ruff-0.11.13.tar.gz", hash = "sha256:26fa247dc68d1d4e72c179e08889a25ac0c7ba4d78aecfc835d49cbfd60bf514", size = 4282054, upload-time = "2025-06-05T21:00:15.721Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6c/3d/d9a195676f25d00dbfcf3cf95fdd4c685c497fcfa7e862a44ac5e4e96480/ruff-0.12.2.tar.gz", hash = "sha256:d7b4f55cd6f325cb7621244f19c873c565a08aff5a4ba9c69aa7355f3f7afd3e", size = 4432239, upload-time = "2025-07-03T16:40:19.566Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7d/ce/a11d381192966e0b4290842cc8d4fac7dc9214ddf627c11c1afff87da29b/ruff-0.11.13-py3-none-linux_armv6l.whl", hash = "sha256:4bdfbf1240533f40042ec00c9e09a3aade6f8c10b6414cf11b519488d2635d46", size = 10292516, upload-time = "2025-06-05T20:59:32.944Z" },
- { url = "https://files.pythonhosted.org/packages/78/db/87c3b59b0d4e753e40b6a3b4a2642dfd1dcaefbff121ddc64d6c8b47ba00/ruff-0.11.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aef9c9ed1b5ca28bb15c7eac83b8670cf3b20b478195bd49c8d756ba0a36cf48", size = 11106083, upload-time = "2025-06-05T20:59:37.03Z" },
- { url = "https://files.pythonhosted.org/packages/77/79/d8cec175856ff810a19825d09ce700265f905c643c69f45d2b737e4a470a/ruff-0.11.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53b15a9dfdce029c842e9a5aebc3855e9ab7771395979ff85b7c1dedb53ddc2b", size = 10436024, upload-time = "2025-06-05T20:59:39.741Z" },
- { url = "https://files.pythonhosted.org/packages/8b/5b/f6d94f2980fa1ee854b41568368a2e1252681b9238ab2895e133d303538f/ruff-0.11.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab153241400789138d13f362c43f7edecc0edfffce2afa6a68434000ecd8f69a", size = 10646324, upload-time = "2025-06-05T20:59:42.185Z" },
- { url = "https://files.pythonhosted.org/packages/6c/9c/b4c2acf24ea4426016d511dfdc787f4ce1ceb835f3c5fbdbcb32b1c63bda/ruff-0.11.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c51f93029d54a910d3d24f7dd0bb909e31b6cd989a5e4ac513f4eb41629f0dc", size = 10174416, upload-time = "2025-06-05T20:59:44.319Z" },
- { url = "https://files.pythonhosted.org/packages/f3/10/e2e62f77c65ede8cd032c2ca39c41f48feabedb6e282bfd6073d81bb671d/ruff-0.11.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1808b3ed53e1a777c2ef733aca9051dc9bf7c99b26ece15cb59a0320fbdbd629", size = 11724197, upload-time = "2025-06-05T20:59:46.935Z" },
- { url = "https://files.pythonhosted.org/packages/bb/f0/466fe8469b85c561e081d798c45f8a1d21e0b4a5ef795a1d7f1a9a9ec182/ruff-0.11.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d28ce58b5ecf0f43c1b71edffabe6ed7f245d5336b17805803312ec9bc665933", size = 12511615, upload-time = "2025-06-05T20:59:49.534Z" },
- { url = "https://files.pythonhosted.org/packages/17/0e/cefe778b46dbd0cbcb03a839946c8f80a06f7968eb298aa4d1a4293f3448/ruff-0.11.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55e4bc3a77842da33c16d55b32c6cac1ec5fb0fbec9c8c513bdce76c4f922165", size = 12117080, upload-time = "2025-06-05T20:59:51.654Z" },
- { url = "https://files.pythonhosted.org/packages/5d/2c/caaeda564cbe103bed145ea557cb86795b18651b0f6b3ff6a10e84e5a33f/ruff-0.11.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:633bf2c6f35678c56ec73189ba6fa19ff1c5e4807a78bf60ef487b9dd272cc71", size = 11326315, upload-time = "2025-06-05T20:59:54.469Z" },
- { url = "https://files.pythonhosted.org/packages/75/f0/782e7d681d660eda8c536962920c41309e6dd4ebcea9a2714ed5127d44bd/ruff-0.11.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ffbc82d70424b275b089166310448051afdc6e914fdab90e08df66c43bb5ca9", size = 11555640, upload-time = "2025-06-05T20:59:56.986Z" },
- { url = "https://files.pythonhosted.org/packages/5d/d4/3d580c616316c7f07fb3c99dbecfe01fbaea7b6fd9a82b801e72e5de742a/ruff-0.11.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a9ddd3ec62a9a89578c85842b836e4ac832d4a2e0bfaad3b02243f930ceafcc", size = 10507364, upload-time = "2025-06-05T20:59:59.154Z" },
- { url = "https://files.pythonhosted.org/packages/5a/dc/195e6f17d7b3ea6b12dc4f3e9de575db7983db187c378d44606e5d503319/ruff-0.11.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d237a496e0778d719efb05058c64d28b757c77824e04ffe8796c7436e26712b7", size = 10141462, upload-time = "2025-06-05T21:00:01.481Z" },
- { url = "https://files.pythonhosted.org/packages/f4/8e/39a094af6967faa57ecdeacb91bedfb232474ff8c3d20f16a5514e6b3534/ruff-0.11.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26816a218ca6ef02142343fd24c70f7cd8c5aa6c203bca284407adf675984432", size = 11121028, upload-time = "2025-06-05T21:00:04.06Z" },
- { url = "https://files.pythonhosted.org/packages/5a/c0/b0b508193b0e8a1654ec683ebab18d309861f8bd64e3a2f9648b80d392cb/ruff-0.11.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:51c3f95abd9331dc5b87c47ac7f376db5616041173826dfd556cfe3d4977f492", size = 11602992, upload-time = "2025-06-05T21:00:06.249Z" },
+ { url = "https://files.pythonhosted.org/packages/74/b6/2098d0126d2d3318fd5bec3ad40d06c25d377d95749f7a0c5af17129b3b1/ruff-0.12.2-py3-none-linux_armv6l.whl", hash = "sha256:093ea2b221df1d2b8e7ad92fc6ffdca40a2cb10d8564477a987b44fd4008a7be", size = 10369761, upload-time = "2025-07-03T16:39:38.847Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/4b/5da0142033dbe155dc598cfb99262d8ee2449d76920ea92c4eeb9547c208/ruff-0.12.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:09e4cf27cc10f96b1708100fa851e0daf21767e9709e1649175355280e0d950e", size = 11155659, upload-time = "2025-07-03T16:39:42.294Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/21/967b82550a503d7c5c5c127d11c935344b35e8c521f52915fc858fb3e473/ruff-0.12.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8ae64755b22f4ff85e9c52d1f82644abd0b6b6b6deedceb74bd71f35c24044cc", size = 10537769, upload-time = "2025-07-03T16:39:44.75Z" },
+ { url = "https://files.pythonhosted.org/packages/33/91/00cff7102e2ec71a4890fb7ba1803f2cdb122d82787c7d7cf8041fe8cbc1/ruff-0.12.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eb3a6b2db4d6e2c77e682f0b988d4d61aff06860158fdb413118ca133d57922", size = 10717602, upload-time = "2025-07-03T16:39:47.652Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/eb/928814daec4e1ba9115858adcda44a637fb9010618721937491e4e2283b8/ruff-0.12.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73448de992d05517170fc37169cbca857dfeaeaa8c2b9be494d7bcb0d36c8f4b", size = 10198772, upload-time = "2025-07-03T16:39:49.641Z" },
+ { url = "https://files.pythonhosted.org/packages/50/fa/f15089bc20c40f4f72334f9145dde55ab2b680e51afb3b55422effbf2fb6/ruff-0.12.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8b94317cbc2ae4a2771af641739f933934b03555e51515e6e021c64441532d", size = 11845173, upload-time = "2025-07-03T16:39:52.069Z" },
+ { url = "https://files.pythonhosted.org/packages/43/9f/1f6f98f39f2b9302acc161a4a2187b1e3a97634fe918a8e731e591841cf4/ruff-0.12.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:45fc42c3bf1d30d2008023a0a9a0cfb06bf9835b147f11fe0679f21ae86d34b1", size = 12553002, upload-time = "2025-07-03T16:39:54.551Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/70/08991ac46e38ddd231c8f4fd05ef189b1b94be8883e8c0c146a025c20a19/ruff-0.12.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce48f675c394c37e958bf229fb5c1e843e20945a6d962cf3ea20b7a107dcd9f4", size = 12171330, upload-time = "2025-07-03T16:39:57.55Z" },
+ { url = "https://files.pythonhosted.org/packages/88/a9/5a55266fec474acfd0a1c73285f19dd22461d95a538f29bba02edd07a5d9/ruff-0.12.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793d8859445ea47591272021a81391350205a4af65a9392401f418a95dfb75c9", size = 11774717, upload-time = "2025-07-03T16:39:59.78Z" },
+ { url = "https://files.pythonhosted.org/packages/87/e5/0c270e458fc73c46c0d0f7cf970bb14786e5fdb88c87b5e423a4bd65232b/ruff-0.12.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6932323db80484dda89153da3d8e58164d01d6da86857c79f1961934354992da", size = 11646659, upload-time = "2025-07-03T16:40:01.934Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/b6/45ab96070c9752af37f0be364d849ed70e9ccede07675b0ec4e3ef76b63b/ruff-0.12.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6aa7e623a3a11538108f61e859ebf016c4f14a7e6e4eba1980190cacb57714ce", size = 10604012, upload-time = "2025-07-03T16:40:04.363Z" },
+ { url = "https://files.pythonhosted.org/packages/86/91/26a6e6a424eb147cc7627eebae095cfa0b4b337a7c1c413c447c9ebb72fd/ruff-0.12.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2a4a20aeed74671b2def096bdf2eac610c7d8ffcbf4fb0e627c06947a1d7078d", size = 10176799, upload-time = "2025-07-03T16:40:06.514Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/0c/9f344583465a61c8918a7cda604226e77b2c548daf8ef7c2bfccf2b37200/ruff-0.12.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:71a4c550195612f486c9d1f2b045a600aeba851b298c667807ae933478fcef04", size = 11241507, upload-time = "2025-07-03T16:40:08.708Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/b7/99c34ded8fb5f86c0280278fa89a0066c3760edc326e935ce0b1550d315d/ruff-0.12.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4987b8f4ceadf597c927beee65a5eaf994c6e2b631df963f86d8ad1bdea99342", size = 11717609, upload-time = "2025-07-03T16:40:10.836Z" },
]
+[[package]]
+name = "safetensors"
+version = "0.5.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/71/7e/2d5d6ee7b40c0682315367ec7475693d110f512922d582fef1bd4a63adc3/safetensors-0.5.3.tar.gz", hash = "sha256:b6b0d6ecacec39a4fdd99cc19f4576f5219ce858e6fd8dbe7609df0b8dc56965", size = 67210, upload-time = "2025-02-26T09:15:13.155Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/ae/88f6c49dbd0cc4da0e08610019a3c78a7d390879a919411a410a1876d03a/safetensors-0.5.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd20eb133db8ed15b40110b7c00c6df51655a2998132193de2f75f72d99c7073", size = 436917, upload-time = "2025-02-26T09:15:03.702Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/3b/11f1b4a2f5d2ab7da34ecc062b0bc301f2be024d110a6466726bec8c055c/safetensors-0.5.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:21d01c14ff6c415c485616b8b0bf961c46b3b343ca59110d38d744e577f9cce7", size = 418419, upload-time = "2025-02-26T09:15:01.765Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/9a/add3e6fef267658075c5a41573c26d42d80c935cdc992384dfae435feaef/safetensors-0.5.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11bce6164887cd491ca75c2326a113ba934be596e22b28b1742ce27b1d076467", size = 459493, upload-time = "2025-02-26T09:14:51.812Z" },
+ { url = "https://files.pythonhosted.org/packages/df/5c/bf2cae92222513cc23b3ff85c4a1bb2811a2c3583ac0f8e8d502751de934/safetensors-0.5.3-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4a243be3590bc3301c821da7a18d87224ef35cbd3e5f5727e4e0728b8172411e", size = 472400, upload-time = "2025-02-26T09:14:53.549Z" },
+ { url = "https://files.pythonhosted.org/packages/58/11/7456afb740bd45782d0f4c8e8e1bb9e572f1bf82899fb6ace58af47b4282/safetensors-0.5.3-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8bd84b12b1670a6f8e50f01e28156422a2bc07fb16fc4e98bded13039d688a0d", size = 522891, upload-time = "2025-02-26T09:14:55.717Z" },
+ { url = "https://files.pythonhosted.org/packages/57/3d/fe73a9d2ace487e7285f6e157afee2383bd1ddb911b7cb44a55cf812eae3/safetensors-0.5.3-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:391ac8cab7c829452175f871fcaf414aa1e292b5448bd02620f675a7f3e7abb9", size = 537694, upload-time = "2025-02-26T09:14:57.036Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/f8/dae3421624fcc87a89d42e1898a798bc7ff72c61f38973a65d60df8f124c/safetensors-0.5.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cead1fa41fc54b1e61089fa57452e8834f798cb1dc7a09ba3524f1eb08e0317a", size = 471642, upload-time = "2025-02-26T09:15:00.544Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/20/1fbe16f9b815f6c5a672f5b760951e20e17e43f67f231428f871909a37f6/safetensors-0.5.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d", size = 502241, upload-time = "2025-02-26T09:14:58.303Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/18/8e108846b506487aa4629fe4116b27db65c3dde922de2c8e0cc1133f3f29/safetensors-0.5.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:799021e78287bac619c7b3f3606730a22da4cda27759ddf55d37c8db7511c74b", size = 638001, upload-time = "2025-02-26T09:15:05.79Z" },
+ { url = "https://files.pythonhosted.org/packages/82/5a/c116111d8291af6c8c8a8b40628fe833b9db97d8141c2a82359d14d9e078/safetensors-0.5.3-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df26da01aaac504334644e1b7642fa000bfec820e7cef83aeac4e355e03195ff", size = 734013, upload-time = "2025-02-26T09:15:07.892Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/ff/41fcc4d3b7de837963622e8610d998710705bbde9a8a17221d85e5d0baad/safetensors-0.5.3-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:32c3ef2d7af8b9f52ff685ed0bc43913cdcde135089ae322ee576de93eae5135", size = 670687, upload-time = "2025-02-26T09:15:09.979Z" },
+ { url = "https://files.pythonhosted.org/packages/40/ad/2b113098e69c985a3d8fbda4b902778eae4a35b7d5188859b4a63d30c161/safetensors-0.5.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:37f1521be045e56fc2b54c606d4455573e717b2d887c579ee1dbba5f868ece04", size = 643147, upload-time = "2025-02-26T09:15:11.185Z" },
+]
+
+[[package]]
+name = "sentencepiece"
+version = "0.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c9/d2/b9c7ca067c26d8ff085d252c89b5f69609ca93fb85a00ede95f4857865d4/sentencepiece-0.2.0.tar.gz", hash = "sha256:a52c19171daaf2e697dc6cbe67684e0fa341b1248966f6aebb541de654d15843", size = 2632106, upload-time = "2024-02-19T17:06:47.428Z" }
+
[[package]]
name = "sniffio"
version = "1.3.1"
@@ -486,6 +731,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
]
+[[package]]
+name = "tokenizers"
+version = "0.21.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ab/2d/b0fce2b8201635f60e8c95990080f58461cc9ca3d5026de2e900f38a7f21/tokenizers-0.21.2.tar.gz", hash = "sha256:fdc7cffde3e2113ba0e6cc7318c40e3438a4d74bbc62bf04bcc63bdfb082ac77", size = 351545, upload-time = "2025-06-24T10:24:52.449Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1d/cc/2936e2d45ceb130a21d929743f1e9897514691bec123203e10837972296f/tokenizers-0.21.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:342b5dfb75009f2255ab8dec0041287260fed5ce00c323eb6bab639066fef8ec", size = 2875206, upload-time = "2025-06-24T10:24:42.755Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/e6/33f41f2cc7861faeba8988e7a77601407bf1d9d28fc79c5903f8f77df587/tokenizers-0.21.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:126df3205d6f3a93fea80c7a8a266a78c1bd8dd2fe043386bafdd7736a23e45f", size = 2732655, upload-time = "2025-06-24T10:24:41.56Z" },
+ { url = "https://files.pythonhosted.org/packages/33/2b/1791eb329c07122a75b01035b1a3aa22ad139f3ce0ece1b059b506d9d9de/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a32cd81be21168bd0d6a0f0962d60177c447a1aa1b1e48fa6ec9fc728ee0b12", size = 3019202, upload-time = "2025-06-24T10:24:31.791Z" },
+ { url = "https://files.pythonhosted.org/packages/05/15/fd2d8104faa9f86ac68748e6f7ece0b5eb7983c7efc3a2c197cb98c99030/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8bd8999538c405133c2ab999b83b17c08b7fc1b48c1ada2469964605a709ef91", size = 2934539, upload-time = "2025-06-24T10:24:34.567Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/2e/53e8fd053e1f3ffbe579ca5f9546f35ac67cf0039ed357ad7ec57f5f5af0/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e9944e61239b083a41cf8fc42802f855e1dca0f499196df37a8ce219abac6eb", size = 3248665, upload-time = "2025-06-24T10:24:39.024Z" },
+ { url = "https://files.pythonhosted.org/packages/00/15/79713359f4037aa8f4d1f06ffca35312ac83629da062670e8830917e2153/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:514cd43045c5d546f01142ff9c79a96ea69e4b5cda09e3027708cb2e6d5762ab", size = 3451305, upload-time = "2025-06-24T10:24:36.133Z" },
+ { url = "https://files.pythonhosted.org/packages/38/5f/959f3a8756fc9396aeb704292777b84f02a5c6f25c3fc3ba7530db5feb2c/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1b9405822527ec1e0f7d8d2fdb287a5730c3a6518189c968254a8441b21faae", size = 3214757, upload-time = "2025-06-24T10:24:37.784Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/74/f41a432a0733f61f3d21b288de6dfa78f7acff309c6f0f323b2833e9189f/tokenizers-0.21.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed9a4d51c395103ad24f8e7eb976811c57fbec2af9f133df471afcd922e5020", size = 3121887, upload-time = "2025-06-24T10:24:40.293Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/6a/bc220a11a17e5d07b0dfb3b5c628621d4dcc084bccd27cfaead659963016/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c41862df3d873665ec78b6be36fcc30a26e3d4902e9dd8608ed61d49a48bc19", size = 9091965, upload-time = "2025-06-24T10:24:44.431Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/bd/ac386d79c4ef20dc6f39c4706640c24823dca7ebb6f703bfe6b5f0292d88/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:ed21dc7e624e4220e21758b2e62893be7101453525e3d23264081c9ef9a6d00d", size = 9053372, upload-time = "2025-06-24T10:24:46.455Z" },
+ { url = "https://files.pythonhosted.org/packages/63/7b/5440bf203b2a5358f074408f7f9c42884849cd9972879e10ee6b7a8c3b3d/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:0e73770507e65a0e0e2a1affd6b03c36e3bc4377bd10c9ccf51a82c77c0fe365", size = 9298632, upload-time = "2025-06-24T10:24:48.446Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/d2/faa1acac3f96a7427866e94ed4289949b2524f0c1878512516567d80563c/tokenizers-0.21.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:106746e8aa9014a12109e58d540ad5465b4c183768ea96c03cbc24c44d329958", size = 9470074, upload-time = "2025-06-24T10:24:50.378Z" },
+]
+
[[package]]
name = "tqdm"
version = "4.67.1"
@@ -495,22 +763,49 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" },
]
+[[package]]
+name = "transformers"
+version = "4.53.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "safetensors", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "tokenizers", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/2c/68a0024c311db41bb92d4ec17d22e90b7406a4d28aa18d87662f2bbebcd9/transformers-4.53.1.tar.gz", hash = "sha256:da5a9f66ad480bc2a7f75bc32eaf735fd20ac56af4325ca4ce994021ceb37710", size = 9192189, upload-time = "2025-07-04T08:28:40.571Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8d/10/8cef2288810a3210659eb3a20711e8387cc35a881a7762ae387806e2d651/transformers-4.53.1-py3-none-any.whl", hash = "sha256:c84f3c3e41c71fdf2c60c8a893e1cd31191b0cb463385f4c276302d2052d837b", size = 10825681, upload-time = "2025-07-04T08:28:37.318Z" },
+]
+
+[package.optional-dependencies]
+sentencepiece = [
+ { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "sentencepiece", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+
[[package]]
name = "types-protobuf"
-version = "6.30.2.20250516"
+version = "6.30.2.20250703"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/5cf088aaa3927d1cc39910f60f220f5ff573ab1a6485b2836e8b26beb58c/types_protobuf-6.30.2.20250516.tar.gz", hash = "sha256:aecd1881770a9bb225ede66872ef7f0da4505edd0b193108edd9892e48d49a41", size = 62254, upload-time = "2025-05-16T03:06:50.794Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/54/d63ce1eee8e93c4d710bbe2c663ec68e3672cf4f2fca26eecd20981c0c5d/types_protobuf-6.30.2.20250703.tar.gz", hash = "sha256:609a974754bbb71fa178fc641f51050395e8e1849f49d0420a6281ed8d1ddf46", size = 62300, upload-time = "2025-07-03T03:14:05.74Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c0/66/06a9c161f5dd5deb4f5c016ba29106a8f1903eb9a1ba77d407dd6588fecb/types_protobuf-6.30.2.20250516-py3-none-any.whl", hash = "sha256:8c226d05b5e8b2623111765fa32d6e648bbc24832b4c2fddf0fa340ba5d5b722", size = 76480, upload-time = "2025-05-16T03:06:49.444Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/2b/5d0377c3d6e0f49d4847ad2c40629593fee4a5c9ec56eba26a15c708fbc0/types_protobuf-6.30.2.20250703-py3-none-any.whl", hash = "sha256:fa5aff9036e9ef432d703abbdd801b436a249b6802e4df5ef74513e272434e57", size = 76489, upload-time = "2025-07-03T03:14:04.453Z" },
]
[[package]]
name = "typing-extensions"
-version = "4.14.0"
+version = "4.14.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" },
]
[[package]]
@@ -524,3 +819,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7
wheels = [
{ url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" },
]
+
+[[package]]
+name = "urllib3"
+version = "2.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
+]
diff --git a/worker/pyproject.toml b/worker/pyproject.toml
index 81f07f21..f1f4871a 100644
--- a/worker/pyproject.toml
+++ b/worker/pyproject.toml
@@ -4,12 +4,19 @@ version = "0.1.0"
description = "Worker for the Exo project"
readme = "README.md"
requires-python = ">=3.13"
-dependencies = ["exo-shared"]
+dependencies = [
+ "exo-shared",
+ "mlx>=0.26.1",
+ "mlx-lm>=0.25.3",
+]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
+[tool.hatch.metadata]
+allow-direct-references = true
+
[tool.hatch.build]
clean = true
@@ -21,4 +28,4 @@ exclude = ["*.md", "pyproject.toml"]
[tool.hatch.build.targets.sdist]
packages = []
include = ["*"]
-exclude = ["*.md", "pyproject.toml"]
\ No newline at end of file
+exclude = ["*.md", "pyproject.toml"]
diff --git a/worker/runner/communication.py b/worker/runner/communication.py
new file mode 100644
index 00000000..2b5cee12
--- /dev/null
+++ b/worker/runner/communication.py
@@ -0,0 +1,75 @@
+import asyncio
+import sys
+import traceback
+
+from shared.types.worker.commands_runner import (
+ ErrorResponse,
+ PrintResponse,
+ RunnerMessage,
+ RunnerMessageTypeAdapter,
+ RunnerResponse,
+ RunnerResponseType,
+ RunnerResponseTypeAdapter,
+)
+
+### Utils - MESSAGE TO RUNNER
+
+async def supervisor_write_message(proc: asyncio.subprocess.Process, message: RunnerMessage) -> None:
+ assert proc.stdin is not None, "proc.stdin should not be None when created with stdin=PIPE"
+
+ encoded: bytes = message.model_dump_json().encode('utf-8') + b'\n'
+ proc.stdin.write(encoded)
+ await proc.stdin.drain()
+
+async def runner_read_message() -> RunnerMessage:
+ loop = asyncio.get_running_loop()
+
+ line: bytes = await loop.run_in_executor(None, sys.stdin.buffer.readline)
+ if not line:
+ raise EOFError("No more data to read")
+ line = line.strip()
+
+ try:
+ return RunnerMessageTypeAdapter.validate_json(line)
+ except Exception as e:
+ raise ValueError(f"Error validating message: {line}") from e
+
+### Utils - RESPONSE FROM RUNNER
+
+def runner_write_response(obj: RunnerResponse) -> None:
+ encoded: bytes = obj.model_dump_json().encode('utf-8') + b'\n'
+ _ = sys.stdout.buffer.write(encoded)
+ _ = sys.stdout.buffer.flush()
+
+async def supervisor_read_response(proc: asyncio.subprocess.Process) -> RunnerResponse | None:
+ assert proc.stdout is not None, "proc.stdout should not be None when created with stdout=PIPE"
+ line_bytes: bytes = await asyncio.wait_for(proc.stdout.readline(), timeout=10)
+ line: str = line_bytes.decode('utf-8').strip()
+
+ if not line:
+ raise EOFError("No more data to read")
+
+ try:
+ return RunnerResponseTypeAdapter.validate_json(line)
+ except Exception as err:
+ raise ValueError(f"Error validating response: {line}") from err
+
+
+### Utils - Runner Prints
+
+def runner_print(text: str) -> None:
+ obj = PrintResponse(
+ type=RunnerResponseType.PrintResponse,
+ text=text,
+ )
+
+ runner_write_response(obj)
+
+def runner_write_error(error: Exception) -> None:
+ error_response: ErrorResponse = ErrorResponse(
+ type=RunnerResponseType.ErrorResponse,
+ error_type=type(error).__name__,
+ error_message=str(error),
+ traceback=traceback.format_exc(),
+ )
+ runner_write_response(error_response)
\ No newline at end of file
diff --git a/worker/runner/conftest.py b/worker/runner/conftest.py
new file mode 100644
index 00000000..57c5d8f1
--- /dev/null
+++ b/worker/runner/conftest.py
@@ -0,0 +1,107 @@
+import uuid
+from pathlib import Path
+from typing import Callable, cast
+
+import pytest
+
+from shared.types.models.common import ModelId
+from shared.types.tasks.common import (
+ ChatCompletionMessage,
+ ChatCompletionParams,
+ ChatCompletionStreamingTask,
+ PendingTaskStatus,
+ Task,
+ TaskArtifact,
+ TaskId,
+ TaskState,
+ TaskStatusIncompleteType,
+ TaskStatusType,
+ TaskType,
+)
+from shared.types.worker.common import InstanceId
+from shared.types.worker.mlx import Host
+from shared.types.worker.shards import PipelineShardMeta
+
+
+# Concrete TaskArtifact implementation for pending streaming tasks
+class PendingStreamingTaskArtifact(TaskArtifact[TaskType.ChatCompletionStreaming, TaskStatusIncompleteType.Pending]):
+ pass
+
+@pytest.fixture
+def pipeline_shard_meta():
+ def _pipeline_shard_meta(num_nodes: int = 1, device_rank: int = 0) -> PipelineShardMeta:
+ total_layers = 16
+ layers_per_node = total_layers // num_nodes
+ start_layer = device_rank * layers_per_node
+ end_layer = start_layer + layers_per_node if device_rank < num_nodes - 1 else total_layers
+
+ return PipelineShardMeta(
+ device_rank=device_rank,
+ model_id=ModelId(uuid=uuid.uuid4()),
+ model_path=Path("~/.exo/models/mlx-community--Llama-3.2-1B-Instruct-4bit/").expanduser(),
+ start_layer=start_layer,
+ end_layer=end_layer,
+ world_size=num_nodes,
+ )
+ return _pipeline_shard_meta
+
+@pytest.fixture
+def hosts():
+ def _hosts(count: int, offset: int = 0) -> list[Host]:
+ return [
+ Host(
+ host="127.0.0.1",
+ port=5000 + offset + i,
+ )
+ for i in range(count)
+ ]
+ return _hosts
+
+@pytest.fixture
+def hosts_one(hosts: Callable[[int], list[Host]]):
+ return hosts(1)
+
+@pytest.fixture
+def hosts_two(hosts: Callable[[int], list[Host]]):
+ return hosts(2)
+
+@pytest.fixture
+def user_message():
+ """Override this fixture in tests to customize the message"""
+ return "Hello, how are you?"
+
+@pytest.fixture
+def chat_completion_params(user_message: str):
+ """Creates ChatCompletionParams with the given message"""
+ return ChatCompletionParams(
+ model="gpt-4",
+ messages=[
+ ChatCompletionMessage(
+ role="user",
+ content=user_message
+ )
+ ],
+ stream=True
+ )
+
+@pytest.fixture
+def chat_completion_streaming_task_data(chat_completion_params: ChatCompletionParams):
+ """Creates ChatCompletionStreamingTask from params"""
+ return ChatCompletionStreamingTask(
+ task_data=chat_completion_params
+ )
+
+@pytest.fixture
+def streaming_task(chat_completion_streaming_task_data: ChatCompletionStreamingTask) -> Task[TaskType, TaskStatusType]:
+ """Creates the final Task object"""
+ task = Task(
+ task_id=TaskId(),
+ task_type=TaskType.ChatCompletionStreaming,
+ task_data=chat_completion_streaming_task_data,
+ task_state=TaskState(
+ task_status=PendingTaskStatus(),
+ task_artifact=PendingStreamingTaskArtifact(),
+ ),
+ on_instance=InstanceId(),
+ )
+ return cast(Task[TaskType, TaskStatusType], task)
diff --git a/worker/runner/runner.py b/worker/runner/runner.py
new file mode 100644
index 00000000..b7a7f852
--- /dev/null
+++ b/worker/runner/runner.py
@@ -0,0 +1,151 @@
+import asyncio
+import concurrent.futures
+from asyncio.events import AbstractEventLoop
+from collections.abc import AsyncGenerator
+from concurrent.futures.thread import ThreadPoolExecutor
+from functools import partial
+from typing import Callable, cast
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx_lm.generate import stream_generate # type: ignore
+from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+from shared.mlx.utils_mlx import apply_chat_template, initialize_mlx
+from shared.openai import FinishReason
+from shared.types.tasks.common import (
+ TaskData,
+)
+from shared.types.worker.commands_runner import (
+ ChatTaskMessage,
+ ExitMessage,
+ FinishedResponse,
+ GenerationResponse,
+ RunnerMessage,
+ SetupMessage,
+)
+from shared.types.worker.mlx import Host
+from shared.types.worker.shards import ShardMeta
+from shared.utils import ensure_type
+from worker.runner.communication import (
+ runner_print,
+ runner_read_message,
+ runner_write_error,
+ runner_write_response,
+)
+
+
+async def _mlx_generate(
+ mlx_executor: concurrent.futures.ThreadPoolExecutor,
+ model: nn.Module,
+ tokenizer: TokenizerWrapper,
+ sampler: Callable[[mx.array], mx.array],
+ task: TaskData,
+) -> AsyncGenerator[GenerationResponse]:
+ loop = asyncio.get_running_loop()
+ queue: asyncio.Queue[GenerationResponse | Exception | object] = asyncio.Queue()
+ sentinel = object()
+
+ def _generate_tokens(prompt: str, max_tokens: int) -> None:
+ try:
+ for generation_response in stream_generate(
+ model=model,
+ tokenizer=tokenizer,
+ prompt=prompt,
+ max_tokens=max_tokens,
+ sampler=sampler,
+ ):
+ response = GenerationResponse(
+ text=generation_response.text,
+ token=generation_response.token,
+ finish_reason=cast(FinishReason | None, generation_response.finish_reason), # has to be considered as a FinishReason instead of a str.
+ )
+ _ = loop.call_soon_threadsafe(queue.put_nowait, response)
+ except Exception as e:
+ _ = loop.call_soon_threadsafe(queue.put_nowait, e)
+ finally:
+ _ = loop.call_soon_threadsafe(queue.put_nowait, sentinel)
+
+ # Currently we support chat-completion tasks only.
+ task_data = task.task_data
+
+ runner_print(f"task_data: {task_data}")
+
+ prompt = await apply_chat_template(
+ mlx_executor=mlx_executor,
+ tokenizer=tokenizer,
+ chat_task=task_data,
+ )
+
+ max_tokens = task_data.max_tokens or 100
+ generation_fn = partial(_generate_tokens, prompt, max_tokens)
+
+ future = loop.run_in_executor(mlx_executor, generation_fn)
+
+ while True:
+ item = await queue.get()
+ queue.task_done()
+
+ if item is sentinel:
+ break
+
+ if isinstance(item, Exception):
+ raise item
+
+ assert isinstance(item, GenerationResponse) # constrain datatype
+ yield item
+
+ assert future.done()
+
+async def main():
+ try:
+ runner_print('hello from the runner')
+
+ # Get setup info from worker
+ init_message: RunnerMessage = await runner_read_message()
+ setup_message: SetupMessage = ensure_type(init_message, SetupMessage)
+ model_shard_meta: ShardMeta = setup_message.model_shard_meta
+ hosts: list[Host] = setup_message.hosts
+
+ mlx_executor: ThreadPoolExecutor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+ loop: AbstractEventLoop = asyncio.get_running_loop()
+
+ runner_print(f'got here; {model_shard_meta.model_path}')
+
+ model, tokenizer, sampler = await loop.run_in_executor(
+ mlx_executor,
+ partial(initialize_mlx, model_shard_meta=model_shard_meta, hosts=hosts),
+ )
+
+ while True:
+ message: RunnerMessage = await runner_read_message()
+ match message:
+ case ChatTaskMessage(task=task_data):
+ runner_print(f"received chat request: {task_data}")
+
+ # Ensure we have a chat-completion task subtype
+ messages = task_data.task_data.messages
+ messages_dicts = [msg.model_dump() for msg in messages]
+ runner_print(f"messages_dicts RUNNER: {messages_dicts}")
+
+ # Generate responses using the actual MLX generation
+ async for generation_response in _mlx_generate(
+ mlx_executor=mlx_executor,
+ model=model,
+ tokenizer=tokenizer,
+ sampler=sampler,
+ task=task_data,
+ ):
+ runner_write_response(generation_response)
+
+ runner_write_response(FinishedResponse())
+ case ExitMessage():
+ break
+ case _:
+ raise ValueError(f"Unknown message: {message}")
+
+ except Exception as e:
+ runner_write_error(e)
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/worker/runner/runner_supervisor.py b/worker/runner/runner_supervisor.py
new file mode 100644
index 00000000..2b85d82b
--- /dev/null
+++ b/worker/runner/runner_supervisor.py
@@ -0,0 +1,175 @@
+import asyncio
+import contextlib
+import sys
+from collections.abc import AsyncGenerator
+from typing import Callable
+
+from shared.types.events.chunks import GenerationChunk, TokenChunk, TokenChunkData
+from shared.types.tasks.common import Task, TaskStatusType, TaskType
+from shared.types.worker.commands_runner import (
+ ChatTaskMessage,
+ ErrorResponse,
+ ExitMessage,
+ FinishedResponse,
+ GenerationResponse,
+ PrintResponse,
+ RunnerResponse,
+ SetupMessage,
+)
+from shared.types.worker.mlx import Host
+from shared.types.worker.runners import RunnerError
+from shared.types.worker.shards import ShardMeta
+from worker.runner.communication import (
+ supervisor_read_response,
+ supervisor_write_message,
+)
+from worker.runner.utils import get_runner_command
+
+
+class RunnerSupervisor:
+ """
+ RunnerSupervisor manages the lifecycle of a runner subprocess for model inference.
+ Use the class method `create` to properly initialize an instance.
+ """
+
+ def __init__(
+ self,
+ model_shard_meta: ShardMeta,
+ hosts: list[Host],
+ runner_process: asyncio.subprocess.Process,
+ ):
+ """Private constructor. Use RunnerSupervisor.create() instead."""
+ self.model_shard_meta: ShardMeta = model_shard_meta
+ self.hosts: list[Host] = hosts
+ self.runner_process: asyncio.subprocess.Process = runner_process
+ self.running: bool = True
+
+ self.running_task: asyncio.Task[None] = asyncio.create_task(self._watch_runner())
+
+ @classmethod
+ async def create(
+ cls,
+ model_shard_meta: ShardMeta,
+ hosts: list[Host],
+ ) -> "RunnerSupervisor":
+ """
+ Create and initialize a RunnerSupervisor instance.
+ The .create() classmethod pattern is used to ensure the constructor is asynchronous.
+ """
+ cmd: list[str] = get_runner_command()
+
+ runner_process: asyncio.subprocess.Process = await asyncio.create_subprocess_exec(
+ *cmd,
+ stdin=asyncio.subprocess.PIPE,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=sys.stderr,
+ )
+
+ await supervisor_write_message(
+ runner_process,
+ SetupMessage(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts,
+ ),
+ )
+
+ return cls(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts,
+ runner_process=runner_process,
+ )
+
+ async def astop(self) -> None:
+ async def terminate() -> None:
+ self.runner_process.terminate()
+ _ = await self.runner_process.wait()
+
+ if not self.healthy:
+ print("Runner process is not healthy, killing...")
+ await terminate()
+
+ if self.runner_process.stdout is not None:
+ while True:
+ try:
+ line = await asyncio.wait_for(self.runner_process.stdout.readline(), timeout=0.01)
+ if not line:
+ break
+ print(f"Remaining stdout: {line.decode('utf-8').strip()}")
+ except asyncio.TimeoutError:
+ break
+
+ try:
+ # Give the process a moment to exit gracefully
+ await supervisor_write_message(proc=self.runner_process, message=ExitMessage())
+ _ = await asyncio.wait_for(self.runner_process.wait(), timeout=0.1)
+ except asyncio.TimeoutError:
+ print("Runner process did not terminate, killing...")
+ await terminate()
+
+ self.running = False
+
+ async def _watch_runner(self) -> None:
+ _ = await self.runner_process.wait()
+ self.running = False
+
+ def __del__(self) -> None:
+ if not self.running:
+ print('Warning: RunnerSupervisor was not stopped cleanly before garbage collection. Force killing process.')
+
+ with contextlib.suppress(ProcessLookupError):
+ self.runner_process.kill()
+
+ @property
+ def healthy(self) -> bool:
+ return (
+ self.running
+ and self.runner_process.returncode is None
+ and self.runner_process.stdin is not None
+ and not self.runner_process.stdin.is_closing()
+ and self.runner_process.stdout is not None
+ )
+
+ async def stream_response(
+ self,
+ task: Task[TaskType, TaskStatusType],
+ request_started_callback: Callable[[], None] | None = None,
+ ) -> AsyncGenerator[GenerationChunk]:
+ """
+ Streams a chat request from the model.
+ The request is pushed to the runner, and if the shard is the terminal shard, the response is streamed back to the worker.
+ request_started_callback is called once the request is pushed to the runner, used to publish InferencePrepareCompleted and InferenceTriggerCompleted events.
+ """
+ if not self.healthy:
+ raise RuntimeError("Runner process was found to be dead")
+
+ await supervisor_write_message(
+ proc=self.runner_process,
+ message=ChatTaskMessage(
+ task=task.task_data,
+ ),
+ )
+
+ while True:
+ line: RunnerResponse | None = await supervisor_read_response(self.runner_process)
+ if line is None:
+ continue
+ else:
+ match line:
+ case GenerationResponse(text=text, token=token, finish_reason=finish_reason):
+ yield TokenChunk(
+ task_id=task.task_id,
+ idx=token,
+ model=self.model_shard_meta.model_id,
+ chunk_data=TokenChunkData(
+ text=text,
+ token_id=token,
+ finish_reason=finish_reason,
+ ),
+ )
+ case FinishedResponse():
+ break
+ case PrintResponse(text=text):
+ print(f'runner printed: {text}')
+ case ErrorResponse(error_type=error_type, error_message=error_message, traceback=traceback):
+ await self.astop()
+ raise RunnerError(error_type, error_message, traceback or "")
diff --git a/worker/runner/test_serdes.py b/worker/runner/test_serdes.py
new file mode 100644
index 00000000..fe85da0e
--- /dev/null
+++ b/worker/runner/test_serdes.py
@@ -0,0 +1,33 @@
+from typing import Callable, Literal, TypeVar
+
+from pydantic import BaseModel, TypeAdapter
+
+from shared.types.tasks.common import Task, TaskStatusIncompleteType, TaskType
+from shared.types.worker.commands_runner import (
+ ChatTaskMessage,
+ RunnerMessageTypeAdapter,
+ SetupMessage,
+)
+from shared.types.worker.mlx import Host
+from shared.types.worker.shards import PipelineShardMeta
+
+T = TypeVar('T', bound=BaseModel)
+
+def assert_equal_serdes(obj: T, typeadapter: TypeAdapter[T]):
+ encoded: bytes = obj.model_dump_json().encode('utf-8') + b'\n'
+ decoded: T = typeadapter.validate_json(encoded)
+
+ assert decoded == obj, f"Decoded: {decoded} != \nOriginal: {obj}. \n binary encoded: {encoded}"
+
+def test_supervisor_setup_message_serdes(pipeline_shard_meta: Callable[..., PipelineShardMeta], hosts: Callable[..., list[Host]]):
+ setup_message = SetupMessage(
+ model_shard_meta=pipeline_shard_meta(1, 0),
+ hosts=hosts(1),
+ )
+ assert_equal_serdes(setup_message, RunnerMessageTypeAdapter)
+
+def test_supervisor_task_message_serdes(streaming_task: Task[TaskType, Literal[TaskStatusIncompleteType.Pending]]):
+ task_message = ChatTaskMessage(
+ task=streaming_task.task_data,
+ )
+ assert_equal_serdes(task_message, RunnerMessageTypeAdapter)
diff --git a/worker/runner/test_supervisor.py b/worker/runner/test_supervisor.py
new file mode 100644
index 00000000..46a93883
--- /dev/null
+++ b/worker/runner/test_supervisor.py
@@ -0,0 +1,190 @@
+import asyncio
+from typing import Callable
+
+import pytest
+
+from shared.openai import FinishReason
+from shared.types.events.chunks import TokenChunk
+from shared.types.tasks.common import Task, TaskStatusType, TaskType
+from shared.types.worker.mlx import Host
+from shared.types.worker.shards import PipelineShardMeta
+from worker.runner.runner_supervisor import RunnerSupervisor
+
+
+@pytest.fixture
+def user_message():
+ """Override the default message to ask about France's capital"""
+ return "What is the capital of France?"
+
+
+@pytest.mark.asyncio
+async def test_supervisor_single_node_response(
+ pipeline_shard_meta: Callable[..., PipelineShardMeta],
+ hosts: Callable[..., list[Host]],
+ streaming_task: Task[TaskType, TaskStatusType],
+):
+ """Test that asking for the capital of France returns 'Paris' in the response"""
+ model_shard_meta = pipeline_shard_meta(1, 0)
+
+ supervisor = await RunnerSupervisor.create(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts(1, offset=10),
+ )
+
+ try:
+ full_response = ""
+ stop_reason: FinishReason | None = None
+
+ async for chunk in supervisor.stream_response(task=streaming_task):
+ if isinstance(chunk, TokenChunk):
+ full_response += chunk.chunk_data.text
+ if chunk.chunk_data.finish_reason:
+ stop_reason = chunk.chunk_data.finish_reason
+
+ # Case-insensitive check for Paris in the response
+ assert "paris" in full_response.lower(), f"Expected 'Paris' in response, but got: {full_response}"
+ assert stop_reason == 'stop'
+
+ finally:
+ await supervisor.astop()
+
+@pytest.mark.asyncio
+async def test_supervisor_two_node_response(
+ pipeline_shard_meta: Callable[..., PipelineShardMeta],
+ hosts: Callable[..., list[Host]],
+ streaming_task: Task[TaskType, TaskStatusType],
+):
+ """Test that asking for the capital of France returns 'Paris' in the response"""
+ supervisor_0 = await RunnerSupervisor.create(
+ model_shard_meta=pipeline_shard_meta(2, 0),
+ hosts=hosts(2, offset=15),
+ )
+
+ supervisor_1 = await RunnerSupervisor.create(
+ model_shard_meta=pipeline_shard_meta(2, 1),
+ hosts=hosts(2, offset=15),
+ )
+
+ await asyncio.sleep(0.1)
+
+ try:
+ full_response_0 = ""
+ full_response_1 = ""
+
+ async def collect_response_0():
+ nonlocal full_response_0
+ async for chunk in supervisor_0.stream_response(task=streaming_task):
+ if isinstance(chunk, TokenChunk):
+ full_response_0 += chunk.chunk_data.text
+
+ async def collect_response_1():
+ nonlocal full_response_1
+ async for chunk in supervisor_1.stream_response(task=streaming_task):
+ if isinstance(chunk, TokenChunk):
+ full_response_1 += chunk.chunk_data.text
+
+ # Run both stream responses simultaneously
+ _ = await asyncio.gather(collect_response_0(), collect_response_1())
+
+ print(f"full_response_0: {full_response_0}")
+ print(f"full_response_1: {full_response_1}")
+
+ # Case-insensitive check for Paris in both responses
+ assert "paris" in full_response_0.lower(), f"Expected 'Paris' in response, but got: {full_response_0}"
+ assert "paris" in full_response_1.lower(), f"Expected 'Paris' in response, but got: {full_response_1}"
+
+ finally:
+ await supervisor_0.astop()
+ await supervisor_1.astop()
+
+@pytest.mark.asyncio
+async def test_supervisor_early_stopping(
+ pipeline_shard_meta: Callable[..., PipelineShardMeta],
+ hosts: Callable[..., list[Host]],
+ streaming_task: Task[TaskType, TaskStatusType],
+):
+ """Test that asking for the capital of France returns 'Paris' in the response"""
+ model_shard_meta = pipeline_shard_meta(1, 0)
+
+ supervisor = await RunnerSupervisor.create(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts(1, offset=10),
+ )
+
+ max_tokens = 50
+
+ try:
+ streaming_task.task_data.task_data.max_tokens = max_tokens
+ streaming_task.task_data.task_data.messages[0].content = "Please count from 1 to 100"
+
+ full_response = ""
+ count = 0
+ stop_reason: FinishReason | None = None
+
+ async for chunk in supervisor.stream_response(task=streaming_task):
+ if isinstance(chunk, TokenChunk):
+ full_response += chunk.chunk_data.text
+ count += 1
+ if chunk.chunk_data.finish_reason:
+ stop_reason = chunk.chunk_data.finish_reason
+
+ print(f"full_response: {full_response}")
+
+ assert count == max_tokens + 1
+ assert '7' in full_response.lower()
+ assert '99' not in full_response.lower()
+
+ assert stop_reason == 'length'
+
+ finally:
+ await supervisor.astop()
+
+
+@pytest.mark.asyncio
+async def test_supervisor_handles_terminated_runner(
+ pipeline_shard_meta: Callable[..., PipelineShardMeta],
+ hosts: Callable[..., list[Host]],
+ streaming_task: Task[TaskType, TaskStatusType],
+):
+ """Test that the supervisor handles a terminated runner"""
+ model_shard_meta = pipeline_shard_meta(1, 0)
+
+ supervisor = await RunnerSupervisor.create(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts(1, offset=10),
+ )
+
+ # Terminate the runner
+ supervisor.runner_process.terminate()
+ await asyncio.sleep(0.1)
+
+ assert not supervisor.healthy
+ assert supervisor.runner_process.returncode is not None
+
+ del supervisor
+
+
+@pytest.mark.asyncio
+async def test_supervisor_handles_killed_runner(
+ pipeline_shard_meta: Callable[..., PipelineShardMeta],
+ hosts: Callable[..., list[Host]],
+ streaming_task: Task[TaskType, TaskStatusType],
+):
+ """Test that the supervisor handles a killed runner"""
+ model_shard_meta = pipeline_shard_meta(1, 0)
+
+ supervisor = await RunnerSupervisor.create(
+ model_shard_meta=model_shard_meta,
+ hosts=hosts(1, offset=10),
+ )
+
+ assert supervisor.healthy
+
+ # Forcibly kill the runner
+ supervisor.runner_process.kill()
+ await asyncio.sleep(0.1)
+
+ assert not supervisor.healthy
+ assert supervisor.runner_process.returncode is not None
+
+ del supervisor
diff --git a/worker/runner/utils.py b/worker/runner/utils.py
new file mode 100644
index 00000000..0f252633
--- /dev/null
+++ b/worker/runner/utils.py
@@ -0,0 +1,8 @@
+import sys
+
+
+def get_runner_command() -> list[str]:
+ python = sys.executable
+ return [
+ python, '-m', 'worker.runner.runner'
+ ]
\ No newline at end of file
← b0bd9510 Merge Basic Interfaces
·
back to Exo
·
fix: Use Nix-compatible LSP set-up 4e4dbf52 →