closely match prev impl mlx non blocking

9345684b383ebf7aa63a516f9266cf106e7ad24e · 2024-09-05 17:16:11 +0100 · Alex Cheema
Files touched

M exo/inference/mlx/sharded_inference_engine.py
M exo/inference/tinygrad/inference.py
Diff

commit 9345684b383ebf7aa63a516f9266cf106e7ad24e
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Sep 5 17:16:11 2024 +0100

    closely match prev impl mlx non blocking
---
 exo/inference/mlx/sharded_inference_engine.py | 32 +++++----------------------
 exo/inference/tinygrad/inference.py           |  4 +---
 2 files changed, 7 insertions(+), 29 deletions(-)

diff --git a/exo/inference/mlx/sharded_inference_engine.py b/exo/inference/mlx/sharded_inference_engine.py
index 5fc5d404..5d6378b7 100644
--- a/exo/inference/mlx/sharded_inference_engine.py
+++ b/exo/inference/mlx/sharded_inference_engine.py
@@ -23,21 +23,15 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
       inputs = await loop.run_in_executor(self.executor, self.tokenizer, prompt, image, return_tensors="np")
       pixel_values = mx.array(inputs["pixel_values"])
       input_ids = mx.array(inputs["input_ids"])
-      output_data = np.array(await loop.run_in_executor(self.executor, self.stateful_sharded_model.step, request_id, input_ids, pixel_values))
+      output_data: np.ndarray = np.array(await loop.run_in_executor(self.executor, self.stateful_sharded_model.step, request_id, input_ids, pixel_values))
     else:
       input_ids = await loop.run_in_executor(self.executor, lambda: mx.array(self.tokenizer.encode(prompt)))
-      output_data = np.array(await loop.run_in_executor(self.executor, self.stateful_sharded_model.step, request_id, input_ids))
+      output_data: np.ndarray = np.array(await loop.run_in_executor(self.executor, self.stateful_sharded_model.step, request_id, input_ids))
     return output_data, "", output_data.size == 1 and output_data.item() == self.tokenizer.eos_token_id
 
   async def infer_tensor(self, request_id: str, shard: Shard, input_data: np.ndarray, inference_state: Optional[str] = None) -> (np.ndarray, str, bool):
     await self.ensure_shard(shard)
-    input_tensor = mx.array(input_data)
-    output_data = np.array(await asyncio.get_running_loop().run_in_executor(
-      self.executor,
-      self.stateful_sharded_model.step,
-      request_id,
-      input_tensor
-    ))
+    output_data: np.ndarray = np.array(await asyncio.get_running_loop().run_in_executor(self.executor, self.stateful_sharded_model.step, request_id, mx.array(input_data)))
     return output_data, "", output_data.size == 1 and output_data.item() == self.tokenizer.eos_token_id
 
   async def ensure_shard(self, shard: Shard):
@@ -48,21 +42,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
 
     if self.shard != shard:
       loop = asyncio.get_running_loop()
-
-      # Run load_shard in a separate thread
-      def load_shard_wrapper():
-        return asyncio.run(load_shard(model_path, shard))
-
-      model_shard, self.tokenizer = await loop.run_in_executor(
-        self.executor,
-        load_shard_wrapper
-      )
-
-      # Create StatefulShardedModel in the executor
-      self.stateful_sharded_model = await loop.run_in_executor(
-        self.executor,
-        StatefulShardedModel,
-        shard,
-        model_shard
-      )
+      def load_shard_wrapper(): return asyncio.run(load_shard(model_path, shard))
+      model_shard, self.tokenizer = await loop.run_in_executor(self.executor, load_shard_wrapper)
+      self.stateful_sharded_model = await loop.run_in_executor(self.executor, StatefulShardedModel, shard, model_shard)
       self.shard = shard
diff --git a/exo/inference/tinygrad/inference.py b/exo/inference/tinygrad/inference.py
index ae378d55..63519488 100644
--- a/exo/inference/tinygrad/inference.py
+++ b/exo/inference/tinygrad/inference.py
@@ -64,9 +64,7 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
     n_captured_toks = json.loads(inference_state or "{}").get("n_captured_toks", 0)
 
     toks = await asyncio.get_event_loop().run_in_executor(self.executor, self.tokenizer.encode, prompt)
-    input_tensor = Tensor([toks])
-
-    h = await asyncio.get_event_loop().run_in_executor(self.executor, self.model, input_tensor, start_pos, TEMPERATURE)
+    h = await asyncio.get_event_loop().run_in_executor(self.executor, self.model, Tensor([toks]), start_pos, TEMPERATURE)
 
     if h.shape == (1,):
       start_pos += len(toks)
← d6e661fd match previous impl with np.array in mlx · back to Exo · run realize on the result in tinygrad e616d4e8 →