formatting

58f535d0b0bb581597267a0efffece7f271913f1 · 2024-09-05 14:23:32 +0100 · Alex Cheema

Files touched

M exo/inference/tinygrad/inference.py

Diff

commit 58f535d0b0bb581597267a0efffece7f271913f1
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Sep 5 14:23:32 2024 +0100

    formatting
---
 exo/inference/tinygrad/inference.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/exo/inference/tinygrad/inference.py b/exo/inference/tinygrad/inference.py
index b60d3e9e..ff2d4daa 100644
--- a/exo/inference/tinygrad/inference.py
+++ b/exo/inference/tinygrad/inference.py
@@ -94,13 +94,7 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
 
   async def _run_inference(self, input_tensor, start_pos):
     with self.model_lock:
-      return await asyncio.get_event_loop().run_in_executor(
-        self.executor,
-        self.model,
-        input_tensor,
-        start_pos,
-        TEMPERATURE
-      )
+      return await asyncio.get_event_loop().run_in_executor(self.executor, self.model, input_tensor, start_pos, TEMPERATURE)
 
   async def ensure_shard(self, shard: Shard):
     if self.shard == shard:
@@ -110,13 +104,7 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
 
     with self.model_lock:
       if self.shard != shard:
-        self.model = await asyncio.get_event_loop().run_in_executor(
-          self.executor,
-          build_transformer,
-          model_path,
-          shard,
-          "8B" if "8b" in shard.model_id.lower() else "70B"
-        )
+        self.model = await asyncio.get_event_loop().run_in_executor(self.executor, build_transformer, model_path, shard, "8B" if "8b" in shard.model_id.lower() else "70B")
         tokenizer_path = str((model_path if model_path.is_dir() else model_path.parent))
         self.tokenizer = await resolve_tokenizer(tokenizer_path)
         self.shard = shard

← 2950373d experiment with tinygrad on its own thread, so it doesnt blo · back to Exo · run mlx inference engine on a single thread too 0ca5c260 →