add redundant temp and top_p to dummy inference engine

832e6052198eecea6d0a01b4ebfa16b36a49bc3c · 2024-11-28 10:36:43 +0400 · Alex Cheema

Files touched

M exo/inference/dummy_inference_engine.py

Diff

commit 832e6052198eecea6d0a01b4ebfa16b36a49bc3c
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Nov 28 10:36:43 2024 +0400

    add redundant temp and top_p to dummy inference engine
---
 exo/inference/dummy_inference_engine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exo/inference/dummy_inference_engine.py b/exo/inference/dummy_inference_engine.py
index 0aaadcdd..09026109 100644
--- a/exo/inference/dummy_inference_engine.py
+++ b/exo/inference/dummy_inference_engine.py
@@ -18,7 +18,7 @@ class DummyInferenceEngine(InferenceEngine):
   async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
     return np.array(self.tokenizer.encode(prompt))
   
-  async def sample(self, x: np.ndarray) -> np.ndarray:
+  async def sample(self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0) -> np.ndarray:
     if x[0] > self.num_generate_dummy_tokens: return np.array([self.tokenizer.eos_token_id])
     return x

← ac321705 removing console log in initial models · back to Exo · better pagination to avoid rate limits on dashboard e0c87113 →