default to llama-3-8b and temperature=0 if not provided

5de2ea51f516c63ff3dd74c5e4660701d167c536 · 2024-07-17 17:12:14 -0700 · Alex Cheema

Files touched

M exo/api/chatgpt_api.py

Diff

commit 5de2ea51f516c63ff3dd74c5e4660701d167c536
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Wed Jul 17 17:12:14 2024 -0700

    default to llama-3-8b and temperature=0 if not provided
---
 exo/api/chatgpt_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/exo/api/chatgpt_api.py b/exo/api/chatgpt_api.py
index d5ba24fc..b1968b98 100644
--- a/exo/api/chatgpt_api.py
+++ b/exo/api/chatgpt_api.py
@@ -70,7 +70,7 @@ class ChatGPTAPI:
     async def handle_post(self, request):
         data = await request.json()
         messages = [Message(**msg) for msg in data['messages']]
-        chat_request = ChatCompletionRequest(data['model'], messages, data['temperature'])
+        chat_request = ChatCompletionRequest(data.get('model', 'llama-3-8b'), messages, data.get('temperature', 0.0))
         prompt = " ".join([msg.content for msg in chat_request.messages if msg.role == "user"])
         shard = shard_mappings.get(chat_request.model, {}).get(self.inference_engine_classname)
         if not shard:
@@ -137,7 +137,7 @@ class ChatGPTAPI:
         await runner.setup()
         site = web.TCPSite(runner, host, port)
         await site.start()
-        if DEBUG >= 1: print(f"Starting ChatGPT API server at {host}:{port}")
+        if DEBUG >= 0: print(f"ChatGPT API server started at http://{host}:{port}")
 
 # Usage example
 if __name__ == "__main__":

← 5c3f0e3a faster initial node discovery · back to Exo · chatgpt api repsonse streaming solves #20 8762effa →