← back to Exo
add exo run
d154d37ac4b4506f437ead58d0a62f7a9a42ffcb · 2024-12-06 18:59:25 +0000 · Glen
Files touched
A .github/bench.pyM .github/workflows/build_and_test.yml
Diff
commit d154d37ac4b4506f437ead58d0a62f7a9a42ffcb
Author: Glen <glen@192.168.1.214>
Date: Fri Dec 6 18:59:25 2024 +0000
add exo run
---
.github/bench.py | 116 +++++++++++++++++++++++++++++++++++
.github/workflows/build_and_test.yml | 57 +++++++++++++++++
2 files changed, 173 insertions(+)
diff --git a/.github/bench.py b/.github/bench.py
new file mode 100644
index 00000000..b189525e
--- /dev/null
+++ b/.github/bench.py
@@ -0,0 +1,116 @@
+import aiohttp
+import asyncio
+import time
+import json
+import os
+from typing import Dict, Any
+
+
+async def measure_performance(api_endpoint: str, prompt: str) -> Dict[str, Any]:
+ """
+ Measures the performance of an API endpoint by sending a prompt and recording metrics.
+
+ Args:
+ api_endpoint (str): The API endpoint URL.
+ prompt (str): The prompt to send to the API.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing performance metrics or error information.
+ """
+ results: Dict[str, Any] = {}
+ request_payload = {
+ "model": "llama-3.2-3b",
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": 0,
+ "stream": True
+ }
+
+ async with aiohttp.ClientSession() as session:
+ try:
+ start_time = time.time()
+ first_token_time = None
+ total_tokens = 0
+
+ async with session.post(api_endpoint, json=request_payload) as response:
+ if response.status != 200:
+ results["error"] = f"HTTP {response.status}: {response.reason}"
+ return results
+
+ async for raw_line in response.content:
+ line = raw_line.decode('utf-8').strip()
+ if not line or not line.startswith('data: '):
+ continue
+
+ line_content = line[6:] # Remove 'data: ' prefix
+ if line_content == '[DONE]':
+ break
+
+ try:
+ chunk = json.loads(line_content)
+ choice = chunk.get('choices', [{}])[0]
+ content = choice.get('delta', {}).get('content')
+
+ if content:
+ if first_token_time is None:
+ first_token_time = time.time()
+ results["time_to_first_token"] = first_token_time - start_time
+
+ total_tokens += 1
+ except json.JSONDecodeError:
+ # Log or handle malformed JSON if necessary
+ continue
+
+ end_time = time.time()
+ total_time = end_time - start_time
+
+ if total_tokens > 0:
+ results.update({
+ "tokens_per_second": total_tokens / total_time,
+ "total_tokens": total_tokens,
+ "total_time": total_time
+ })
+ else:
+ results["error"] = "No tokens were generated"
+
+ except aiohttp.ClientError as e:
+ results["error"] = f"Client error: {e}"
+ except Exception as e:
+ results["error"] = f"Unexpected error: {e}"
+
+ return results
+
+
+async def main() -> None:
+ api_endpoint = "http://localhost:52415/v1/chat/completions"
+
+ # Define prompts
+ prompt_basic = "this is a ping"
+ prompt_essay = "write an essay about cats"
+
+ # Measure performance for the basic prompt
+ print("Measuring performance for the basic prompt...")
+ results_basic = await measure_performance(api_endpoint, prompt_basic)
+ print("Basic prompt performance metrics:")
+ print(json.dumps(results_basic, indent=4))
+
+ # Measure performance for the essay prompt, which depends on the first measurement
+ print("\nMeasuring performance for the essay prompt...")
+ results = await measure_performance(api_endpoint, prompt_essay)
+
+ # Save metrics from the "universe and everything" prompt
+ metrics_file = os.path.join("artifacts", "benchmark.json")
+ os.makedirs(os.path.dirname(metrics_file), exist_ok=True)
+ try:
+ with open(metrics_file, "w", encoding="utf-8") as f:
+ json.dump(results, f, indent=4)
+ print(f"Performance metrics saved to {metrics_file}")
+ except IOError as e:
+ print(f"Failed to save metrics: {e}")
+
+ # Optionally print the metrics for visibility
+ print("Performance metrics:")
+ print(json.dumps(results, indent=4))
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 53bd9ceb..d840b9fa 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -19,6 +19,63 @@ jobs:
cpu: ['M3', 'M4']
runs-on: ['self-hosted', 'macOS', '${{ matrix.cpu }}']
steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ run: |
+ python -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+
+ - name: Run discovery integration test
+ run: |
+ source env/bin/activate
+ ALL_NODE_IDS=$(for i in $(seq 0 ${{ strategy.job-total }} -1); do echo -n "${GITHUB_JOB}_${i},"; done | sed 's/,$//')
+ MY_NODE_ID="${GITHUB_JOB}_${{ strategy.job-index }}"
+ DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id="${MY_NODE_ID}" --filter-node-ids="${ALL_NODE_IDS}" --chatgpt-api-port 52415 --disable-tui > output1.log 2>&1 &
+ PID1=$!
+ tail -f output1.log &
+ TAIL1=$!
+
+ trap 'kill $TAIL1' EXIT
+
+ for i in {1..100}; do
+ nodes=$(curl http://localhost:52415/topology | jq .nodes | length)
+ if [ "$nodes" -eq "${{ strategy.job-total }}" ]; then
+ break
+ fi
+ sleep 5
+ done
+
+ if ! kill -0 $PID1 2>/dev/null; then
+ echo "Instance (PID $PID1) died unexpectedly. Log output:"
+ cat output1.log
+ exit 1
+ fi
+
+
+
+ kill $PID1 $PID2
+ if grep -q "Peer statuses: {.*'node2': 'is_connected=True, health_check=True" output1.log && \
+ ! grep -q "Failed to connect peers:" output1.log && \
+ grep -q "Peer statuses: {.*'node1': 'is_connected=True, health_check=True" output2.log && \
+ ! grep -q "Failed to connect peers:" output2.log; then
+ echo "Test passed: Both instances discovered each other"
+ exit 0
+ else
+ echo "Test failed: Devices did not discover each other"
+ echo "Output of first instance:"
+ cat output1.log
+ echo "Output of second instance:"
+ cat output2.log
+ exit 1
+ fi
- name: Test
run: |
echo "GITHUB_JOB: ${GITHUB_JOB}, GITHUB_RUN_ID: ${GITHUB_RUN_ID}, GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}, GITHUB_WORKFLOW: ${GITHUB_WORKFLOW}"
\ No newline at end of file
← 90fd5c13 matrix
·
back to Exo
·
tweak bdf417f2 →