← back to Exo
add back .circleci
98d6e986bd3e15ee85e5cb55eac4b250740ac388 · 2025-01-22 21:58:46 +0000 · Alex Cheema
Files touched
Diff
commit 98d6e986bd3e15ee85e5cb55eac4b250740ac388
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Wed Jan 22 21:58:46 2025 +0000
add back .circleci
---
.circleci/config.yml | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 346 insertions(+)
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 00000000..07e685b8
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,346 @@
+version: 2.1
+
+orbs:
+ python: circleci/python@2
+
+commands:
+ run_chatgpt_api_test:
+ parameters:
+ inference_engine:
+ type: string
+ model_id:
+ type: string
+ expected_output:
+ type: string
+ prompt:
+ type: string
+ steps:
+ - run:
+ name: Run chatgpt api integration test (<<parameters.inference_engine>>, <<parameters.model_id>>)
+ command: |
+ source env/bin/activate
+
+ # Set CLANG=1 for tinygrad only
+ if [ "<<parameters.inference_engine>>" = "tinygrad" ]; then
+ pip install llvmlite
+ export TOKENIZERS_PARALLELISM=true SUPPORT_BF16=0 CLANG=1
+ fi
+
+ # Start first instance
+ HF_HOME="$(pwd)/.hf_cache_node1" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine <<parameters.inference_engine>> \
+ --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 \
+ --chatgpt-api-response-timeout 900 --disable-tui > output1.log &
+ PID1=$!
+ tail -f output1.log &
+ TAIL1=$!
+
+ # Start second instance
+ HF_HOME="$(pwd)/.hf_cache_node2" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine <<parameters.inference_engine>> \
+ --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 \
+ --chatgpt-api-response-timeout 900 --disable-tui > output2.log &
+ PID2=$!
+ tail -f output2.log &
+ TAIL2=$!
+
+ # Remember to kill the tail processes at the end
+ trap 'kill $TAIL1 $TAIL2' EXIT
+
+ # Wait for discovery
+ sleep 10
+
+ # Function to check if processes are still running
+ check_processes() {
+ if ! kill -0 $PID1 2>/dev/null; then
+ echo "First instance (PID $PID1) died unexpectedly. Log output:"
+ cat output1.log
+ exit 1
+ fi
+ if ! kill -0 $PID2 2>/dev/null; then
+ echo "Second instance (PID $PID2) died unexpectedly. Log output:"
+ cat output2.log
+ exit 1
+ fi
+ }
+
+ # Check processes before proceeding
+ check_processes
+
+ echo "Sending request to first instance..."
+ response_1=$(curl -s http://localhost:8000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "<<parameters.model_id>>",
+ "messages": [{"role": "user", "content": "<<parameters.prompt>>"}],
+ "temperature": 0.7
+ }')
+ echo "Response 1: $response_1"
+
+ # Check processes after first response
+ check_processes
+
+ echo "Sending request to second instance..."
+ response_2=$(curl -s http://localhost:8001/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "<<parameters.model_id>>",
+ "messages": [{"role": "user", "content": "<<parameters.prompt>>"}],
+ "temperature": 0.7
+ }')
+ echo "Response 2: $response_2"
+
+ # Check processes after second response
+ check_processes
+
+ # Stop both instances
+ kill $PID1 $PID2
+
+ echo ""
+ # Extract content using jq and check if it contains expected output
+ content1=$(echo "$response_1" | jq -r '.choices[0].message.content')
+ content2=$(echo "$response_2" | jq -r '.choices[0].message.content')
+
+ if [[ "$content1" != *"<<parameters.expected_output>>"* ]] || [[ "$content2" != *"<<parameters.expected_output>>"* ]]; then
+ echo "Test failed: Response does not match '<<parameters.expected_output>>'"
+ echo "Response 1 content: $content1"
+ echo ""
+ echo "Response 2 content: $content2"
+ echo "Output of first instance:"
+ cat output1.log
+ echo "Output of second instance:"
+ cat output2.log
+ exit 1
+ else
+ echo "Test passed: Response from both nodes matches '<<parameters.expected_output>>'"
+ fi
+
+jobs:
+ unit_test:
+ macos:
+ xcode: "16.0.0"
+ resource_class: m2pro.large
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ - run:
+ name: Run tests
+ command: |
+ source env/bin/activate
+ # set TEMPERATURE to 0 for deterministic sampling
+ echo "Running inference engine tests..."
+ METAL_DEVICE_WRAPPER_TYPE=1 METAL_DEBUG_ERROR_MODE=0 METAL_XCODE=1 TEMPERATURE=0 python3 -m exo.inference.test_inference_engine
+ echo "Running tokenizer tests..."
+ python3 ./test/test_tokenizers.py
+ python3 ./test/test_model_helpers.py
+
+ discovery_integration_test:
+ macos:
+ xcode: "16.0.0"
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ - run:
+ name: Run discovery integration test
+ command: |
+ source env/bin/activate
+ DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 --disable-tui > output1.log 2>&1 &
+ PID1=$!
+ DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 --disable-tui > output2.log 2>&1 &
+ PID2=$!
+ sleep 10
+ kill $PID1 $PID2
+ if grep -q "Peer statuses: {\\'node2\\': \\'is_connected=True, health_check=True" output1.log && ! grep -q "Failed to connect peers:" output1.log && grep -q "Peer statuses: {\\'node1\\': \\'is_connected=True, health_check=True" output2.log && ! grep -q "Failed to connect peers:" output2.log; then
+ echo "Test passed: Both instances discovered each other"
+ exit 0
+ else
+ echo "Test failed: Devices did not discover each other"
+ echo "Output of first instance:"
+ cat output1.log
+ echo "Output of second instance:"
+ cat output2.log
+ exit 1
+ fi
+
+ chatgpt_api_integration_test_mlx:
+ macos:
+ xcode: "16.0.0"
+ resource_class: m2pro.large
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ - run_chatgpt_api_test:
+ inference_engine: mlx
+ model_id: llama-3.2-1b
+ prompt: "Keep responses concise. Who was the king of pop?"
+ expected_output: "Michael Jackson"
+
+ chatgpt_api_integration_test_dummy:
+ macos:
+ xcode: "16.0.0"
+ resource_class: m2pro.large
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ - run_chatgpt_api_test:
+ inference_engine: dummy
+ model_id: dummy
+ prompt: "Dummy prompt."
+ expected_output: "dummy"
+
+ chatgpt_api_integration_test_tinygrad:
+ macos:
+ xcode: "16.0.0"
+ resource_class: m2pro.large
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ - run_chatgpt_api_test:
+ inference_engine: tinygrad
+ model_id: llama-3.2-1b
+ prompt: "Keep responses concise. Who was the king of pop?"
+ expected_output: "Michael Jackson"
+
+ measure_pip_sizes:
+ macos:
+ xcode: "16.0.0"
+ steps:
+ - checkout
+ - run:
+ name: Set up Python
+ command: |
+ brew install python@3.12
+ python3.12 -m venv env
+ source env/bin/activate
+ - run:
+ name: Install dependencies and measure sizes
+ command: |
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ python ./extra/pipsize.py --json ./pipsize.json
+ - store_artifacts:
+ path: ./pipsize.json
+ destination: pip-sizes.json
+
+ check_line_count:
+ docker:
+ - image: cimg/python:3.10
+ steps:
+ - checkout
+
+ - run:
+ name: Setup git for PR comparison
+ command: |
+ if [[ -n "$CIRCLE_PULL_REQUEST" ]]; then
+ PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | rev | cut -d'/' -f1 | rev)
+ BASE_BRANCH=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
+ "https://circleci.com/api/v2/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pipeline/$CIRCLE_WORKFLOW_ID" \
+ | jq -r '.target_branch')
+
+ git clone -b $BASE_BRANCH --single-branch \
+ https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git \
+ base_branch
+ fi
+
+ - run:
+ name: Install dependencies
+ command: |
+ python -m pip install --upgrade pip
+ pip install tabulate
+
+ - run:
+ name: Run line count check
+ command: |
+ if [[ -n "$CIRCLE_PULL_REQUEST" ]]; then
+ python extra/line_counter.py base_branch .
+ else
+ python extra/line_counter.py .
+ fi
+
+ - store_artifacts:
+ path: line-count-snapshot.json
+ destination: line-count-snapshot.json
+
+ - store_artifacts:
+ path: line-count-diff.json
+ destination: line-count-diff.json
+
+ - run:
+ name: Create test results directory
+ command: |
+ mkdir -p test-results/line-count
+ cp line-count-*.json test-results/line-count/
+
+ - store_test_results:
+ path: test-results
+
+workflows:
+ version: 2
+ build_and_test:
+ jobs:
+ - check_line_count:
+ filters:
+ branches:
+ only: /.*/
+ tags:
+ only: /.*/
+ - unit_test
+ - discovery_integration_test
+ - chatgpt_api_integration_test_mlx
+ - chatgpt_api_integration_test_tinygrad
+ - chatgpt_api_integration_test_dummy
+ - measure_pip_sizes
← d80324fe disable test-m3-single-node
·
back to Exo
·
temporarily disable github runner benchmarks 09e12d86 →