← back to Exo
migrate from circleci to github actions
0d0338f871649e501c41a827f65d280cbabbc4d0 · 2024-12-06 13:56:44 +0000 · Alex Cheema
Files touched
D .circleci/config.ymlA .github/workflows/build_and_test.yml
Diff
commit 0d0338f871649e501c41a827f65d280cbabbc4d0
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Fri Dec 6 13:56:44 2024 +0000
migrate from circleci to github actions
---
.circleci/config.yml | 346 -----------------------------------
.github/workflows/build_and_test.yml | 204 +++++++++++++++++++++
2 files changed, 204 insertions(+), 346 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index 07e685b8..00000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,346 +0,0 @@
-version: 2.1
-
-orbs:
- python: circleci/python@2
-
-commands:
- run_chatgpt_api_test:
- parameters:
- inference_engine:
- type: string
- model_id:
- type: string
- expected_output:
- type: string
- prompt:
- type: string
- steps:
- - run:
- name: Run chatgpt api integration test (<<parameters.inference_engine>>, <<parameters.model_id>>)
- command: |
- source env/bin/activate
-
- # Set CLANG=1 for tinygrad only
- if [ "<<parameters.inference_engine>>" = "tinygrad" ]; then
- pip install llvmlite
- export TOKENIZERS_PARALLELISM=true SUPPORT_BF16=0 CLANG=1
- fi
-
- # Start first instance
- HF_HOME="$(pwd)/.hf_cache_node1" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine <<parameters.inference_engine>> \
- --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 \
- --chatgpt-api-response-timeout 900 --disable-tui > output1.log &
- PID1=$!
- tail -f output1.log &
- TAIL1=$!
-
- # Start second instance
- HF_HOME="$(pwd)/.hf_cache_node2" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine <<parameters.inference_engine>> \
- --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 \
- --chatgpt-api-response-timeout 900 --disable-tui > output2.log &
- PID2=$!
- tail -f output2.log &
- TAIL2=$!
-
- # Remember to kill the tail processes at the end
- trap 'kill $TAIL1 $TAIL2' EXIT
-
- # Wait for discovery
- sleep 10
-
- # Function to check if processes are still running
- check_processes() {
- if ! kill -0 $PID1 2>/dev/null; then
- echo "First instance (PID $PID1) died unexpectedly. Log output:"
- cat output1.log
- exit 1
- fi
- if ! kill -0 $PID2 2>/dev/null; then
- echo "Second instance (PID $PID2) died unexpectedly. Log output:"
- cat output2.log
- exit 1
- fi
- }
-
- # Check processes before proceeding
- check_processes
-
- echo "Sending request to first instance..."
- response_1=$(curl -s http://localhost:8000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -d '{
- "model": "<<parameters.model_id>>",
- "messages": [{"role": "user", "content": "<<parameters.prompt>>"}],
- "temperature": 0.7
- }')
- echo "Response 1: $response_1"
-
- # Check processes after first response
- check_processes
-
- echo "Sending request to second instance..."
- response_2=$(curl -s http://localhost:8001/v1/chat/completions \
- -H "Content-Type: application/json" \
- -d '{
- "model": "<<parameters.model_id>>",
- "messages": [{"role": "user", "content": "<<parameters.prompt>>"}],
- "temperature": 0.7
- }')
- echo "Response 2: $response_2"
-
- # Check processes after second response
- check_processes
-
- # Stop both instances
- kill $PID1 $PID2
-
- echo ""
- # Extract content using jq and check if it contains expected output
- content1=$(echo "$response_1" | jq -r '.choices[0].message.content')
- content2=$(echo "$response_2" | jq -r '.choices[0].message.content')
-
- if [[ "$content1" != *"<<parameters.expected_output>>"* ]] || [[ "$content2" != *"<<parameters.expected_output>>"* ]]; then
- echo "Test failed: Response does not match '<<parameters.expected_output>>'"
- echo "Response 1 content: $content1"
- echo ""
- echo "Response 2 content: $content2"
- echo "Output of first instance:"
- cat output1.log
- echo "Output of second instance:"
- cat output2.log
- exit 1
- else
- echo "Test passed: Response from both nodes matches '<<parameters.expected_output>>'"
- fi
-
-jobs:
- unit_test:
- macos:
- xcode: "16.0.0"
- resource_class: m2pro.large
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- - run:
- name: Run tests
- command: |
- source env/bin/activate
- # set TEMPERATURE to 0 for deterministic sampling
- echo "Running inference engine tests..."
- METAL_DEVICE_WRAPPER_TYPE=1 METAL_DEBUG_ERROR_MODE=0 METAL_XCODE=1 TEMPERATURE=0 python3 -m exo.inference.test_inference_engine
- echo "Running tokenizer tests..."
- python3 ./test/test_tokenizers.py
- python3 ./test/test_model_helpers.py
-
- discovery_integration_test:
- macos:
- xcode: "16.0.0"
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- - run:
- name: Run discovery integration test
- command: |
- source env/bin/activate
- DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 --disable-tui > output1.log 2>&1 &
- PID1=$!
- DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 --disable-tui > output2.log 2>&1 &
- PID2=$!
- sleep 10
- kill $PID1 $PID2
- if grep -q "Peer statuses: {\\'node2\\': \\'is_connected=True, health_check=True" output1.log && ! grep -q "Failed to connect peers:" output1.log && grep -q "Peer statuses: {\\'node1\\': \\'is_connected=True, health_check=True" output2.log && ! grep -q "Failed to connect peers:" output2.log; then
- echo "Test passed: Both instances discovered each other"
- exit 0
- else
- echo "Test failed: Devices did not discover each other"
- echo "Output of first instance:"
- cat output1.log
- echo "Output of second instance:"
- cat output2.log
- exit 1
- fi
-
- chatgpt_api_integration_test_mlx:
- macos:
- xcode: "16.0.0"
- resource_class: m2pro.large
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- - run_chatgpt_api_test:
- inference_engine: mlx
- model_id: llama-3.2-1b
- prompt: "Keep responses concise. Who was the king of pop?"
- expected_output: "Michael Jackson"
-
- chatgpt_api_integration_test_dummy:
- macos:
- xcode: "16.0.0"
- resource_class: m2pro.large
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- - run_chatgpt_api_test:
- inference_engine: dummy
- model_id: dummy
- prompt: "Dummy prompt."
- expected_output: "dummy"
-
- chatgpt_api_integration_test_tinygrad:
- macos:
- xcode: "16.0.0"
- resource_class: m2pro.large
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- - run_chatgpt_api_test:
- inference_engine: tinygrad
- model_id: llama-3.2-1b
- prompt: "Keep responses concise. Who was the king of pop?"
- expected_output: "Michael Jackson"
-
- measure_pip_sizes:
- macos:
- xcode: "16.0.0"
- steps:
- - checkout
- - run:
- name: Set up Python
- command: |
- brew install python@3.12
- python3.12 -m venv env
- source env/bin/activate
- - run:
- name: Install dependencies and measure sizes
- command: |
- source env/bin/activate
- pip install --upgrade pip
- pip install .
- python ./extra/pipsize.py --json ./pipsize.json
- - store_artifacts:
- path: ./pipsize.json
- destination: pip-sizes.json
-
- check_line_count:
- docker:
- - image: cimg/python:3.10
- steps:
- - checkout
-
- - run:
- name: Setup git for PR comparison
- command: |
- if [[ -n "$CIRCLE_PULL_REQUEST" ]]; then
- PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | rev | cut -d'/' -f1 | rev)
- BASE_BRANCH=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
- "https://circleci.com/api/v2/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pipeline/$CIRCLE_WORKFLOW_ID" \
- | jq -r '.target_branch')
-
- git clone -b $BASE_BRANCH --single-branch \
- https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git \
- base_branch
- fi
-
- - run:
- name: Install dependencies
- command: |
- python -m pip install --upgrade pip
- pip install tabulate
-
- - run:
- name: Run line count check
- command: |
- if [[ -n "$CIRCLE_PULL_REQUEST" ]]; then
- python extra/line_counter.py base_branch .
- else
- python extra/line_counter.py .
- fi
-
- - store_artifacts:
- path: line-count-snapshot.json
- destination: line-count-snapshot.json
-
- - store_artifacts:
- path: line-count-diff.json
- destination: line-count-diff.json
-
- - run:
- name: Create test results directory
- command: |
- mkdir -p test-results/line-count
- cp line-count-*.json test-results/line-count/
-
- - store_test_results:
- path: test-results
-
-workflows:
- version: 2
- build_and_test:
- jobs:
- - check_line_count:
- filters:
- branches:
- only: /.*/
- tags:
- only: /.*/
- - unit_test
- - discovery_integration_test
- - chatgpt_api_integration_test_mlx
- - chatgpt_api_integration_test_tinygrad
- - chatgpt_api_integration_test_dummy
- - measure_pip_sizes
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
new file mode 100644
index 00000000..02289150
--- /dev/null
+++ b/.github/workflows/build_and_test.yml
@@ -0,0 +1,204 @@
+name: Build and Test
+
+on:
+ push:
+ branches: [ '*' ]
+ tags: [ '*' ]
+ pull_request:
+ branches: [ '*' ]
+
+env:
+ PYTHON_VERSION: "3.12"
+
+jobs:
+ check_line_count:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install tabulate
+
+ - name: Run line count check
+ run: |
+ if [[ -n "${{ github.event.pull_request }}" ]]; then
+ git fetch origin ${{ github.base_ref }}
+ git clone -b ${{ github.base_ref }} --single-branch \
+ https://github.com/${{ github.repository }}.git base_branch
+ python extra/line_counter.py base_branch .
+ else
+ python extra/line_counter.py .
+ fi
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: line-count-results
+ path: |
+ line-count-snapshot.json
+ line-count-diff.json
+
+ unit_test:
+ runs-on: macos-14
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ run: |
+ python -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+
+ - name: Run tests
+ run: |
+ source env/bin/activate
+ # set TEMPERATURE to 0 for deterministic sampling
+ echo "Running inference engine tests..."
+ METAL_DEVICE_WRAPPER_TYPE=1 METAL_DEBUG_ERROR_MODE=0 METAL_XCODE=1 TEMPERATURE=0 python3 -m exo.inference.test_inference_engine
+ echo "Running tokenizer tests..."
+ python3 ./test/test_tokenizers.py
+ python3 ./test/test_model_helpers.py
+
+ discovery_integration_test:
+ runs-on: macos-14
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ run: |
+ python -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+
+ - name: Run discovery integration test
+ run: |
+ source env/bin/activate
+ DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 --disable-tui > output1.log 2>&1 &
+ PID1=$!
+ DEBUG_DISCOVERY=7 DEBUG=7 exo --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 --disable-tui > output2.log 2>&1 &
+ PID2=$!
+ sleep 10
+ kill $PID1 $PID2
+ if grep -q "Peer statuses: {\\'node2\\': \\'is_connected=True, health_check=True" output1.log && ! grep -q "Failed to connect peers:" output1.log && grep -q "Peer statuses: {\\'node1\\': \\'is_connected=True, health_check=True" output2.log && ! grep -q "Failed to connect peers:" output2.log; then
+ echo "Test passed: Both instances discovered each other"
+ exit 0
+ else
+ echo "Test failed: Devices did not discover each other"
+ echo "Output of first instance:"
+ cat output1.log
+ echo "Output of second instance:"
+ cat output2.log
+ exit 1
+ fi
+
+ chatgpt_api_tests:
+ runs-on: macos-14
+ strategy:
+ matrix:
+ inference_engine: [mlx, tinygrad, dummy]
+ include:
+ - inference_engine: mlx
+ model_id: llama-3.2-1b
+ prompt: "Keep responses concise. Who was the king of pop?"
+ expected_output: "Michael Jackson"
+ - inference_engine: tinygrad
+ model_id: llama-3.2-1b
+ prompt: "Keep responses concise. Who was the king of pop?"
+ expected_output: "Michael Jackson"
+ - inference_engine: dummy
+ model_id: dummy
+ prompt: "Dummy prompt."
+ expected_output: "dummy"
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ run: |
+ python -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ if [ "${{ matrix.inference_engine }}" = "tinygrad" ]; then
+ pip install llvmlite
+ fi
+
+ - name: Run ChatGPT API test
+ env:
+ TOKENIZERS_PARALLELISM: ${{ matrix.inference_engine == 'tinygrad' && 'true' || 'false' }}
+ SUPPORT_BF16: ${{ matrix.inference_engine == 'tinygrad' && '0' || '' }}
+ CLANG: ${{ matrix.inference_engine == 'tinygrad' && '1' || '' }}
+ run: |
+ source env/bin/activate
+
+ # Start first instance
+ HF_HOME="$(pwd)/.hf_cache_node1" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine ${{ matrix.inference_engine }} \
+ --node-id "node1" --listen-port 5678 --broadcast-port 5679 --chatgpt-api-port 8000 \
+ --chatgpt-api-response-timeout 900 --disable-tui > output1.log &
+ PID1=$!
+ tail -f output1.log &
+ TAIL1=$!
+
+ # Start second instance
+ HF_HOME="$(pwd)/.hf_cache_node2" DEBUG_DISCOVERY=7 DEBUG=7 exo --inference-engine ${{ matrix.inference_engine }} \
+ --node-id "node2" --listen-port 5679 --broadcast-port 5678 --chatgpt-api-port 8001 \
+ --chatgpt-api-response-timeout 900 --disable-tui > output2.log &
+ PID2=$!
+ tail -f output2.log &
+ TAIL2=$!
+
+ # Remember to kill the tail processes at the end
+ trap 'kill $TAIL1 $TAIL2' EXIT
+
+ # Rest of the test script remains the same as in your CircleCI config
+ # ... (Copy the remaining test logic from the CircleCI config)
+
+ measure_pip_sizes:
+ runs-on: macos-14
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies and measure sizes
+ run: |
+ python -m venv env
+ source env/bin/activate
+ pip install --upgrade pip
+ pip install .
+ python ./extra/pipsize.py --json ./pipsize.json
+
+ - name: Upload pip sizes artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: pip-sizes
+ path: ./pipsize.json
← c5934348 fix encode endpoint
·
back to Exo
·
macos 15 972aea44 →