fail if we never get the desired node count

e2474c3f15f4614d67862ad879b39e158be5cf68 · 2024-12-16 21:59:02 +0000 · Alex Cheema

Files touched

M .github/workflows/bench_job.yml

Diff

commit e2474c3f15f4614d67862ad879b39e158be5cf68
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Mon Dec 16 21:59:02 2024 +0000

    fail if we never get the desired node count
---
 .github/workflows/bench_job.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bench_job.yml b/.github/workflows/bench_job.yml
index 3bc2a6b1..6143f239 100644
--- a/.github/workflows/bench_job.yml
+++ b/.github/workflows/bench_job.yml
@@ -150,7 +150,7 @@ jobs:
           trap 'kill $PID1' EXIT
 
           echo "Waiting for all nodes to connect..."
-          for i in {1..100}; do
+          for i in {1..20}; do
             echo "Attempt $i: Checking node count..."
             nodes=$(curl -s http://localhost:52415/topology | jq ".nodes | length")
             echo "Current node count: $nodes"
@@ -158,6 +158,10 @@ jobs:
               echo "All nodes connected successfully!"
               break
             fi
+            if [ $i -eq 20 ]; then
+              echo "ERROR: Failed to connect all nodes after 20 attempts. Expected ${{ strategy.job-total }} nodes, but got $nodes"
+              exit 1
+            fi
             sleep 5
           done

← 1b14be60 make device_capabilities async running on a thread pool · back to Exo · Update strength image to image gen 5c0cd183 →