[object Object]

← back to Exo

up http timeout to 3 seconds and retry on BadStatusLine (#1164)

3e623ccf0d18fb0e2d4262e3e57a6743ed0087be · 2026-01-15 18:15:12 +0000 · Evan Quiney

we're seeing a lot of network churn - perhaps this is a connection
timing out issue? lets also re-try after a second

## testing
none yet

---------

Co-authored-by: Alex Cheema <alexcheema123@gmail.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Files touched

Diff

commit 3e623ccf0d18fb0e2d4262e3e57a6743ed0087be
Author: Evan Quiney <evanev7@gmail.com>
Date:   Thu Jan 15 18:15:12 2026 +0000

    up http timeout to 3 seconds and retry on BadStatusLine (#1164)
    
    we're seeing a lot of network churn - perhaps this is a connection
    timing out issue? lets also re-try after a second
    
    ## testing
    none yet
    
    ---------
    
    Co-authored-by: Alex Cheema <alexcheema123@gmail.com>
    Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
---
 src/exo/worker/utils/net_profile.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/exo/worker/utils/net_profile.py b/src/exo/worker/utils/net_profile.py
index b4deabb4..23003045 100644
--- a/src/exo/worker/utils/net_profile.py
+++ b/src/exo/worker/utils/net_profile.py
@@ -1,4 +1,5 @@
 import http.client
+import time
 
 from anyio import create_task_group, to_thread
 from loguru import logger
@@ -6,6 +7,8 @@ from loguru import logger
 from exo.shared.topology import Topology
 from exo.shared.types.common import NodeId
 
+BAD_STATUSLINE_ATTEMPTS = 3
+
 
 async def check_reachability(
     target_ip: str,
@@ -15,8 +18,9 @@ async def check_reachability(
 ) -> None:
     """Check if a node is reachable at the given IP and verify its identity."""
 
-    def _fetch_remote_node_id() -> NodeId | None:
-        connection = http.client.HTTPConnection(target_ip, 52415, timeout=1)
+    # TODO: use an async http client
+    def _fetch_remote_node_id(*, attempt: int = 1) -> NodeId | None:
+        connection = http.client.HTTPConnection(target_ip, 52415, timeout=3)
         try:
             connection.request("GET", "/node_id")
             response = connection.getresponse()
@@ -32,7 +36,16 @@ async def check_reachability(
             return NodeId(body) or None
         except OSError:
             return None
-        except http.client.HTTPException:
+        except http.client.BadStatusLine:
+            if attempt >= BAD_STATUSLINE_ATTEMPTS:
+                logger.warning(
+                    f"BadStatusLine from {target_ip}, after {attempt} attempts, assuming connection to {expected_node_id} has dropped"
+                )
+                return None
+            time.sleep(1)
+            return _fetch_remote_node_id(attempt=attempt + 1)
+        except http.client.HTTPException as e:
+            logger.warning(f"HTTPException from {target_ip}: {type(e).__name__}: {e}")
             return None
         finally:
             connection.close()

← c22dad8a dashboard: add peer: true to package lock (#1162)  ·  back to Exo  ·  FIX GPT OSS (#1165) aaf4e36b →