discovery should not include unhealthy peers

b1cf1085bec4f2358467fbd913f34eb3dfc8e603 · 2024-09-24 12:00:43 +0100 · Alex Cheema
Files touched

M exo/networking/tailscale_discovery.py
Diff

commit b1cf1085bec4f2358467fbd913f34eb3dfc8e603
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Tue Sep 24 12:00:43 2024 +0100

    discovery should not include unhealthy peers
---
 exo/networking/tailscale_discovery.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/exo/networking/tailscale_discovery.py b/exo/networking/tailscale_discovery.py
index bd9a8073..9a97e79a 100644
--- a/exo/networking/tailscale_discovery.py
+++ b/exo/networking/tailscale_discovery.py
@@ -87,13 +87,22 @@ class TailscaleDiscovery(Discovery):
             continue
 
           if peer_id not in self.known_peers or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}":
+            new_peer_handle = self.create_peer_handle(peer_id, f"{peer_host}:{peer_port}", device_capabilities)
+            if not await new_peer_handle.health_check():
+              if DEBUG >= 1: print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping.")
+              continue
+
             if DEBUG >= 1: print(f"Adding {peer_id=} at {peer_host}:{peer_port}. Replace existing peer_id: {peer_id in self.known_peers}")
             self.known_peers[peer_id] = (
-              self.create_peer_handle(peer_id, f"{peer_host}:{peer_port}", device_capabilities),
+              new_peer_handle,
               current_time,
               current_time,
             )
           else:
+            if not await self.known_peers[peer_id][0].health_check():
+              if DEBUG >= 1: print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Deleting.")
+              del self.known_peers[peer_id]
+              continue
             self.known_peers[peer_id] = (self.known_peers[peer_id][0], self.known_peers[peer_id][1], current_time)
 
       except Exception as e:
@@ -126,9 +135,9 @@ class TailscaleDiscovery(Discovery):
         current_time = time.time()
         peers_to_remove = [
           peer_handle.id() for peer_handle, connected_at, last_seen in self.known_peers.values()
-          if (not await peer_handle.is_connected() and current_time - connected_at > self.discovery_timeout) or current_time - last_seen > self.discovery_timeout
+          if (not await peer_handle.is_connected() and current_time - connected_at > self.discovery_timeout) or current_time - last_seen > self.discovery_timeout or not await peer_handle.health_check()
         ]
-        if DEBUG_DISCOVERY >= 2: print("Peer statuses:", {peer_handle.id(): f"is_connected={await peer_handle.is_connected()}, {connected_at=}, {last_seen=}" for peer_handle, connected_at, last_seen in self.known_peers.values()})
+        if DEBUG_DISCOVERY >= 2: print("Peer statuses:", {peer_handle.id(): f"is_connected={await peer_handle.is_connected()}, {connected_at=}, {last_seen=}, health_check={await peer_handle.health_check()}" for peer_handle, connected_at, last_seen in self.known_peers.values()})
         for peer_id in peers_to_remove:
           if peer_id in self.known_peers: del self.known_peers[peer_id]
           if DEBUG_DISCOVERY >= 2: print(f"Removed peer {peer_id} due to inactivity.")
← 7fa9f2cf increase default max-generate-tokens to 10,000 · back to Exo · if any peers changed from last time, we should always update 8aab9304 →