Keep Chief Ops watching Exo ring

717d12ea4c2f12a821c07968111e993308630a41 · 2026-09-17 09:21:34 -0700 · Steve Abrams
Files touched

M chief_ops.py
M supervisor_loop.sh
Diff

commit 717d12ea4c2f12a821c07968111e993308630a41
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Sep 17 09:21:34 2026 -0700

    Keep Chief Ops watching Exo ring
---
 chief_ops.py       | 36 ++++++++++++++++++++++++++++--------
 supervisor_loop.sh |  1 +
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/chief_ops.py b/chief_ops.py
index e52c506..e109850 100755
--- a/chief_ops.py
+++ b/chief_ops.py
@@ -59,18 +59,37 @@ def exo_ring():
     results=[]
     for item in hosts:
         host,_,port=item.rpartition(':'); port=int(port or 52415)
-        s=socket.socket(); s.settimeout(2); started=time.time()
-        try:
-            s.connect((host,port)); results.append({'host':host,'port':port,'online':True,'latency_ms':round((time.time()-started)*1000,1)})
-        except Exception as e:
-            msg=str(e)
-            results.append({'host':host,'port':port,'online':None if 'Operation not permitted' in msg else False,'error':msg})
-        finally: s.close()
+        last_error='timeout'
+        for attempt in range(3):
+            s=socket.socket(); s.settimeout(2); started=time.time()
+            try:
+                s.connect((host,port)); results.append({'host':host,'port':port,'online':True,'latency_ms':round((time.time()-started)*1000,1),'attempt':attempt+1}); break
+            except Exception as e:
+                last_error=str(e)
+                if 'Operation not permitted' in last_error:
+                    results.append({'host':host,'port':port,'online':None,'error':last_error}); break
+            finally: s.close()
+        else:
+            results.append({'host':host,'port':port,'online':False,'error':last_error,'attempts':3})
     online=sum(x['online'] is True for x in results); unknown=sum(x['online'] is None for x in results)
     ok=None if unknown else (bool(results) and online==len(results))
     return {'ok':ok,'online':online,'unknown':unknown,'configured':len(results),'hosts':results,
             'status':'unverified-in-sandbox' if unknown else ('all-online' if online==len(results) else ('partial' if online else 'offline'))}
 
+def watch_exo():
+    """Persist the Exo ring verdict and log only state transitions."""
+    STATE.mkdir(exist_ok=True)
+    verdict=exo_ring(); path=STATE/'exo-watch.json'; previous={}
+    if path.exists():
+        try: previous=json.loads(path.read_text())
+        except Exception: previous={}
+    verdict['checked_at']=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())
+    path.write_text(json.dumps(verdict,indent=2)+'\n')
+    old=previous.get('status'); new=verdict.get('status')
+    if old != new and new in ('partial','offline'):
+        cmd([str(TK),'log',TICKET,f'Chief Ops Exo watchdog: ring {new} ({verdict.get("online",0)}/{verdict.get("configured",0)} online); investigate host reachability or Mac keepalive.'], timeout=10)
+    return verdict
+
 
 def router_evidence():
     log = ROUTER/'data/router.jsonl'
@@ -200,10 +219,11 @@ class Handler(BaseHTTPRequestHandler):
         b=json.dumps(data, indent=2).encode(); self.send_response(200); self.send_header('Content-Type','application/json'); self.send_header('Content-Length',str(len(b))); self.end_headers(); self.wfile.write(b)
 
 def main():
-    ap=argparse.ArgumentParser(); ap.add_argument('action', choices=['check','arrange','queue','claim','serve'], nargs='?', default='check'); ap.add_argument('--port',type=int,default=9896); ap.add_argument('--write',action='store_true'); a=ap.parse_args()
+    ap=argparse.ArgumentParser(); ap.add_argument('action', choices=['check','arrange','queue','claim','watch-exo','serve'], nargs='?', default='check'); ap.add_argument('--port',type=int,default=9896); ap.add_argument('--write',action='store_true'); a=ap.parse_args()
     if a.action=='check': print(json.dumps(check(), indent=2)); return
     if a.action=='arrange': print(json.dumps(arrange(), indent=2)); return
     if a.action=='queue': print(json.dumps(ticket_queue(write=True), indent=2)); return
     if a.action=='claim': print(json.dumps(auto_claim(), indent=2)); return
+    if a.action=='watch-exo': print(json.dumps(watch_exo(), indent=2)); return
     print(f'Chief of Operations on http://127.0.0.1:{a.port}', flush=True); ThreadingHTTPServer(('127.0.0.1',a.port), Handler).serve_forever()
 if __name__=='__main__': main()
diff --git a/supervisor_loop.sh b/supervisor_loop.sh
index 9f500e3..9081ae5 100755
--- a/supervisor_loop.sh
+++ b/supervisor_loop.sh
@@ -3,6 +3,7 @@ set -u
 cd "$(dirname "$0")"
 INTERVAL="${CHIEF_OPS_INTERVAL:-60}"
 while true; do
+  ./chief_ops.py watch-exo >> state/supervisor.log 2>&1 || true
   ./chief_ops.py claim >> state/supervisor.log 2>&1 || true
   ./chief_ops.py queue --write >> state/supervisor.log 2>&1 || true
   sleep "$INTERVAL"
← 777f586 monitor all configured exo hosts · back to Chief Of Operations · auto-data-snapshot: 2026-09-17T09:20:48 (3 data files) — REA 11f814f →