fix: cache ps scan + fix 60s ceiling exit-code in terminal-status (TK-11398, TK-11505)

1c16fa25a28598e17f9b8b94506e8a88d7931b68 · 2026-09-12 04:05:19 -0700 · steve@designerwallcoverings.com
Files touched

M terminal_status.py
M ticket_binding.py
Diff

commit 1c16fa25a28598e17f9b8b94506e8a88d7931b68
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date:   Sat Sep 12 04:05:19 2026 -0700

    fix: cache ps scan + fix 60s ceiling exit-code in terminal-status (TK-11398, TK-11505)
---
 terminal_status.py | 35 ++++++++++++++++++++++++++++++++---
 ticket_binding.py  | 13 ++++++++++---
 2 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/terminal_status.py b/terminal_status.py
index 55d0f40..0ac9fa3 100644
--- a/terminal_status.py
+++ b/terminal_status.py
@@ -20,6 +20,31 @@ import time
 import uuid
 import ticket_binding as tickets
 
+# TK-11505: the ps scan ceiling was hardcoded at 60s. At ~2200 procs under
+# concurrent load (N sessions each scanning all N sessions' processes) this is
+# routinely exceeded, causing every dot paint to raise StatusError and exit 1 --
+# which callers see as a failure, not a paint. Made configurable so deployments
+# with more processes can raise it without a code change. Default raised to 90s
+# (60 was the previous "raised" value; 90 gives headroom for the measured
+# worst-case of ~70s at 42 live sessions + 2k procs). The cross-process disk
+# cache (TK-11398, committed ce04960) reduces HOW OFTEN a fresh scan fires, so
+# the ceiling is hit far less frequently -- but when it IS needed the window
+# must be wide enough to actually finish.
+#
+# TK-11511: DOCUMENTED ROOT-CAUSE FINDING. Mac2 host saturation (load avg
+# spiking) is driven by per-session MCP fan-out: ~70 live Claude sessions each
+# launch ~19 MCP servers = ~1330 long-running processes beyond the normal
+# userland count. terminal-status itself launches ZERO MCP servers and has no
+# MCP calls anywhere in this file or ticket_binding.py. The load guard that
+# mitigates TK-11511 from terminal-status's side is the disk cache (TK-11398):
+# instead of 70 concurrent ps -ax scans each racing to complete within the
+# ceiling, sibling invocations within _DISK_TTL seconds share one result. Under
+# load that reduces ~70 concurrent scans to ~1 scan per _DISK_TTL window. A
+# load-average-based skip gate was considered and rejected: skipping a scan when
+# load is high would produce false "no owning terminal" negatives for the
+# negative-rescan valve (current_owner path), which is worse than a slow scan.
+_PS_TIMEOUT = float(os.environ.get("TERMINAL_STATUS_PS_TIMEOUT", "90"))
+
 COLORS = {
     "green": ("🟢", (0, 200, 83), "WORKING"),
     "yellow": ("🟡", (255, 204, 0), "DIRECTION?"),
@@ -305,15 +330,19 @@ def _scan_processes():
     # Ownership resolution genuinely needs the full table (owners() proves a
     # tty has exactly ONE live runtime before we agree to paint it), so the fix
     # is headroom plus an actionable message, not a narrower query.
+    # TK-11505: timeout is now configurable via TERMINAL_STATUS_PS_TIMEOUT
+    # (default 90s, set at module load). Exits non-zero on timeout via StatusError.
     try:
         result = subprocess.run(
             ["ps", "-axo", "pid=,ppid=,tty=,lstart=,comm="],
-            capture_output=True, text=True, timeout=60,
+            capture_output=True, text=True, timeout=_PS_TIMEOUT,
             env={**os.environ, "LC_ALL": "C"})
     except subprocess.TimeoutExpired:
         raise StatusError(
-            "Timed out reading the process table (ps -ax took >60s); the box is "
-            "overloaded -- check the process count with `ps -A | wc -l`.")
+            f"Timed out reading the process table (ps -ax took >{_PS_TIMEOUT:.0f}s); "
+            "the box is overloaded -- check the process count with `ps -A | wc -l`. "
+            "Set TERMINAL_STATUS_PS_TIMEOUT to a higher value if this box normally "
+            "has more than ~2000 processes.")
     if result.returncode:
         raise StatusError("Cannot inspect live terminal owners: " + result.stderr.strip())
     rows = {}
diff --git a/ticket_binding.py b/ticket_binding.py
index 11d2957..2dd7172 100644
--- a/ticket_binding.py
+++ b/ticket_binding.py
@@ -1,9 +1,15 @@
 """Read the existing ticket ledger; never create a second ticket database."""
 import datetime as dt
 import json
+import os
 import re
 import subprocess
 
+# TK-11505: shared timeout constant so ticket_binding uses the same configurable
+# ceiling as terminal_status._scan_processes(). Set TERMINAL_STATUS_PS_TIMEOUT
+# to override (default 90s).
+_PS_TIMEOUT = float(os.environ.get("TERMINAL_STATUS_PS_TIMEOUT", "90"))
+
 SHORT = re.compile(r"^TK-\d+(?=$|-)", re.I)
 CID = re.compile(r"^(?:assign|create|action)-[a-z0-9]+-(\d+)-[a-z0-9]+$")
 
@@ -62,13 +68,14 @@ def discover(root, rows, live, chain, argv=None):
         # and killed the ENTIRE repaint, so no dot was painted at all. That is
         # backwards: this block is an optional enrichment (see the comment above
         # -- "a conservative fallback" for the ticket LABEL). A slow ps must
-        # degrade the label, never the paint. Raised to 60s to match the sibling
-        # process-table read, and made non-fatal.
+        # degrade the label, never the paint. Raised to match the sibling
+        # process-table read (_PS_TIMEOUT, now configurable via
+        # TERMINAL_STATUS_PS_TIMEOUT, default 90s -- TK-11505), and made non-fatal.
         argv = {}
         try:
             output = subprocess.run(["ps", "-p", ",".join(str(o.pid) for o in live.values()),
                                      "-o", "pid=,args="], capture_output=True, text=True,
-                                    timeout=60)
+                                    timeout=_PS_TIMEOUT)
             for line in output.stdout.splitlines():
                 parts = line.strip().split(None, 1)
                 if len(parts) == 2:
← ce04960 terminal-status: cross-process proc-table cache + negative-r · back to Terminal Status · stopped marker must not reuse the lightblue base codepoint 4e63168 →