← back to Terminal Status
backfill: honest heartbeat verdict (kill the hardcoded-PASS false green)
a1d75ad48cf5c9776a4111b96ad5f486993eddb8 · 2026-09-17 07:16:31 -0700 · Steve Abrams
TK-11870. The backfill supervisor's heartbeat wrote verdict:PASS unconditionally.
Two false-green shapes it masked (CLAUDE.md TK-11431): (1) a ps-timeout -- the exact
failure this ticket is about -- raised inside processes(fresh=True) and wrote NO
heartbeat at all, a silence byte-identical to an idle box; (2) a partial/degraded
scan wrote PASS with a shrunken live_owners and no cross-check.
Fix: extract a pure, testable backfill_heartbeat() whose verdict is DERIVED from what
the cycle measured -- WARN + NOT-MEASURED (live_owners:null) when the scan failed,
WARN when live tabs exist but none could be asserted this pass, PASS only when it ran
and could act; live_owners always carried so a measured 0-of-0 stays distinguishable
from 0-of-N. The handler now catches the scan StatusError, writes the honest heartbeat
(fresh mtime so who-watches-the-watcher liveness still passes, failure in the verdict),
and stays fail-loud (stderr + exit 1). Ships 4 negative selftests proving it reddens on
the injected faults and greens only when it acted (selftest 11/11, unittest 49/49).
Also: gitignore + untrack data/backfill-latest.json (it rewrote every 20s, thrashing).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RaaqshnC3QW5DfzotaMZfM
Files touched
M .gitignoreD data/backfill-latest.jsonM terminal_status.py
Diff
commit a1d75ad48cf5c9776a4111b96ad5f486993eddb8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 17 07:16:31 2026 -0700
backfill: honest heartbeat verdict (kill the hardcoded-PASS false green)
TK-11870. The backfill supervisor's heartbeat wrote verdict:PASS unconditionally.
Two false-green shapes it masked (CLAUDE.md TK-11431): (1) a ps-timeout -- the exact
failure this ticket is about -- raised inside processes(fresh=True) and wrote NO
heartbeat at all, a silence byte-identical to an idle box; (2) a partial/degraded
scan wrote PASS with a shrunken live_owners and no cross-check.
Fix: extract a pure, testable backfill_heartbeat() whose verdict is DERIVED from what
the cycle measured -- WARN + NOT-MEASURED (live_owners:null) when the scan failed,
WARN when live tabs exist but none could be asserted this pass, PASS only when it ran
and could act; live_owners always carried so a measured 0-of-0 stays distinguishable
from 0-of-N. The handler now catches the scan StatusError, writes the honest heartbeat
(fresh mtime so who-watches-the-watcher liveness still passes, failure in the verdict),
and stays fail-loud (stderr + exit 1). Ships 4 negative selftests proving it reddens on
the injected faults and greens only when it acted (selftest 11/11, unittest 49/49).
Also: gitignore + untrack data/backfill-latest.json (it rewrote every 20s, thrashing).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RaaqshnC3QW5DfzotaMZfM
---
.gitignore | 2 ++
data/backfill-latest.json | 9 -----
terminal_status.py | 87 +++++++++++++++++++++++++++++++++++++++++------
3 files changed, 79 insertions(+), 19 deletions(-)
diff --git a/.gitignore b/.gitignore
index 93a53ff..e9c007f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,5 @@ __pycache__/
*.pyc
.venv/
verification/private/
+data/*.json
+data/*.log
diff --git a/data/backfill-latest.json b/data/backfill-latest.json
deleted file mode 100644
index a8b8396..0000000
--- a/data/backfill-latest.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "ts": "2026-09-17T14:12:37.371631+00:00",
- "verdict": "PASS",
- "status": "PASS",
- "painted": 24,
- "floored": 0,
- "skipped": 0,
- "live_owners": 24
-}
diff --git a/terminal_status.py b/terminal_status.py
index ee87193..e20d67e 100644
--- a/terminal_status.py
+++ b/terminal_status.py
@@ -563,6 +563,20 @@ def selftest():
case("backfill green-floors the recordless live owner",
f == 1 and green_rec is not None and green_rec["state"] == "green")
+ # 6. TK-11870 heartbeat honesty (CLAUDE.md TK-11431: an unmeasured input is
+ # NEVER green; severity maps to capability). The injected faults are the
+ # two false-green shapes the old hardcoded-PASS heartbeat masked.
+ case("heartbeat WARN + NOT-MEASURED when the ps scan failed",
+ backfill_heartbeat(0, 0, 0, None, scan_error="ps -ax timed out")["verdict"] == "WARN"
+ and backfill_heartbeat(0, 0, 0, None, scan_error="x")["live_owners"] is None)
+ case("heartbeat WARN when live tabs exist but NONE could be asserted",
+ backfill_heartbeat(0, 0, 3, 3)["verdict"] == "WARN")
+ case("heartbeat PASS when it acted on the live tabs",
+ backfill_heartbeat(30, 0, 3, 33)["verdict"] == "PASS")
+ case("heartbeat PASS on a measured 0-of-0 (no live tabs), still carrying the count",
+ backfill_heartbeat(0, 0, 0, 0)["verdict"] == "PASS"
+ and backfill_heartbeat(0, 0, 0, 0)["live_owners"] == 0)
+
ok = all(v for _, v in checks)
print(("selftest: PASS (%d/%d)" if ok else "selftest: FAIL (%d/%d)")
% (sum(v for _, v in checks), len(checks)))
@@ -1098,6 +1112,43 @@ def scan(store, rows=None, sessions=None):
PRIORITY.get(r["color"], len(PRIORITY)), r["tty"]))
+def backfill_heartbeat(painted, floored, skipped, live_owners, *, scan_error=None, ts=None):
+ """Honest heartbeat verdict for the backfill supervisor (TK-11870).
+
+ The verdict is DERIVED from what the cycle actually measured, never hardcoded
+ PASS -- severity maps to CAPABILITY, not counts (CLAUDE.md TK-11431):
+
+ * scan_error set -> WARN + NOT-MEASURED. The `ps` scan (the very input this
+ supervisor exists to keep warm) failed, so it could not act at ALL this
+ cycle. `live_owners` is null -- an unmeasured input is NEVER green. The
+ job still FIRED (fresh mtime), so the who-watches-the-watcher liveness
+ passes; the FAILURE rides in the verdict, not in the file's absence (the
+ old path raised before writing, so a ps-timeout produced NO heartbeat and
+ a byte-identical-to-idle silence).
+ * live_owners > 0 but painted+floored == 0 (every live tab skipped) -> WARN.
+ Live tabs were measured but NONE could be asserted this pass -- "could not
+ act this pass" = WARN (keep-alive doctrine). A live tab may be dark.
+ * otherwise -> PASS. It ran and could act; and live_owners is always carried
+ so a measured 0-of-0 (no live tabs) stays distinguishable from 0-of-N (the
+ false-green rule-1 example) rather than both reading green off a bare count.
+ """
+ if scan_error:
+ verdict = "WARN"
+ elif live_owners and painted + floored == 0:
+ verdict = "WARN"
+ else:
+ verdict = "PASS"
+ hb = {
+ "ts": ts or dt.datetime.now(dt.timezone.utc).isoformat(),
+ "verdict": verdict, "status": verdict,
+ "painted": painted, "floored": floored, "skipped": skipped,
+ "live_owners": live_owners,
+ }
+ if scan_error:
+ hb["error"] = scan_error
+ return hb
+
+
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="command", required=True)
@@ -1174,24 +1225,40 @@ def main(argv=None):
# `ps eww` env dump we don't need): backfill only REPAINTS existing records,
# it never resolves or creates a ticket label. Runs from a clean launchd env
# (no CLAUDE_CODE_CHILD_SESSION), so the child-session paint rail never trips.
- rows = processes(fresh=True)
- painted, floored, skipped = store.backfill(rows)
+ # The `ps` scan is the very input a ps-timeout kills -- catch it so the
+ # heartbeat CARRIES the failure (WARN + NOT-MEASURED) instead of the old
+ # path where processes() raised and NO heartbeat was written (silence that
+ # is byte-identical to an idle box). Still fail loud below (nonzero exit +
+ # stderr) so launchd + the drift canary also see it.
+ scan_error = None
+ try:
+ rows = processes(fresh=True)
+ except StatusError as exc:
+ scan_error = str(exc)
+ painted = floored = skipped = 0
+ live_owners = None
+ else:
+ painted, floored, skipped = store.backfill(rows)
+ live_owners = len(owners(rows))
# Liveness heartbeat, written AFTER the side effect (memory
# liveness-artifact-must-follow-the-side-effect): its mtime proves the sole
# producer actually FIRED (not just that launchd reports it "running"), and
- # its counts show what it did. Bounded (overwritten each run). Never breaks
- # the paint if the write fails.
+ # its verdict is derived from what the cycle measured (backfill_heartbeat),
+ # never a hardcoded PASS. Bounded (overwritten each run). Never breaks the
+ # paint if the write fails.
try:
hb = store.user_root / "Projects/terminal-status/data/backfill-latest.json"
hb.parent.mkdir(parents=True, exist_ok=True)
- atomic_write(hb, json.dumps({
- "ts": dt.datetime.now(dt.timezone.utc).isoformat(),
- "verdict": "PASS", "status": "PASS",
- "painted": painted, "floored": floored, "skipped": skipped,
- "live_owners": len(owners(rows)),
- }, indent=2) + "\n")
+ atomic_write(hb, json.dumps(
+ backfill_heartbeat(painted, floored, skipped, live_owners,
+ scan_error=scan_error), indent=2) + "\n")
except OSError:
pass
+ if scan_error:
+ # Fail loud: stderr + nonzero exit, so the failure is not visible ONLY
+ # in a heartbeat nobody happens to read.
+ print("backfill: " + scan_error, file=sys.stderr)
+ return 1
if not getattr(args, "quiet", False):
print("backfill: repainted %d, floored %d, skipped %d"
% (painted, floored, skipped))
← 8b9212d auto-data-snapshot: 2026-09-17T07:12:37 (1 data files) — dat
·
back to Terminal Status
·
dot.sh: auto-place window into its colour column on any succ 56c764a →