← back to Terminal Status
test_tk_required_fp_metric.py
126 lines
"""TK-11666 — tests for the TK-REQUIRED false-positive metric.
Includes the mandatory NEGATIVE test (TK-11431 rule 3): the metric must go RED on
an injected `contradicted` row and be non-vacuous."""
import unittest
import tk_required_fp_metric as m
class MetricTests(unittest.TestCase):
KNOWN = {"TK-11666", "TK-11317"}
def green(self, tty, pid, tsid, label, age_s=9000, color="green"):
return {"tty": tty, "pid": pid, "tsid": tsid, "label": label,
"age_s": age_s, "color": color}
def test_contradicted_row_goes_red(self):
# A live GREEN pane reading "TK REQUIRED" whose TERM_SESSION_ID HAS logged
# work on a KNOWN ticket = the false positive. Metric must FAIL.
s = self.green("ttys099", 4242, "wXtYpZ:FP", "🟢 TK REQUIRED · WORKING")
ledger = {"wXtYpZ:FP": [(1000.0, "TK-11666-fix-x")]}
b = m.classify([s], ledger, {}, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 1)
self.assertEqual(b["contradicted"][0]["resolved_via"], "ledger")
self.assertEqual(m.verdict_of(b, True), "FAIL")
def test_argv_channel_also_contradicts(self):
s = self.green("ttys099", 4242, "wX:none", "🟢 TK REQUIRED · WORKING")
argv = {4242: "claude export TK_AGENT=claude-run-11666. You are driving TK-11666-x."}
b = m.classify([s], {}, argv, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 1)
self.assertEqual(b["contradicted"][0]["resolved_via"], "argv_agent")
self.assertEqual(m.verdict_of(b, True), "FAIL")
def test_clean_bound_session_passes(self):
s = self.green("ttys098", 4243, "wA:OK", "🟢 TK-11317 · WORKING")
b = m.classify([s], {}, {}, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 0)
self.assertEqual(len(b["bound"]), 1)
self.assertEqual(m.verdict_of(b, True), "PASS")
def test_not_measured_is_never_an_accusation_and_does_not_fail(self):
# TK REQUIRED with NO resolvable channel = honest not_measured, not a false
# positive, and it does NOT block PASS (rule 1: never an accusation).
s = self.green("ttys097", 4244, "wP:NONE", "🟢 TK REQUIRED · WORKING")
b = m.classify([s], {}, {}, self.KNOWN)
self.assertEqual(len(b["not_measured"]), 1)
self.assertEqual(len(b["contradicted"]), 0)
self.assertEqual(m.verdict_of(b, True), "PASS")
def test_ledger_ticket_must_be_known(self):
# A ledger event under this session for a ticket NEVER created cannot prove
# tracking (never invents a ticket) → not_measured, not contradicted.
s = self.green("ttys096", 4245, "wG:GHOST", "🟢 TK REQUIRED · WORKING")
ledger = {"wG:GHOST": [(1000.0, "TK-99999999-ghost")]}
b = m.classify([s], ledger, {}, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 0)
self.assertEqual(len(b["not_measured"]), 1)
def test_read_events_are_noise_not_tracking(self):
# `read` events are excluded upstream in gather_ledger (only WORK_TYPES are
# collected), so a session that only READ the board has no ledger entry and
# stays not_measured.
self.assertNotIn("read", m.WORK_TYPES)
s = self.green("ttys095", 4246, "wR:READONLY", "🟢 TK REQUIRED · WORKING")
b = m.classify([s], {}, {}, self.KNOWN) # no work events for this tsid
self.assertEqual(len(b["not_measured"]), 1)
self.assertEqual(len(b["contradicted"]), 0)
def test_launch_grace_defers_young_sessions_to_pending(self):
# A tracked-but-young session is `pending`, never `contradicted` — the
# launch-latency noise that spawned ~24 spurious tickets.
s = self.green("ttys094", 4247, "wY:YOUNG", "🟢 TK REQUIRED · WORKING", age_s=5)
ledger = {"wY:YOUNG": [(1000.0, "TK-11666-x")]}
b = m.classify([s], ledger, {}, self.KNOWN)
self.assertEqual(len(b["pending"]), 1)
self.assertEqual(len(b["contradicted"]), 0)
self.assertEqual(m.verdict_of(b, True), "PASS")
def test_empty_enumeration_is_warn_not_pass(self):
# Rule 1: an enumeration that returned zero rows is WARN, never a green PASS
# ("0 of 0" != "0 of 60").
b = m.classify([], {}, {}, self.KNOWN)
self.assertEqual(m.verdict_of(b, observed_ok=False), "WARN")
def test_unmeasured_input_is_warn_not_pass(self):
# allcolordots/ledger unavailable → observed_ok False → WARN even with rows.
s = self.green("ttys093", 4248, "wU:UNSEEN", "🟢 TK-11317 · WORKING")
b = m.classify([s], {}, {}, self.KNOWN)
self.assertEqual(m.verdict_of(b, observed_ok=False), "WARN")
def test_only_green_sessions_are_scored(self):
# A purple/gated pane reading TK REQUIRED is NOT part of the false-positive
# question (the ticket scopes it to live GREEN sessions).
s = self.green("ttys092", 4249, "wPur:X", "🟣 TK REQUIRED · gated", color="purple")
ledger = {"wPur:X": [(1000.0, "TK-11666-x")]}
b = m.classify([s], ledger, {}, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 0)
self.assertEqual(len(b["bound"]) + len(b["not_measured"]) + len(b["pending"]), 0)
def test_unknown_age_is_not_hidden_as_young(self):
# A tracked TK-REQUIRED pane with UNKNOWN age (age_s None — e.g. ps lstart
# unparseable) must be counted contradicted, NOT excused as pending. Hiding
# it would be the "measuring the wrong thing" false green.
s = self.green("ttys091", 4250, "wN:NOAGE", "🟢 TK REQUIRED · WORKING", age_s=None)
ledger = {"wN:NOAGE": [(1000.0, "TK-11666-x")]}
b = m.classify([s], ledger, {}, self.KNOWN)
self.assertEqual(len(b["contradicted"]), 1)
self.assertEqual(len(b["pending"]), 0)
self.assertEqual(m.verdict_of(b, True), "FAIL")
def test_proc_ages_resolve_for_a_live_pid(self):
# Guard the macOS lstart path: a real live pid must yield a positive age
# (the etimes bug returned {} → every false positive hidden as young).
import os
ages = m._proc_ages([os.getpid()])
self.assertIn(os.getpid(), ages)
self.assertGreaterEqual(ages[os.getpid()], 0.0)
def test_self_test_entrypoint_runs(self):
m._self_test() # raises on any regression
if __name__ == "__main__":
unittest.main()