[object Object]

← back to Cli Printing Press

refactor(cli): share verify report finalization (#319)

ee923f84f3723d8b4e04daeb1b998093a7d7620a · 2026-04-26 17:57:02 -0700 · Trevin Chow

Add a verify runtime golden matrix before collapsing duplicated PASS/WARN/FAIL aggregation logic. The matrix freezes structural PASS, structural FAIL, and mock verify behavior so future runtime splits have direct golden coverage.

Files touched

Diff

commit ee923f84f3723d8b4e04daeb1b998093a7d7620a
Author: Trevin Chow <trevin@trevinchow.com>
Date:   Sun Apr 26 17:57:02 2026 -0700

    refactor(cli): share verify report finalization (#319)
    
    Add a verify runtime golden matrix before collapsing duplicated PASS/WARN/FAIL aggregation logic. The matrix freezes structural PASS, structural FAIL, and mock verify behavior so future runtime splits have direct golden coverage.
---
 internal/pipeline/runtime.go                       | 55 +---------------------
 internal/pipeline/runtime_report.go                | 34 +++++++++++++
 .../cases/verify-runtime-matrix/artifacts.txt      |  3 ++
 .../golden/cases/verify-runtime-matrix/command.txt |  5 ++
 .../golden/expected/verify-runtime-matrix/exit.txt |  1 +
 .../expected/verify-runtime-matrix/mock-pass.json  | 29 ++++++++++++
 .../expected/verify-runtime-matrix/stderr.txt      |  1 +
 .../expected/verify-runtime-matrix/stdout.txt      |  0
 .../verify-runtime-matrix/structural-fail.json     | 29 ++++++++++++
 .../verify-runtime-matrix/structural-pass.json     | 29 ++++++++++++
 .../mock-pass-pp-cli/cmd/mock-pass-pp-cli/main.go  | 31 ++++++++++++
 .../verify-runtime-matrix/mock-pass-pp-cli/go.mod  |  3 ++
 .../mock-pass-pp-cli/spec.yaml                     | 11 +++++
 .../cmd/structural-fail-pp-cli/main.go             | 20 ++++++++
 .../structural-fail-pp-cli/go.mod                  |  3 ++
 .../cmd/structural-pass-pp-cli/main.go             | 20 ++++++++
 .../structural-pass-pp-cli/go.mod                  |  3 ++
 17 files changed, 224 insertions(+), 53 deletions(-)

diff --git a/internal/pipeline/runtime.go b/internal/pipeline/runtime.go
index 6ba0f62f..9801faab 100644
--- a/internal/pipeline/runtime.go
+++ b/internal/pipeline/runtime.go
@@ -211,34 +211,7 @@ func RunVerify(cfg VerifyConfig) (*VerifyReport, error) {
 		}
 	}
 
-	// 9. Compute aggregate
-	for _, r := range report.Results {
-		report.Total++
-		if r.Score >= 2 {
-			report.Passed++
-		} else {
-			report.Failed++
-			if r.Score == 0 {
-				report.Critical++
-			}
-		}
-	}
-	if report.Total > 0 {
-		report.PassRate = float64(report.Passed) / float64(report.Total) * 100
-	}
-
-	// 10. Verdict
-	switch {
-	case report.PassRate >= float64(cfg.Threshold) && report.DataPipeline && report.Critical == 0:
-		report.Verdict = "PASS"
-	case report.PassRate >= 60 && report.Critical <= 3:
-		report.Verdict = "WARN"
-	default:
-		report.Verdict = "FAIL"
-	}
-	if report.BrowserSessionRequired && report.BrowserSessionProof != "valid" {
-		report.Verdict = "FAIL"
-	}
+	finalizeVerifyReport(report, cfg.Threshold, true)
 
 	return report, nil
 }
@@ -289,31 +262,7 @@ func runStructuralVerify(cfg VerifyConfig) (*VerifyReport, error) {
 	}
 	report.Freshness = runFreshnessContractTest(cfg.Dir)
 
-	// 5. Aggregate
-	for _, r := range report.Results {
-		report.Total++
-		if r.Score >= 2 {
-			report.Passed++
-		} else {
-			report.Failed++
-			if r.Score == 0 {
-				report.Critical++
-			}
-		}
-	}
-	if report.Total > 0 {
-		report.PassRate = float64(report.Passed) / float64(report.Total) * 100
-	}
-
-	// 6. Verdict
-	switch {
-	case report.PassRate >= float64(cfg.Threshold) && report.Critical == 0:
-		report.Verdict = "PASS"
-	case report.PassRate >= 60 && report.Critical <= 3:
-		report.Verdict = "WARN"
-	default:
-		report.Verdict = "FAIL"
-	}
+	finalizeVerifyReport(report, cfg.Threshold, false)
 
 	return report, nil
 }
diff --git a/internal/pipeline/runtime_report.go b/internal/pipeline/runtime_report.go
new file mode 100644
index 00000000..8f3dae97
--- /dev/null
+++ b/internal/pipeline/runtime_report.go
@@ -0,0 +1,34 @@
+package pipeline
+
+func finalizeVerifyReport(report *VerifyReport, threshold int, requireDataPipeline bool) {
+	for _, result := range report.Results {
+		report.Total++
+		if result.Score >= 2 {
+			report.Passed++
+			continue
+		}
+		report.Failed++
+		if result.Score == 0 {
+			report.Critical++
+		}
+	}
+	if report.Total > 0 {
+		report.PassRate = float64(report.Passed) / float64(report.Total) * 100
+	}
+
+	passGate := report.PassRate >= float64(threshold) && report.Critical == 0
+	if requireDataPipeline {
+		passGate = passGate && report.DataPipeline
+	}
+	switch {
+	case passGate:
+		report.Verdict = "PASS"
+	case report.PassRate >= 60 && report.Critical <= 3:
+		report.Verdict = "WARN"
+	default:
+		report.Verdict = "FAIL"
+	}
+	if report.BrowserSessionRequired && report.BrowserSessionProof != "valid" {
+		report.Verdict = "FAIL"
+	}
+}
diff --git a/testdata/golden/cases/verify-runtime-matrix/artifacts.txt b/testdata/golden/cases/verify-runtime-matrix/artifacts.txt
new file mode 100644
index 00000000..abcad15e
--- /dev/null
+++ b/testdata/golden/cases/verify-runtime-matrix/artifacts.txt
@@ -0,0 +1,3 @@
+structural-pass.json
+structural-fail.json
+mock-pass.json
diff --git a/testdata/golden/cases/verify-runtime-matrix/command.txt b/testdata/golden/cases/verify-runtime-matrix/command.txt
new file mode 100644
index 00000000..577a5ddf
--- /dev/null
+++ b/testdata/golden/cases/verify-runtime-matrix/command.txt
@@ -0,0 +1,5 @@
+set -euo pipefail
+cp -R testdata/golden/fixtures/verify-runtime-matrix "$CASE_ACTUAL_DIR/fixtures"
+"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/structural-pass-pp-cli" --no-spec --json > "$CASE_ACTUAL_DIR/structural-pass.json"
+"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/structural-fail-pp-cli" --no-spec --json > "$CASE_ACTUAL_DIR/structural-fail.json"
+"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/mock-pass-pp-cli" --spec "$CASE_ACTUAL_DIR/fixtures/mock-pass-pp-cli/spec.yaml" --json > "$CASE_ACTUAL_DIR/mock-pass.json"
diff --git a/testdata/golden/expected/verify-runtime-matrix/exit.txt b/testdata/golden/expected/verify-runtime-matrix/exit.txt
new file mode 100644
index 00000000..573541ac
--- /dev/null
+++ b/testdata/golden/expected/verify-runtime-matrix/exit.txt
@@ -0,0 +1 @@
+0
diff --git a/testdata/golden/expected/verify-runtime-matrix/mock-pass.json b/testdata/golden/expected/verify-runtime-matrix/mock-pass.json
new file mode 100644
index 00000000..c36d5671
--- /dev/null
+++ b/testdata/golden/expected/verify-runtime-matrix/mock-pass.json
@@ -0,0 +1,29 @@
+{
+  "verify": {
+    "binary": "<ARTIFACT_DIR>/verify-runtime-matrix/fixtures/mock-pass-pp-cli/mock-pass-pp-cli",
+    "critical": 0,
+    "data_pipeline": true,
+    "data_pipeline_detail": "PASS: 1 domain tables created",
+    "failed": 0,
+    "freshness": {
+      "detail": "cache freshness helper not emitted",
+      "enabled": false,
+      "verdict": "SKIP"
+    },
+    "mode": "mock",
+    "pass_rate": 100,
+    "passed": 1,
+    "results": [
+      {
+        "command": "items",
+        "dry_run": true,
+        "execute": true,
+        "help": true,
+        "kind": "read",
+        "score": 3
+      }
+    ],
+    "total": 1,
+    "verdict": "PASS"
+  }
+}
diff --git a/testdata/golden/expected/verify-runtime-matrix/stderr.txt b/testdata/golden/expected/verify-runtime-matrix/stderr.txt
new file mode 100644
index 00000000..a6f63c2a
--- /dev/null
+++ b/testdata/golden/expected/verify-runtime-matrix/stderr.txt
@@ -0,0 +1 @@
+warning: no servers defined in spec; generated CLI will require base_url in config
diff --git a/testdata/golden/expected/verify-runtime-matrix/stdout.txt b/testdata/golden/expected/verify-runtime-matrix/stdout.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/testdata/golden/expected/verify-runtime-matrix/structural-fail.json b/testdata/golden/expected/verify-runtime-matrix/structural-fail.json
new file mode 100644
index 00000000..dcb7b33b
--- /dev/null
+++ b/testdata/golden/expected/verify-runtime-matrix/structural-fail.json
@@ -0,0 +1,29 @@
+{
+  "verify": {
+    "binary": "<ARTIFACT_DIR>/verify-runtime-matrix/fixtures/structural-fail-pp-cli/structural-fail-pp-cli",
+    "critical": 0,
+    "data_pipeline": true,
+    "data_pipeline_detail": "PASS (version command)",
+    "failed": 1,
+    "freshness": {
+      "detail": "cache freshness helper not emitted",
+      "enabled": false,
+      "verdict": "SKIP"
+    },
+    "mode": "structural",
+    "pass_rate": 0,
+    "passed": 0,
+    "results": [
+      {
+        "command": "broken",
+        "dry_run": false,
+        "execute": true,
+        "help": false,
+        "kind": "structural",
+        "score": 1
+      }
+    ],
+    "total": 1,
+    "verdict": "FAIL"
+  }
+}
diff --git a/testdata/golden/expected/verify-runtime-matrix/structural-pass.json b/testdata/golden/expected/verify-runtime-matrix/structural-pass.json
new file mode 100644
index 00000000..ccb4a1e0
--- /dev/null
+++ b/testdata/golden/expected/verify-runtime-matrix/structural-pass.json
@@ -0,0 +1,29 @@
+{
+  "verify": {
+    "binary": "<ARTIFACT_DIR>/verify-runtime-matrix/fixtures/structural-pass-pp-cli/structural-pass-pp-cli",
+    "critical": 0,
+    "data_pipeline": true,
+    "data_pipeline_detail": "PASS (version command)",
+    "failed": 0,
+    "freshness": {
+      "detail": "cache freshness helper not emitted",
+      "enabled": false,
+      "verdict": "SKIP"
+    },
+    "mode": "structural",
+    "pass_rate": 100,
+    "passed": 1,
+    "results": [
+      {
+        "command": "items",
+        "dry_run": true,
+        "execute": true,
+        "help": true,
+        "kind": "structural",
+        "score": 3
+      }
+    ],
+    "total": 1,
+    "verdict": "PASS"
+  }
+}
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/cmd/mock-pass-pp-cli/main.go b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/cmd/mock-pass-pp-cli/main.go
new file mode 100644
index 00000000..45165c9c
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/cmd/mock-pass-pp-cli/main.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func main() {
+	args := os.Args[1:]
+	switch {
+	case len(args) == 0 || args[0] == "--help":
+		fmt.Println("Usage:\n  mock-pass [command]\n\nAvailable Commands:\n  items   List items\n\nFlags:\n  --help   help for mock-pass")
+	case args[0] == "items":
+		return
+	case args[0] == "sync":
+		return
+	case args[0] == "health":
+		return
+	case args[0] == "sql":
+		if strings.Contains(strings.Join(args[1:], " "), "sqlite_master") {
+			fmt.Println("items")
+			return
+		}
+		fmt.Println("1")
+	case args[0] == "version" || args[0] == "--version":
+		fmt.Println("mock-pass 1.0.0")
+	default:
+		os.Exit(1)
+	}
+}
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/go.mod b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/go.mod
new file mode 100644
index 00000000..5d9504e6
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/go.mod
@@ -0,0 +1,3 @@
+module example.com/mock-pass
+
+go 1.24
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/spec.yaml b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/spec.yaml
new file mode 100644
index 00000000..62f00ffe
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/mock-pass-pp-cli/spec.yaml
@@ -0,0 +1,11 @@
+openapi: 3.0.0
+info:
+  title: Verify Runtime Matrix
+  version: "1.0"
+paths:
+  /items:
+    get:
+      operationId: listItems
+      responses:
+        "200":
+          description: ok
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/cmd/structural-fail-pp-cli/main.go b/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/cmd/structural-fail-pp-cli/main.go
new file mode 100644
index 00000000..3e935a67
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/cmd/structural-fail-pp-cli/main.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	args := os.Args[1:]
+	switch {
+	case len(args) == 0 || args[0] == "--help":
+		fmt.Println("Usage:\n  structural-fail [command]\n\nAvailable Commands:\n  broken   Broken command\n\nFlags:\n  --help   help for structural-fail")
+	case args[0] == "broken":
+		os.Exit(1)
+	case args[0] == "version" || args[0] == "--version":
+		fmt.Println("structural-fail 1.0.0")
+	default:
+		os.Exit(1)
+	}
+}
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/go.mod b/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/go.mod
new file mode 100644
index 00000000..c7e897f8
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/structural-fail-pp-cli/go.mod
@@ -0,0 +1,3 @@
+module example.com/structural-fail
+
+go 1.24
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/cmd/structural-pass-pp-cli/main.go b/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/cmd/structural-pass-pp-cli/main.go
new file mode 100644
index 00000000..eae727bf
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/cmd/structural-pass-pp-cli/main.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	args := os.Args[1:]
+	switch {
+	case len(args) == 0 || args[0] == "--help":
+		fmt.Println("Usage:\n  structural-pass [command]\n\nAvailable Commands:\n  items   List items\n\nFlags:\n  --help   help for structural-pass")
+	case args[0] == "items":
+		return
+	case args[0] == "version" || args[0] == "--version":
+		fmt.Println("structural-pass 1.0.0")
+	default:
+		os.Exit(1)
+	}
+}
diff --git a/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/go.mod b/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/go.mod
new file mode 100644
index 00000000..3cd9c1ad
--- /dev/null
+++ b/testdata/golden/fixtures/verify-runtime-matrix/structural-pass-pp-cli/go.mod
@@ -0,0 +1,3 @@
+module example.com/structural-pass
+
+go 1.24

← faac0d39 refactor(cli): table-drive dogfood verdict rules (#318)  ·  back to Cli Printing Press  ·  refactor(cli): split structural verify runtime (#320) 7e733aa7 →