← back to Cli Printing Press
refactor(cli): table-drive fullrun comparison metrics (#316)
a5c59b330452f93d17c0bb5fb336640184e4a20b · 2026-04-26 15:37:17 -0700 · Trevin Chow
Share the full-run score dimension registry across comparison and learnings output, and table-drive repeated comparison rows while preserving row order with focused test coverage.
Files touched
M internal/pipeline/fullrun.goM internal/pipeline/fullrun_test.go
Diff
commit a5c59b330452f93d17c0bb5fb336640184e4a20b
Author: Trevin Chow <trevin@trevinchow.com>
Date: Sun Apr 26 15:37:17 2026 -0700
refactor(cli): table-drive fullrun comparison metrics (#316)
Share the full-run score dimension registry across comparison and learnings output, and table-drive repeated comparison rows while preserving row order with focused test coverage.
---
internal/pipeline/fullrun.go | 288 ++++++++++++++++++--------------------
internal/pipeline/fullrun_test.go | 54 +++++++
2 files changed, 187 insertions(+), 155 deletions(-)
diff --git a/internal/pipeline/fullrun.go b/internal/pipeline/fullrun.go
index a1829543..252c8a58 100644
--- a/internal/pipeline/fullrun.go
+++ b/internal/pipeline/fullrun.go
@@ -64,6 +64,23 @@ type FullRunResult struct {
Duration time.Duration
}
+type fullRunScoreDimension struct {
+ name string
+ label string
+ score func(*SteinerScore) int
+}
+
+var fullRunScoreDimensions = []fullRunScoreDimension{
+ {"output_modes", "Output Modes", func(s *SteinerScore) int { return s.OutputModes }},
+ {"auth", "Auth", func(s *SteinerScore) int { return s.Auth }},
+ {"error_handling", "Error Handling", func(s *SteinerScore) int { return s.ErrorHandling }},
+ {"terminal_ux", "Terminal UX", func(s *SteinerScore) int { return s.TerminalUX }},
+ {"readme", "README", func(s *SteinerScore) int { return s.README }},
+ {"doctor", "Doctor", func(s *SteinerScore) int { return s.Doctor }},
+ {"agent_native", "Agent Native", func(s *SteinerScore) int { return s.AgentNative }},
+ {"local_cache", "Local Cache", func(s *SteinerScore) int { return s.LocalCache }},
+}
+
// MakeBestCLI runs the full printing press pipeline for a single API and
// returns a result summarizing every phase.
func MakeBestCLI(apiName, level, specFlag, specURL, outputDir, pressBinary string) *FullRunResult {
@@ -400,158 +417,131 @@ func PrintComparisonTable(results []*FullRunResult) string {
}
b.WriteString("|\n")
- // Quality Gates
- writeRow(&b, "Quality Gates", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d/7 PASS", r.GatesPassed)
- })
-
- // Commands
- writeRow(&b, "Commands", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d", r.CommandCount)
- })
-
- // Resources
- writeRow(&b, "Resources", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d", r.ResourceCount)
- })
-
- // API Coverage
- writeRow(&b, "API Coverage", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d%%", r.CoveragePercent)
+ writeComparisonRows(&b, results, []comparisonRow{
+ {"Quality Gates", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d/7 PASS", r.GatesPassed)
+ }},
+ {"Commands", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d", r.CommandCount)
+ }},
+ {"Resources", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d", r.ResourceCount)
+ }},
+ {"API Coverage", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d%%", r.CoveragePercent)
+ }},
})
// Steinberger dimensions
- steinDimensions := []struct {
- label string
- get func(*Scorecard) int
- }{
- {"Output Modes", func(s *Scorecard) int { return s.Steinberger.OutputModes }},
- {"Auth", func(s *Scorecard) int { return s.Steinberger.Auth }},
- {"Error Handling", func(s *Scorecard) int { return s.Steinberger.ErrorHandling }},
- {"Terminal UX", func(s *Scorecard) int { return s.Steinberger.TerminalUX }},
- {"README", func(s *Scorecard) int { return s.Steinberger.README }},
- {"Doctor", func(s *Scorecard) int { return s.Steinberger.Doctor }},
- {"Agent Native", func(s *Scorecard) int { return s.Steinberger.AgentNative }},
- {"Local Cache", func(s *Scorecard) int { return s.Steinberger.LocalCache }},
- }
-
- for _, dim := range steinDimensions {
+ for _, dim := range fullRunScoreDimensions {
writeRow(&b, " "+dim.label, results, func(r *FullRunResult) string {
if r.Scorecard == nil {
return "n/a"
}
- return fmt.Sprintf("%d/10", dim.get(r.Scorecard))
+ return fmt.Sprintf("%d/10", dim.score(&r.Scorecard.Steinberger))
})
}
- // Steinberger total + %
- writeRow(&b, "Steinberger Total", results, func(r *FullRunResult) string {
- if r.Scorecard == nil {
- return "n/a"
- }
- return fmt.Sprintf("%d/80 (%d%%)", r.Scorecard.Steinberger.Total, r.Scorecard.Steinberger.Percentage)
- })
-
- // Grade
- writeRow(&b, "Grade", results, func(r *FullRunResult) string {
- if r.Scorecard == nil {
- return "n/a"
- }
- return r.Scorecard.OverallGrade
- })
-
- // Competitors found
- writeRow(&b, "Competitors Found", results, func(r *FullRunResult) string {
- if r.Research == nil {
- return "0"
- }
- return fmt.Sprintf("%d", len(r.Research.Alternatives))
- })
-
- // We Win?
- writeRow(&b, "We Win?", results, func(r *FullRunResult) string {
- if r.Scorecard == nil || len(r.Scorecard.CompetitorScores) == 0 {
- return "n/a"
- }
- wins := 0
- for _, cs := range r.Scorecard.CompetitorScores {
- if cs.WeWin {
- wins++
+ writeComparisonRows(&b, results, []comparisonRow{
+ {"Steinberger Total", func(r *FullRunResult) string {
+ if r.Scorecard == nil {
+ return "n/a"
}
- }
- return fmt.Sprintf("%d/%d", wins, len(r.Scorecard.CompetitorScores))
- })
-
- // Dogfood result
- writeRow(&b, "Dogfood", results, func(r *FullRunResult) string {
- if r.Dogfood == nil {
- return "n/a"
- }
- if r.Dogfood.PathCheck.Tested > 0 {
- return fmt.Sprintf("%s %d%%", r.Dogfood.Verdict, r.Dogfood.PathCheck.Pct)
- }
- return r.Dogfood.Verdict
- })
-
- // Verification
- writeRow(&b, "Verification", results, func(r *FullRunResult) string {
- if r.Verification == nil {
- return "n/a"
- }
- summary := r.Verification.Verdict
- if r.Verification.HallucinatedPaths > 0 {
- summary += fmt.Sprintf(" %dp", r.Verification.HallucinatedPaths)
- }
- if r.Verification.DeadFlags > 0 {
- summary += fmt.Sprintf(" %df", r.Verification.DeadFlags)
- }
- if r.Verification.GhostTables > 0 {
- summary += fmt.Sprintf(" %dg", r.Verification.GhostTables)
- }
- return summary
- })
-
- // Workflow Verification
- writeRow(&b, "Workflow Verify", results, func(r *FullRunResult) string {
- if r.WorkflowVerify == nil {
- return "n/a"
- }
- return string(r.WorkflowVerify.Verdict)
- })
-
- // LLM Polish
- writeRow(&b, "LLM Polish", results, func(r *FullRunResult) string {
- if r.PolishResult == nil {
- return "n/a"
- }
- if r.PolishResult.Skipped {
- return "skipped"
- }
- return fmt.Sprintf("%dh/%de/%v",
- r.PolishResult.HelpTextsImproved,
- r.PolishResult.ExamplesAdded,
- r.PolishResult.READMERewritten)
- })
-
- // Fix plans generated
- writeRow(&b, "Fix Plans", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d", len(r.FixPlans))
- })
-
- // Duration
- writeRow(&b, "Duration", results, func(r *FullRunResult) string {
- return r.Duration.Round(time.Second).String()
- })
-
- // Errors
- writeRow(&b, "Errors", results, func(r *FullRunResult) string {
- return fmt.Sprintf("%d", len(r.Errors))
+ return fmt.Sprintf("%d/80 (%d%%)", r.Scorecard.Steinberger.Total, r.Scorecard.Steinberger.Percentage)
+ }},
+ {"Grade", func(r *FullRunResult) string {
+ if r.Scorecard == nil {
+ return "n/a"
+ }
+ return r.Scorecard.OverallGrade
+ }},
+ {"Competitors Found", func(r *FullRunResult) string {
+ if r.Research == nil {
+ return "0"
+ }
+ return fmt.Sprintf("%d", len(r.Research.Alternatives))
+ }},
+ {"We Win?", func(r *FullRunResult) string {
+ if r.Scorecard == nil || len(r.Scorecard.CompetitorScores) == 0 {
+ return "n/a"
+ }
+ wins := 0
+ for _, cs := range r.Scorecard.CompetitorScores {
+ if cs.WeWin {
+ wins++
+ }
+ }
+ return fmt.Sprintf("%d/%d", wins, len(r.Scorecard.CompetitorScores))
+ }},
+ {"Dogfood", func(r *FullRunResult) string {
+ if r.Dogfood == nil {
+ return "n/a"
+ }
+ if r.Dogfood.PathCheck.Tested > 0 {
+ return fmt.Sprintf("%s %d%%", r.Dogfood.Verdict, r.Dogfood.PathCheck.Pct)
+ }
+ return r.Dogfood.Verdict
+ }},
+ {"Verification", func(r *FullRunResult) string {
+ if r.Verification == nil {
+ return "n/a"
+ }
+ summary := r.Verification.Verdict
+ if r.Verification.HallucinatedPaths > 0 {
+ summary += fmt.Sprintf(" %dp", r.Verification.HallucinatedPaths)
+ }
+ if r.Verification.DeadFlags > 0 {
+ summary += fmt.Sprintf(" %df", r.Verification.DeadFlags)
+ }
+ if r.Verification.GhostTables > 0 {
+ summary += fmt.Sprintf(" %dg", r.Verification.GhostTables)
+ }
+ return summary
+ }},
+ {"Workflow Verify", func(r *FullRunResult) string {
+ if r.WorkflowVerify == nil {
+ return "n/a"
+ }
+ return string(r.WorkflowVerify.Verdict)
+ }},
+ {"LLM Polish", func(r *FullRunResult) string {
+ if r.PolishResult == nil {
+ return "n/a"
+ }
+ if r.PolishResult.Skipped {
+ return "skipped"
+ }
+ return fmt.Sprintf("%dh/%de/%v",
+ r.PolishResult.HelpTextsImproved,
+ r.PolishResult.ExamplesAdded,
+ r.PolishResult.READMERewritten)
+ }},
+ {"Fix Plans", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d", len(r.FixPlans))
+ }},
+ {"Duration", func(r *FullRunResult) string {
+ return r.Duration.Round(time.Second).String()
+ }},
+ {"Errors", func(r *FullRunResult) string {
+ return fmt.Sprintf("%d", len(r.Errors))
+ }},
})
b.WriteString("\n")
return b.String()
}
+type comparisonRow struct {
+ label string
+ value func(*FullRunResult) string
+}
+
+func writeComparisonRows(b *strings.Builder, results []*FullRunResult, rows []comparisonRow) {
+ for _, row := range rows {
+ writeRow(b, row.label, results, row.value)
+ }
+}
+
func writeRow(b *strings.Builder, label string, results []*FullRunResult, fn func(*FullRunResult) string) {
fmt.Fprintf(b, "%-25s", label)
for _, r := range results {
@@ -589,25 +579,13 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
total int
sum int
}
- dimNames := []string{"output_modes", "auth", "error_handling", "terminal_ux", "readme", "doctor", "agent_native", "local_cache"}
- dimGetters := []func(*SteinerScore) int{
- func(s *SteinerScore) int { return s.OutputModes },
- func(s *SteinerScore) int { return s.Auth },
- func(s *SteinerScore) int { return s.ErrorHandling },
- func(s *SteinerScore) int { return s.TerminalUX },
- func(s *SteinerScore) int { return s.README },
- func(s *SteinerScore) int { return s.Doctor },
- func(s *SteinerScore) int { return s.AgentNative },
- func(s *SteinerScore) int { return s.LocalCache },
- }
-
- tallies := make([]dimTally, len(dimNames))
+ tallies := make([]dimTally, len(fullRunScoreDimensions))
for _, r := range results {
if r.Scorecard == nil {
continue
}
- for i, getter := range dimGetters {
- score := getter(&r.Scorecard.Steinberger)
+ for i, dim := range fullRunScoreDimensions {
+ score := dim.score(&r.Scorecard.Steinberger)
tallies[i].total++
tallies[i].sum += score
if score < 5 {
@@ -619,7 +597,7 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
b.WriteString("## Consistent Gaps\n\n")
b.WriteString("Dimensions scoring below 5/10 across multiple runs:\n\n")
hasGaps := false
- for i, name := range dimNames {
+ for i, dim := range fullRunScoreDimensions {
t := tallies[i]
if t.total == 0 {
continue
@@ -627,7 +605,7 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
avg := t.sum / t.total
if t.lowCount > 0 {
hasGaps = true
- fmt.Fprintf(&b, "- **%s** - low in %d/%d runs (avg %d/10)\n", name, t.lowCount, t.total, avg)
+ fmt.Fprintf(&b, "- **%s** - low in %d/%d runs (avg %d/10)\n", dim.name, t.lowCount, t.total, avg)
}
}
if !hasGaps {
@@ -639,14 +617,14 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
b.WriteString("## Recommended Fixes\n\n")
b.WriteString("Priority order (most impactful first):\n\n")
priority := 1
- for i, name := range dimNames {
+ for i, dim := range fullRunScoreDimensions {
t := tallies[i]
if t.total == 0 || t.lowCount == 0 {
continue
}
fmt.Fprintf(&b, "%d. **Improve %s templates** - affects %d/%d APIs tested\n",
- priority, name, t.lowCount, t.total)
- if advice, ok := dimensionAdvice[name]; ok {
+ priority, dim.name, t.lowCount, t.total)
+ if advice, ok := dimensionAdvice[dim.name]; ok {
// Include first line of advice
lines := strings.SplitN(advice, "\n", 2)
fmt.Fprintf(&b, " - %s\n", strings.TrimSpace(lines[0]))
diff --git a/internal/pipeline/fullrun_test.go b/internal/pipeline/fullrun_test.go
index 4f3ef5a5..47ef27ee 100644
--- a/internal/pipeline/fullrun_test.go
+++ b/internal/pipeline/fullrun_test.go
@@ -5,9 +5,11 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strings"
"testing"
"time"
+ "github.com/mvanhorn/cli-printing-press/v2/internal/llmpolish"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -69,6 +71,58 @@ func TestFullRunQualitySpecPath(t *testing.T) {
assert.Equal(t, "", fullRunQualitySpecPath("--docs", "https://example.com/docs"))
}
+func TestPrintComparisonTableRows(t *testing.T) {
+ t.Parallel()
+
+ table := PrintComparisonTable([]*FullRunResult{{
+ APIName: "demo",
+ Level: "EASY",
+ GatesPassed: 6,
+ CommandCount: 9,
+ ResourceCount: 5,
+ CoveragePercent: 83,
+ Research: &ResearchResult{Alternatives: []Alternative{{Name: "alt"}}},
+ Dogfood: &DogfoodReport{Verdict: "PASS", PathCheck: PathCheckResult{Tested: 2, Pct: 100}},
+ Verification: &VerificationReport{Verdict: "WARN", HallucinatedPaths: 1, DeadFlags: 2, GhostTables: 3},
+ WorkflowVerify: &WorkflowVerifyReport{Verdict: WorkflowVerdictPass},
+ PolishResult: &llmpolish.PolishResult{HelpTextsImproved: 2, ExamplesAdded: 3, READMERewritten: true},
+ FixPlans: []string{"fix-auth", "fix-docs"},
+ Duration: 2 * time.Second,
+ Errors: []string{"boom"},
+ Scorecard: &Scorecard{
+ OverallGrade: "B",
+ Steinberger: SteinerScore{
+ OutputModes: 8, Auth: 7, ErrorHandling: 6, TerminalUX: 5,
+ README: 4, Doctor: 3, AgentNative: 2, LocalCache: 1,
+ Total: 72, Percentage: 72,
+ },
+ CompetitorScores: []CompScore{{WeWin: true}, {WeWin: false}},
+ },
+ }})
+
+ expectedOrder := []string{
+ "Quality Gates", "Commands", "Resources", "API Coverage",
+ "Output Modes", "Auth", "Error Handling", "Terminal UX", "README", "Doctor", "Agent Native", "Local Cache",
+ "Steinberger Total", "Grade", "Competitors Found", "We Win?", "Dogfood", "Verification", "Workflow Verify", "LLM Polish", "Fix Plans", "Duration", "Errors",
+ }
+ last := -1
+ for _, label := range expectedOrder {
+ idx := strings.Index(table, label)
+ require.NotEqual(t, -1, idx, "missing row %q in:\n%s", label, table)
+ require.Greater(t, idx, last, "row %q out of order in:\n%s", label, table)
+ last = idx
+ }
+
+ assert.Contains(t, table, "6/7 PASS")
+ assert.Contains(t, table, "72/80 (72%)")
+ assert.Contains(t, table, "1/2")
+ assert.Contains(t, table, "PASS 100%")
+ assert.Contains(t, table, "WARN 1p 2f 3g")
+ assert.Contains(t, table, string(WorkflowVerdictPass))
+ assert.Contains(t, table, "2h/3e/true")
+ assert.Contains(t, table, "2s")
+}
+
func findRepoRoot() string {
dir, _ := os.Getwd()
for {
← 015ad88e refactor(cli): split scorecard scoring stages (#315)
·
back to Cli Printing Press
·
fix(skills): inherit Codex model from ~/.codex/config.toml i b54882aa →