← back to Cli Printing Press
fix(scorecard): improve accuracy for non-trivial CLIs
1bebcb0a41f0d58de6e94b322a89c021bfa310b7 · 2026-03-27 21:37:55 -0700 · Trevin Chow
The scorecard was producing scores that diverged significantly from
actual quality (e.g., 57/100 for a CLI with 91% verify pass rate).
Six root causes fixed:
- scoreSyncCorrectness/scoreDataPipelineIntegrity: search all CLI files
instead of hardcoding sync.go (sync logic often lives elsewhere)
- scoreDeadCode: count flags struct passed as arg, check intra-file
helper calls to eliminate false positives
- scoreWorkflows: expand prefix list + detect store-using commands
structurally (any command using the data layer is a workflow)
- scoreInsight: expand prefix list + detect store-querying commands
with aggregation patterns (COUNT, SUM, GROUP BY)
- RunScorecard: accept optional VerifyReport to calibrate scores
(verify pass rate sets a floor, failures cap dimensions)
🤖 Generated with Claude Opus 4.6 (1M context) via Claude Code(https://claude.com/claude-code) + Compound Engineering v2.56.0
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
M internal/cli/emboss.goM internal/cli/scorecard.goM internal/pipeline/fullrun.goM internal/pipeline/scorecard.goM internal/pipeline/scorecard_run_test.goM internal/pipeline/scorecard_tier2_test.go
Diff
commit 1bebcb0a41f0d58de6e94b322a89c021bfa310b7
Author: Trevin Chow <trevin@trevinchow.com>
Date: Fri Mar 27 21:37:55 2026 -0700
fix(scorecard): improve accuracy for non-trivial CLIs
The scorecard was producing scores that diverged significantly from
actual quality (e.g., 57/100 for a CLI with 91% verify pass rate).
Six root causes fixed:
- scoreSyncCorrectness/scoreDataPipelineIntegrity: search all CLI files
instead of hardcoding sync.go (sync logic often lives elsewhere)
- scoreDeadCode: count flags struct passed as arg, check intra-file
helper calls to eliminate false positives
- scoreWorkflows: expand prefix list + detect store-using commands
structurally (any command using the data layer is a workflow)
- scoreInsight: expand prefix list + detect store-querying commands
with aggregation patterns (COUNT, SUM, GROUP BY)
- RunScorecard: accept optional VerifyReport to calibrate scores
(verify pass rate sets a floor, failures cap dimensions)
🤖 Generated with Claude Opus 4.6 (1M context) via Claude Code(https://claude.com/claude-code) + Compound Engineering v2.56.0
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/cli/emboss.go | 2 +-
internal/cli/scorecard.go | 2 +-
internal/pipeline/fullrun.go | 2 +-
internal/pipeline/scorecard.go | 120 ++++++++++++++--
internal/pipeline/scorecard_run_test.go | 2 +-
internal/pipeline/scorecard_tier2_test.go | 225 +++++++++++++++++++++++++++++-
6 files changed, 333 insertions(+), 20 deletions(-)
diff --git a/internal/cli/emboss.go b/internal/cli/emboss.go
index 34764b90..07e730b9 100644
--- a/internal/cli/emboss.go
+++ b/internal/cli/emboss.go
@@ -170,7 +170,7 @@ func runEmbossAudit(dir, specPath, apiKey, envVar, label string) EmbossSnapshot
fmt.Fprintf(os.Stderr, " verify error: %v (continuing with partial %s)\n", err, label)
}
- scorecardReport, err := pipeline.RunScorecard(dir, "", specPath)
+ scorecardReport, err := pipeline.RunScorecard(dir, "", specPath, verifyReport)
if err != nil {
fmt.Fprintf(os.Stderr, " scorecard error: %v (continuing with partial %s)\n", err, label)
}
diff --git a/internal/cli/scorecard.go b/internal/cli/scorecard.go
index 8779bbe2..add388cc 100644
--- a/internal/cli/scorecard.go
+++ b/internal/cli/scorecard.go
@@ -34,7 +34,7 @@ func newScorecardCmd() *cobra.Command {
}
defer os.RemoveAll(pipelineDir)
- sc, err := pipeline.RunScorecard(dir, pipelineDir, specPath)
+ sc, err := pipeline.RunScorecard(dir, pipelineDir, specPath, nil)
if err != nil {
return &ExitError{Code: ExitGenerationError, Err: fmt.Errorf("running scorecard: %w", err)}
}
diff --git a/internal/pipeline/fullrun.go b/internal/pipeline/fullrun.go
index 749a2c0c..4f3225a3 100644
--- a/internal/pipeline/fullrun.go
+++ b/internal/pipeline/fullrun.go
@@ -177,7 +177,7 @@ func MakeBestCLI(apiName, level, specFlag, specURL, outputDir, pressBinary strin
}
// Step 6: Scorecard
- scorecard, scErr := RunScorecard(outputDir, pipelineDir, "")
+ scorecard, scErr := RunScorecard(outputDir, pipelineDir, "", nil)
if scErr != nil {
result.ScorecardError = scErr.Error()
result.Errors = append(result.Errors, fmt.Sprintf("scorecard: %v", scErr))
diff --git a/internal/pipeline/scorecard.go b/internal/pipeline/scorecard.go
index a7e4fb28..bfc30cf0 100644
--- a/internal/pipeline/scorecard.go
+++ b/internal/pipeline/scorecard.go
@@ -41,8 +41,9 @@ type SteinerScore struct {
SyncCorrectness int `json:"sync_correctness"` // 0-10
TypeFidelity int `json:"type_fidelity"` // 0-5
DeadCode int `json:"dead_code"` // 0-5
- Total int `json:"total"` // 0-100 (weighted: 50% infrastructure + 50% domain)
- Percentage int `json:"percentage"` // 0-100
+ Total int `json:"total"` // 0-100 (weighted: 50% infrastructure + 50% domain)
+ Percentage int `json:"percentage"` // 0-100
+ CalibrationNote string `json:"calibration_note,omitempty"`
}
// CompScore compares our score against a competitor on a single dimension.
@@ -54,7 +55,8 @@ type CompScore struct {
}
// RunScorecard evaluates generated CLI files and produces a scorecard.
-func RunScorecard(outputDir, pipelineDir, specPath string) (*Scorecard, error) {
+// If verifyReport is non-nil, verify results calibrate the final score.
+func RunScorecard(outputDir, pipelineDir, specPath string, verifyReport *VerifyReport) (*Scorecard, error) {
sc := &Scorecard{}
// Infer API name from outputDir basename
@@ -111,6 +113,25 @@ func RunScorecard(outputDir, pipelineDir, specPath string) (*Scorecard, error) {
sc.Steinberger.Percentage = sc.Steinberger.Total // Total IS the percentage (0-100)
}
+ // Calibrate against verify results when available
+ if verifyReport != nil {
+ verifyScore := int(verifyReport.PassRate * 100)
+ // Verify pass rate sets a floor: 90% verify → 72 floor
+ floor := (verifyScore * 80) / 100
+ if sc.Steinberger.Total < floor {
+ originalTotal := sc.Steinberger.Total
+ sc.Steinberger.Total = floor
+ sc.Steinberger.Percentage = floor
+ sc.Steinberger.CalibrationNote = fmt.Sprintf(
+ "Score raised from %d to %d based on %d%% verify pass rate",
+ originalTotal, floor, verifyScore)
+ }
+ // Verify failures cap data pipeline dimension
+ if !verifyReport.DataPipeline && sc.Steinberger.DataPipelineIntegrity > 5 {
+ sc.Steinberger.DataPipelineIntegrity = 5
+ }
+ }
+
// Grade
sc.OverallGrade = computeGrade(sc.Steinberger.Percentage)
@@ -641,13 +662,23 @@ func scoreWorkflows(dir string) int {
return 0
}
- workflowPrefixes := []string{"stale", "orphan", "triage", "load", "overdue", "standup", "deps", "workflow"}
+ workflowPrefixes := []string{"stale", "orphan", "triage", "load", "overdue", "standup", "deps", "workflow",
+ "agenda", "free", "conflicts", "unconfirmed", "stats", "trends", "health",
+ "reconcile", "revenue", "archive", "search", "sync", "busy", "export",
+ "noshow", "reassign", "clone"}
+
+ infra := map[string]bool{
+ "helpers.go": true, "root.go": true, "doctor.go": true, "auth.go": true,
+ }
compoundCommands := 0
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") {
continue
}
+ if infra[e.Name()] {
+ continue
+ }
name := strings.ToLower(e.Name())
@@ -666,6 +697,12 @@ func scoreWorkflows(dir string) int {
content := readFileContent(filepath.Join(cliDir, e.Name()))
+ // A command that uses the store is a workflow command (it uses the data layer)
+ if strings.Contains(content, "/store") || strings.Contains(content, "store.Open") || strings.Contains(content, "store.New") {
+ compoundCommands++
+ continue
+ }
+
// Count files that make 2+ different API calls in a single RunE.
apiCalls := 0
if strings.Contains(content, "c.Get(") || strings.Contains(content, "c.Get (") {
@@ -680,8 +717,7 @@ func scoreWorkflows(dir string) int {
if strings.Contains(content, "c.Delete(") || strings.Contains(content, "c.Delete (") {
apiCalls++
}
- // Also count store operations as compound behavior.
- if strings.Contains(content, "store.") || strings.Contains(content, "/store") {
+ if strings.Contains(content, "store.") {
apiCalls++
}
if apiCalls >= 2 {
@@ -712,19 +748,47 @@ func scoreInsight(dir string) int {
return 0
}
- insightPrefixes := []string{"health", "similar", "bottleneck", "trends", "patterns", "forecast"}
+ insightPrefixes := []string{"health", "similar", "bottleneck", "trends", "patterns", "forecast",
+ "stats", "conflicts", "stale", "analytics", "busiest", "velocity",
+ "utilization", "coverage", "gaps", "noshow"}
+
+ infra := map[string]bool{
+ "helpers.go": true, "root.go": true, "doctor.go": true, "auth.go": true,
+ }
+
found := 0
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") {
continue
}
+ if infra[e.Name()] {
+ continue
+ }
name := strings.ToLower(e.Name())
+
+ // Check prefix match
+ prefixMatch := false
for _, prefix := range insightPrefixes {
if strings.HasPrefix(name, prefix) {
- found++
+ prefixMatch = true
break
}
}
+ if prefixMatch {
+ found++
+ continue
+ }
+
+ // Structural detection: commands that query the store and produce
+ // aggregated/computed output (COUNT, SUM, GROUP BY, etc.) are insights
+ content := readFileContent(filepath.Join(cliDir, e.Name()))
+ usesStore := strings.Contains(content, "/store") || strings.Contains(content, "store.Open") || strings.Contains(content, "store.New")
+ hasAggregation := strings.Contains(content, "COUNT(") || strings.Contains(content, "SUM(") ||
+ strings.Contains(content, "GROUP BY") || strings.Contains(content, "AVG(") ||
+ strings.Contains(content, "rate") || strings.Contains(content, "Rate")
+ if usesStore && hasAggregation {
+ found++
+ }
}
switch {
@@ -912,8 +976,9 @@ func scoreAuthProtocol(dir, specPath string) int {
func scoreDataPipelineIntegrity(dir string) int {
score := 0
- syncContent := readFileContent(filepath.Join(dir, "internal", "cli", "sync.go"))
- searchContent := readFileContent(filepath.Join(dir, "internal", "cli", "search.go"))
+ cliDir := filepath.Join(dir, "internal", "cli")
+ syncContent := readAllGoFiles(cliDir)
+ searchContent := syncContent // search patterns also live in any CLI file
storeContent := readFileContent(filepath.Join(dir, "internal", "store", "store.go"))
if syncContent != "" && (strings.Contains(syncContent, "/store") || strings.Contains(syncContent, "store.")) {
@@ -944,7 +1009,8 @@ func scoreDataPipelineIntegrity(dir string) int {
}
func scoreSyncCorrectness(dir string) int {
- content := readFileContent(filepath.Join(dir, "internal", "cli", "sync.go"))
+ cliDir := filepath.Join(dir, "internal", "cli")
+ content := readAllGoFiles(cliDir)
if content == "" {
return 0
}
@@ -1041,17 +1107,24 @@ func scoreDeadCode(dir string) int {
flagRe := regexp.MustCompile(`&flags\.(\w+)`)
flagNames := uniqueMatches(flagRe, rootContent)
otherCLI := readOtherGoFiles(cliDir, map[string]bool{"root.go": true})
- for _, name := range flagNames {
- if !strings.Contains(otherCLI, "flags."+name) {
- deadFlags++
+
+ // If the flags struct is passed as a function argument, all fields are reachable
+ flagsPassedAsArg := strings.Contains(otherCLI, "flags,") || strings.Contains(otherCLI, "flags)")
+ if !flagsPassedAsArg {
+ for _, name := range flagNames {
+ if !strings.Contains(otherCLI, "flags."+name) {
+ deadFlags++
+ }
}
}
funcRe := regexp.MustCompile(`(?m)^func\s+([A-Za-z_]\w*)\s*\(`)
funcNames := uniqueMatches(funcRe, helpersContent)
otherHelpers := readOtherGoFiles(cliDir, map[string]bool{"helpers.go": true})
+ // Check both other files AND helpers.go itself for intra-file calls
+ allContent := helpersContent + "\n" + otherHelpers
for _, name := range funcNames {
- if !strings.Contains(otherHelpers, name+"(") {
+ if !strings.Contains(allContent, name+"(") {
deadFunctions++
}
}
@@ -1208,6 +1281,23 @@ func uniqueMatches(re *regexp.Regexp, content string) []string {
return out
}
+// readAllGoFiles concatenates the content of all .go files in dir.
+func readAllGoFiles(dir string) string {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return ""
+ }
+ var b strings.Builder
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") {
+ continue
+ }
+ b.WriteString(readFileContent(filepath.Join(dir, entry.Name())))
+ b.WriteByte('\n')
+ }
+ return b.String()
+}
+
func readOtherGoFiles(dir string, skip map[string]bool) string {
entries, err := os.ReadDir(dir)
if err != nil {
diff --git a/internal/pipeline/scorecard_run_test.go b/internal/pipeline/scorecard_run_test.go
index 82796603..291560fa 100644
--- a/internal/pipeline/scorecard_run_test.go
+++ b/internal/pipeline/scorecard_run_test.go
@@ -16,7 +16,7 @@ func TestScorecardOnRealCLI(t *testing.T) {
pipelineDir = t.TempDir()
}
- sc, err := RunScorecard(outputDir, pipelineDir, "")
+ sc, err := RunScorecard(outputDir, pipelineDir, "", nil)
if err != nil {
t.Fatalf("RunScorecard: %v", err)
}
diff --git a/internal/pipeline/scorecard_tier2_test.go b/internal/pipeline/scorecard_tier2_test.go
index ca4870fb..c3dc6fb6 100644
--- a/internal/pipeline/scorecard_tier2_test.go
+++ b/internal/pipeline/scorecard_tier2_test.go
@@ -44,7 +44,8 @@ func filterFields() {}
func outputCSV() {}
`)
- assert.Equal(t, 1, scoreDeadCode(dir))
+ // 2 dead flags (csvOutput, stdinInput), 0 dead functions (intra-file calls now counted)
+ assert.Equal(t, 3, scoreDeadCode(dir))
})
t.Run("returns full score when nothing is dead", func(t *testing.T) {
@@ -245,6 +246,228 @@ func init() {
})
}
+func TestScoreSyncCorrectness_NonSyncFilename(t *testing.T) {
+ t.Run("finds sync patterns in non-sync.go files", func(t *testing.T) {
+ dir := t.TempDir()
+
+ // Sync logic lives in channel_workflow.go, not sync.go
+ writeScorecardFixture(t, dir, "internal/cli/channel_workflow.go", `
+package cli
+
+func defaultSyncResources() []string {
+ return []string{"bookings", "event_types"}
+}
+
+func runChannelSync(store interface {
+ GetSyncState(string) string
+ SaveSyncState(string, string)
+}) {
+ path := "/v2/bookings"
+ cursor := store.GetSyncState("bookings")
+ paginatedGet(path, cursor)
+ store.SaveSyncState("bookings", "next")
+}
+`)
+
+ score := scoreSyncCorrectness(dir)
+ assert.GreaterOrEqual(t, score, 8, "sync logic in non-sync.go should score high")
+ })
+}
+
+func TestScoreDataPipelineIntegrity_NonSyncFilename(t *testing.T) {
+ t.Run("finds upsert patterns in non-sync.go files", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/sync_cmd.go", `
+package cli
+
+import "example.com/project/internal/store"
+
+func runSync(db *store.DB) {
+ _ = db.UpsertBooking(nil)
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/search_cmd.go", `
+package cli
+
+func runSearch(db interface{ SearchBookings(string) error }) {
+ _ = db.SearchBookings("term")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/store/store.go", `
+package store
+
+const schema = `+"`"+`
+CREATE TABLE bookings (
+ id TEXT,
+ user_id TEXT,
+ event_type_id TEXT,
+ title TEXT,
+ start_time TEXT,
+ end_time TEXT
+);
+`+"`"+`
+`)
+
+ score := scoreDataPipelineIntegrity(dir)
+ assert.GreaterOrEqual(t, score, 7, "domain upserts in non-sync.go should score high")
+ })
+}
+
+func TestScoreDeadCode_FlagsPassedAsArg(t *testing.T) {
+ t.Run("flags struct passed to function counts all fields as used", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `
+package cli
+
+var flags struct {
+ asJSON bool
+ csvOutput bool
+ verbose bool
+}
+
+func init() {
+ rootCmd.Flags().BoolVar(&flags.asJSON, "json", false, "JSON output")
+ rootCmd.Flags().BoolVar(&flags.csvOutput, "csv", false, "CSV output")
+ rootCmd.Flags().BoolVar(&flags.verbose, "verbose", false, "Verbose")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+func runMessages() {
+ printOutput(cmd, flags, data, statusCode)
+}
+`)
+
+ assert.Equal(t, 5, scoreDeadCode(dir))
+ })
+}
+
+func TestScoreDeadCode_IntraFileHelperCalls(t *testing.T) {
+ t.Run("helpers calling other helpers are not dead", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `
+package cli
+`)
+ writeScorecardFixture(t, dir, "internal/cli/helpers.go", `
+package cli
+
+func formatOutput(data interface{}) string {
+ return applyFormat(data)
+}
+
+func applyFormat(data interface{}) string {
+ return ""
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+func runMessages() {
+ formatOutput(data)
+}
+`)
+
+ assert.Equal(t, 5, scoreDeadCode(dir))
+ })
+}
+
+func TestScorecard_VerifyCalibration(t *testing.T) {
+ t.Run("verify pass rate sets floor on total score", func(t *testing.T) {
+ dir := t.TempDir()
+
+ // Minimal CLI that would score low on static analysis
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `
+package cli
+`)
+ writeScorecardFixture(t, dir, "internal/cli/helpers.go", `
+package cli
+`)
+ writeScorecardFixture(t, dir, "README.md", `# Test CLI`)
+
+ pipelineDir := t.TempDir()
+ verifyReport := &VerifyReport{
+ PassRate: 0.91,
+ Total: 33,
+ Passed: 30,
+ DataPipeline: true,
+ Verdict: "PASS",
+ }
+
+ sc, err := RunScorecard(dir, pipelineDir, "", verifyReport)
+ assert.NoError(t, err)
+ // 91% * 80 / 100 = 72 floor
+ assert.GreaterOrEqual(t, sc.Steinberger.Total, 72)
+ assert.Contains(t, sc.Steinberger.CalibrationNote, "verify pass rate")
+ })
+
+ t.Run("verify failure caps data pipeline dimension", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `package cli`)
+ writeScorecardFixture(t, dir, "internal/cli/sync.go", `
+package cli
+
+import "example.com/project/internal/store"
+
+func runSync(db *store.DB) {
+ _ = db.UpsertBooking(nil)
+}
+
+func defaultSyncResources() []string {
+ return []string{"bookings"}
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/search.go", `
+package cli
+
+func runSearch(db interface{ SearchBookings(string) error }) {
+ _ = db.SearchBookings("term")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/store/store.go", `
+package store
+
+const schema = `+"`"+`
+CREATE TABLE bookings (
+ id TEXT,
+ user_id TEXT,
+ event_type_id TEXT,
+ title TEXT,
+ start_time TEXT,
+ end_time TEXT
+);
+`+"`"+`
+`)
+
+ pipelineDir := t.TempDir()
+ verifyReport := &VerifyReport{
+ PassRate: 0.50,
+ DataPipeline: false,
+ Verdict: "FAIL",
+ }
+
+ sc, err := RunScorecard(dir, pipelineDir, "", verifyReport)
+ assert.NoError(t, err)
+ assert.LessOrEqual(t, sc.Steinberger.DataPipelineIntegrity, 5)
+ })
+
+ t.Run("nil verify report has no effect", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `package cli`)
+ writeScorecardFixture(t, dir, "README.md", `# Test`)
+
+ pipelineDir := t.TempDir()
+ sc, err := RunScorecard(dir, pipelineDir, "", nil)
+ assert.NoError(t, err)
+ assert.Empty(t, sc.Steinberger.CalibrationNote)
+ })
+}
+
func writeScorecardFixture(t *testing.T, root, relPath, content string) {
t.Helper()
← 32a4ec90 docs: rename score command spec to -plan suffix
·
back to Cli Printing Press
·
feat(pipeline): copy spec into output dir after generation a00ebdb2 →