← back to Cli Printing Press
feat(scorecard): add Tier 2 domain correctness dimensions
bba34c4295a27b3b25f1f8358b648b947f3aad90 · 2026-03-26 12:03:30 -0700 · Matt Van Horn
Add 6 new semantic scoring dimensions that test behavior, not string presence:
- PathValidity: validates generated paths exist in the OpenAPI spec
- AuthProtocol: checks auth format matches spec's securitySchemes
- DataPipelineIntegrity: verifies sync calls domain-specific upserts
- SyncCorrectness: checks sync has real resources, nested paths, pagination
- TypeFidelity: validates flag types match spec params (string IDs not int)
- DeadCode: detects unwired flags and uncalled functions
Rebalances scoring: Tier 1 (infrastructure) and Tier 2 (domain) each
contribute 50% of the 100-point total. This prevents infrastructure
scores from masking broken domain logic.
Results: discord-cli drops from 96/110 Grade A to 55/100 Grade C.
linear-cli drops from 91/110 Grade B to 64/100 Grade C.
Both are honest - infrastructure is good, domain needs work.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
M internal/cli/scorecard.goM internal/pipeline/fullrun.goM internal/pipeline/scorecard.goM internal/pipeline/scorecard_run_test.goA internal/pipeline/scorecard_tier2_test.go
Diff
commit bba34c4295a27b3b25f1f8358b648b947f3aad90
Author: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Date: Thu Mar 26 12:03:30 2026 -0700
feat(scorecard): add Tier 2 domain correctness dimensions
Add 6 new semantic scoring dimensions that test behavior, not string presence:
- PathValidity: validates generated paths exist in the OpenAPI spec
- AuthProtocol: checks auth format matches spec's securitySchemes
- DataPipelineIntegrity: verifies sync calls domain-specific upserts
- SyncCorrectness: checks sync has real resources, nested paths, pagination
- TypeFidelity: validates flag types match spec params (string IDs not int)
- DeadCode: detects unwired flags and uncalled functions
Rebalances scoring: Tier 1 (infrastructure) and Tier 2 (domain) each
contribute 50% of the 100-point total. This prevents infrastructure
scores from masking broken domain logic.
Results: discord-cli drops from 96/110 Grade A to 55/100 Grade C.
linear-cli drops from 91/110 Grade B to 64/100 Grade C.
Both are honest - infrastructure is good, domain needs work.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/cli/scorecard.go | 16 +-
internal/pipeline/fullrun.go | 18 +-
internal/pipeline/scorecard.go | 535 +++++++++++++++++++++++++++++-
internal/pipeline/scorecard_run_test.go | 2 +-
internal/pipeline/scorecard_tier2_test.go | 261 +++++++++++++++
5 files changed, 809 insertions(+), 23 deletions(-)
diff --git a/internal/cli/scorecard.go b/internal/cli/scorecard.go
index 415d3acc..ed930876 100644
--- a/internal/cli/scorecard.go
+++ b/internal/cli/scorecard.go
@@ -11,11 +11,12 @@ import (
func newScorecardCmd() *cobra.Command {
var dir string
+ var specPath string
var asJSON bool
cmd := &cobra.Command{
Use: "scorecard",
- Short: "Score a generated CLI against the Steinberger bar (10 dimensions, max 100)",
+ Short: "Score a generated CLI against the Steinberger bar",
RunE: func(cmd *cobra.Command, args []string) error {
if dir == "" {
return fmt.Errorf("--dir is required")
@@ -28,7 +29,7 @@ func newScorecardCmd() *cobra.Command {
}
defer os.RemoveAll(pipelineDir)
- sc, err := pipeline.RunScorecard(dir, pipelineDir)
+ sc, err := pipeline.RunScorecard(dir, pipelineDir, specPath)
if err != nil {
return fmt.Errorf("running scorecard: %w", err)
}
@@ -53,7 +54,15 @@ func newScorecardCmd() *cobra.Command {
fmt.Printf(" Breadth %d/10\n", s.Breadth)
fmt.Printf(" Vision %d/10\n", s.Vision)
fmt.Printf(" Workflows %d/10\n", s.Workflows)
- fmt.Printf("\n Total: %d/110 (%d%%) - Grade %s\n", s.Total, s.Percentage, sc.OverallGrade)
+ fmt.Printf(" Insight %d/10\n", s.Insight)
+ fmt.Printf("\n Domain Correctness\n")
+ fmt.Printf(" Path Validity %d/10\n", s.PathValidity)
+ fmt.Printf(" Auth Protocol %d/10\n", s.AuthProtocol)
+ fmt.Printf(" Data Pipeline Integrity %d/10\n", s.DataPipelineIntegrity)
+ fmt.Printf(" Sync Correctness %d/10\n", s.SyncCorrectness)
+ fmt.Printf(" Type Fidelity %d/5\n", s.TypeFidelity)
+ fmt.Printf(" Dead Code %d/5\n", s.DeadCode)
+ fmt.Printf("\n Total: %d/100 - Grade %s\n", s.Total, sc.OverallGrade)
if len(sc.GapReport) > 0 {
fmt.Printf("\nGaps:\n")
@@ -67,6 +76,7 @@ func newScorecardCmd() *cobra.Command {
}
cmd.Flags().StringVar(&dir, "dir", "", "Path to generated CLI directory")
+ cmd.Flags().StringVar(&specPath, "spec", "", "Path to OpenAPI spec JSON for semantic validation")
cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON")
return cmd
diff --git a/internal/pipeline/fullrun.go b/internal/pipeline/fullrun.go
index b216a883..d4a530cb 100644
--- a/internal/pipeline/fullrun.go
+++ b/internal/pipeline/fullrun.go
@@ -13,20 +13,20 @@ import (
// FullRunResult holds everything the press produced for one API.
type FullRunResult struct {
- APIName string
- Level string // "EASY", "MEDIUM", "HARD"
+ APIName string
+ Level string // "EASY", "MEDIUM", "HARD"
// Step 1: Research
Research *ResearchResult
ResearchError string
// Step 2: Generate
- OutputDir string
- GatesPassed int
- GatesFailed int
- GatesOutput string
- CommandCount int
- ResourceCount int
+ OutputDir string
+ GatesPassed int
+ GatesFailed int
+ GatesOutput string
+ CommandCount int
+ ResourceCount int
// Step 3: Coverage
SpecEndpoints int
@@ -148,7 +148,7 @@ func MakeBestCLI(apiName, level, specFlag, specURL, outputDir, pressBinary strin
}
// Step 6: Scorecard
- scorecard, scErr := RunScorecard(outputDir, pipelineDir)
+ scorecard, scErr := RunScorecard(outputDir, pipelineDir, "")
if scErr != nil {
result.ScorecardError = scErr.Error()
result.Errors = append(result.Errors, fmt.Sprintf("scorecard: %v", scErr))
diff --git a/internal/pipeline/scorecard.go b/internal/pipeline/scorecard.go
index 3d65d3a7..35487456 100644
--- a/internal/pipeline/scorecard.go
+++ b/internal/pipeline/scorecard.go
@@ -5,6 +5,9 @@ import (
"fmt"
"os"
"path/filepath"
+ "regexp"
+ "slices"
+ "strconv"
"strings"
)
@@ -31,8 +34,15 @@ type SteinerScore struct {
Vision int `json:"vision"` // 0-10
Workflows int `json:"workflows"` // 0-10
Insight int `json:"insight"` // 0-10
- Total int `json:"total"` // 0-120
- Percentage int `json:"percentage"` // 0-100
+ // Tier 2: Domain Correctness (semantic checks)
+ PathValidity int `json:"path_validity"` // 0-10
+ AuthProtocol int `json:"auth_protocol"` // 0-10
+ DataPipelineIntegrity int `json:"data_pipeline_integrity"` // 0-10
+ SyncCorrectness int `json:"sync_correctness"` // 0-10
+ TypeFidelity int `json:"type_fidelity"` // 0-5
+ DeadCode int `json:"dead_code"` // 0-5
+ Total int `json:"total"` // 0-100 (weighted: 50% infrastructure + 50% domain)
+ Percentage int `json:"percentage"` // 0-100
}
// CompScore compares our score against a competitor on a single dimension.
@@ -44,7 +54,7 @@ type CompScore struct {
}
// RunScorecard evaluates generated CLI files and produces a scorecard.
-func RunScorecard(outputDir, pipelineDir string) (*Scorecard, error) {
+func RunScorecard(outputDir, pipelineDir, specPath string) (*Scorecard, error) {
sc := &Scorecard{}
// Infer API name from outputDir basename
@@ -63,8 +73,15 @@ func RunScorecard(outputDir, pipelineDir string) (*Scorecard, error) {
sc.Steinberger.Vision = scoreVision(outputDir)
sc.Steinberger.Workflows = scoreWorkflows(outputDir)
sc.Steinberger.Insight = scoreInsight(outputDir)
+ sc.Steinberger.PathValidity = scorePathValidity(outputDir, specPath)
+ sc.Steinberger.AuthProtocol = scoreAuthProtocol(outputDir, specPath)
+ sc.Steinberger.DataPipelineIntegrity = scoreDataPipelineIntegrity(outputDir)
+ sc.Steinberger.SyncCorrectness = scoreSyncCorrectness(outputDir)
+ sc.Steinberger.TypeFidelity = scoreTypeFidelity(outputDir)
+ sc.Steinberger.DeadCode = scoreDeadCode(outputDir)
- sc.Steinberger.Total = sc.Steinberger.OutputModes +
+ // Tier 1: Infrastructure (string-matching, 120 max)
+ tier1Raw := sc.Steinberger.OutputModes +
sc.Steinberger.Auth +
sc.Steinberger.ErrorHandling +
sc.Steinberger.TerminalUX +
@@ -77,8 +94,21 @@ func RunScorecard(outputDir, pipelineDir string) (*Scorecard, error) {
sc.Steinberger.Workflows +
sc.Steinberger.Insight
+ // Tier 2: Domain Correctness (semantic, 50 max)
+ tier2Raw := sc.Steinberger.PathValidity +
+ sc.Steinberger.AuthProtocol +
+ sc.Steinberger.DataPipelineIntegrity +
+ sc.Steinberger.SyncCorrectness +
+ sc.Steinberger.TypeFidelity +
+ sc.Steinberger.DeadCode
+
+ // Weighted composite: Tier 1 = 50%, Tier 2 = 50% of final 100-point scale
+ tier1Normalized := (tier1Raw * 50) / 120 // scale 0-120 to 0-50
+ tier2Normalized := (tier2Raw * 50) / 50 // scale 0-50 to 0-50
+ sc.Steinberger.Total = tier1Normalized + tier2Normalized
+
if sc.Steinberger.Total > 0 {
- sc.Steinberger.Percentage = (sc.Steinberger.Total * 100) / 120
+ sc.Steinberger.Percentage = sc.Steinberger.Total // Total IS the percentage (0-100)
}
// Grade
@@ -715,6 +745,324 @@ func scoreInsight(dir string) int {
}
}
+type openAPISecurityScheme struct {
+ Name string
+ Type string
+ In string
+}
+
+type openAPISpecInfo struct {
+ Paths []string
+ SecuritySchemes []openAPISecurityScheme
+}
+
+func loadOpenAPISpec(specPath string) *openAPISpecInfo {
+ if specPath == "" {
+ return nil
+ }
+
+ data, err := os.ReadFile(specPath)
+ if err != nil {
+ return nil
+ }
+
+ var raw map[string]any
+ if err := json.Unmarshal(data, &raw); err != nil {
+ return nil
+ }
+
+ info := &openAPISpecInfo{}
+
+ if paths, ok := raw["paths"].(map[string]any); ok {
+ for path := range paths {
+ info.Paths = append(info.Paths, path)
+ }
+ slices.Sort(info.Paths)
+ }
+
+ if components, ok := raw["components"].(map[string]any); ok {
+ if securitySchemes, ok := components["securitySchemes"].(map[string]any); ok {
+ for schemeName, value := range securitySchemes {
+ scheme := openAPISecurityScheme{Name: schemeName}
+ if fields, ok := value.(map[string]any); ok {
+ scheme.Type = strings.ToLower(asString(fields["scheme"]))
+ if scheme.Type == "" {
+ scheme.Type = strings.ToLower(asString(fields["type"]))
+ }
+ scheme.In = strings.ToLower(asString(fields["in"]))
+ }
+ info.SecuritySchemes = append(info.SecuritySchemes, scheme)
+ }
+ }
+ }
+
+ return info
+}
+
+func scorePathValidity(dir, specPath string) int {
+ if specPath == "" {
+ return 5
+ }
+
+ spec := loadOpenAPISpec(specPath)
+ if spec == nil || len(spec.Paths) == 0 {
+ return 5
+ }
+
+ pathRe := regexp.MustCompile(`\bpath\s*[:=]\s*"([^"]+)"`)
+ cmdFiles := sampleCommandFiles(dir, 10)
+ if len(cmdFiles) == 0 {
+ return 0
+ }
+
+ total := 0
+ matches := 0
+ for _, content := range cmdFiles {
+ match := pathRe.FindStringSubmatch(content)
+ if len(match) < 2 {
+ continue
+ }
+ total++
+ if specPathExists(spec.Paths, match[1]) {
+ matches++
+ }
+ }
+
+ if total == 0 {
+ return 0
+ }
+ return (matches * 10) / total
+}
+
+func scoreAuthProtocol(dir, specPath string) int {
+ if specPath == "" {
+ return 5
+ }
+
+ spec := loadOpenAPISpec(specPath)
+ if spec == nil || len(spec.SecuritySchemes) == 0 {
+ return 5
+ }
+
+ clientContent := readFileContent(filepath.Join(dir, "internal", "client", "client.go"))
+ configContent := readFileContent(filepath.Join(dir, "internal", "config", "config.go"))
+ if clientContent == "" {
+ return 0
+ }
+
+ score := 0
+ authHeaderMatched := false
+ headerNameMatched := false
+ queryMatched := false
+ envMatched := false
+
+ for _, scheme := range spec.SecuritySchemes {
+ nameLower := strings.ToLower(scheme.Name)
+ switch {
+ case strings.Contains(nameLower, "bot"):
+ if strings.Contains(clientContent, `"Bot "`) || strings.Contains(clientContent, "`Bot `") {
+ authHeaderMatched = true
+ }
+ case strings.Contains(nameLower, "bearer") || scheme.Type == "bearer" || scheme.Type == "http":
+ if strings.Contains(clientContent, `"Bearer "`) || strings.Contains(clientContent, "`Bearer `") {
+ authHeaderMatched = true
+ }
+ case strings.Contains(nameLower, "basic") || scheme.Type == "basic":
+ if strings.Contains(clientContent, `"Basic "`) || strings.Contains(clientContent, "`Basic `") {
+ authHeaderMatched = true
+ }
+ }
+
+ headerName := "Authorization"
+ if strings.Contains(nameLower, "bot") {
+ headerName = "Authorization"
+ }
+ if strings.Contains(clientContent, `Header.Set("`+headerName+`"`) ||
+ strings.Contains(clientContent, `Header.Add("`+headerName+`"`) {
+ headerNameMatched = true
+ }
+
+ if scheme.In == "query" && (strings.Contains(clientContent, ".Query()") || strings.Contains(clientContent, "url.Values") || strings.Contains(clientContent, "RawQuery")) {
+ queryMatched = true
+ }
+
+ envNeedle := sanitizeEnvName(scheme.Name)
+ if envNeedle != "" && strings.Contains(strings.ToUpper(configContent), envNeedle) {
+ envMatched = true
+ }
+ }
+
+ if authHeaderMatched {
+ score += 3
+ }
+ if headerNameMatched {
+ score += 3
+ }
+ if queryMatched {
+ score += 2
+ }
+ if envMatched {
+ score += 2
+ }
+ if score > 10 {
+ score = 10
+ }
+ return score
+}
+
+func scoreDataPipelineIntegrity(dir string) int {
+ score := 0
+ syncContent := readFileContent(filepath.Join(dir, "internal", "cli", "sync.go"))
+ searchContent := readFileContent(filepath.Join(dir, "internal", "cli", "search.go"))
+ storeContent := readFileContent(filepath.Join(dir, "internal", "store", "store.go"))
+
+ if syncContent != "" && (strings.Contains(syncContent, "/store") || strings.Contains(syncContent, "store.")) {
+ score++
+ }
+
+ domainUpsertRe := regexp.MustCompile(`\.Upsert[A-Z]\w*\(`)
+ genericUpsertRe := regexp.MustCompile(`\.Upsert\(`)
+ if domainUpsertRe.MatchString(syncContent) {
+ score += 3
+ } else if genericUpsertRe.MatchString(syncContent) {
+ score += 0
+ }
+
+ domainSearchRe := regexp.MustCompile(`\.Search[A-Z]\w*\(`)
+ genericSearchRe := regexp.MustCompile(`\.Search\(`)
+ if domainSearchRe.MatchString(searchContent) {
+ score += 3
+ } else if genericSearchRe.MatchString(searchContent) {
+ score += 0
+ }
+
+ score += scoreDomainTables(storeContent)
+ if score > 10 {
+ score = 10
+ }
+ return score
+}
+
+func scoreSyncCorrectness(dir string) int {
+ content := readFileContent(filepath.Join(dir, "internal", "cli", "sync.go"))
+ if content == "" {
+ return 0
+ }
+
+ score := 0
+ if hasNonEmptySyncResources(content) {
+ score += 2
+ }
+ if strings.Contains(content, "{") {
+ score += 3
+ }
+ if strings.Contains(content, "GetSyncState") || strings.Contains(content, "sync_state") {
+ score += 2
+ }
+ if strings.Contains(content, "SaveSyncState") {
+ score++
+ }
+ if strings.Contains(content, "paginatedGet") || strings.Contains(content, "hasNextPage") || strings.Contains(content, "endCursor") || strings.Contains(content, "cursor") {
+ score += 2
+ }
+ if score > 10 {
+ score = 10
+ }
+ return score
+}
+
+func scoreTypeFidelity(dir string) int {
+ score := 0
+ cmdFiles := sampleCommandFiles(dir, 10)
+ if len(cmdFiles) == 0 {
+ return 0
+ }
+
+ flagDeclRe := regexp.MustCompile(`Flags\(\)\.(StringVar|IntVar|StringVarP|IntVarP)\(&[^,]+,\s*"([^"]+)"(?:,\s*[^,]+){1,2},\s*"([^"]*)"`)
+ requiredRe := regexp.MustCompile(`MarkFlagRequired\("([^"]+)"\)`)
+
+ totalIDFlags := 0
+ stringIDFlags := 0
+ requiredCount := 0
+ descWordCount := 0
+ descCount := 0
+
+ for _, content := range cmdFiles {
+ for _, match := range flagDeclRe.FindAllStringSubmatch(content, -1) {
+ name := strings.ToLower(match[2])
+ if strings.Contains(name, "id") {
+ totalIDFlags++
+ if strings.HasPrefix(match[1], "StringVar") {
+ stringIDFlags++
+ }
+ }
+ descWordCount += len(strings.Fields(match[3]))
+ descCount++
+ }
+ requiredCount += len(requiredRe.FindAllStringSubmatch(content, -1))
+ }
+
+ if totalIDFlags == 0 || stringIDFlags == totalIDFlags {
+ score += 2
+ }
+ if requiredCount >= 3 {
+ score++
+ }
+ if descCount > 0 && descWordCount/descCount > 5 {
+ score++
+ }
+
+ allCLI := ""
+ for _, content := range sampleCommandFiles(dir, 0) {
+ allCLI += content
+ }
+ allCLI += readFileContent(filepath.Join(dir, "internal", "cli", "helpers.go"))
+ allCLI += readFileContent(filepath.Join(dir, "internal", "cli", "root.go"))
+ if !strings.Contains(allCLI, "var _ = strings.ReplaceAll") && !strings.Contains(allCLI, "var _ = fmt.Sprintf") {
+ score++
+ }
+
+ if score > 5 {
+ score = 5
+ }
+ return score
+}
+
+func scoreDeadCode(dir string) int {
+ deadFlags := 0
+ deadFunctions := 0
+ cliDir := filepath.Join(dir, "internal", "cli")
+ rootContent := readFileContent(filepath.Join(cliDir, "root.go"))
+ helpersContent := readFileContent(filepath.Join(cliDir, "helpers.go"))
+ if rootContent == "" && helpersContent == "" {
+ return 0
+ }
+
+ flagRe := regexp.MustCompile(`&flags\.(\w+)`)
+ flagNames := uniqueMatches(flagRe, rootContent)
+ otherCLI := readOtherGoFiles(cliDir, map[string]bool{"root.go": true})
+ for _, name := range flagNames {
+ if !strings.Contains(otherCLI, "flags."+name) {
+ deadFlags++
+ }
+ }
+
+ funcRe := regexp.MustCompile(`(?m)^func\s+([A-Za-z_]\w*)\s*\(`)
+ funcNames := uniqueMatches(funcRe, helpersContent)
+ otherHelpers := readOtherGoFiles(cliDir, map[string]bool{"helpers.go": true})
+ for _, name := range funcNames {
+ if !strings.Contains(otherHelpers, name+"(") {
+ deadFunctions++
+ }
+ }
+
+ score := 5 - (deadFlags + deadFunctions)
+ if score < 0 {
+ return 0
+ }
+ return score
+}
+
// sampleCommandFiles reads up to n command files from internal/cli/.
// If n <= 0, reads all command files.
func sampleCommandFiles(dir string, n int) []string {
@@ -747,6 +1095,148 @@ func sampleCommandFiles(dir string, n int) []string {
return files
}
+func specPathExists(specPaths []string, actual string) bool {
+ for _, candidate := range specPaths {
+ if matchSpecPath(candidate, actual) || matchSpecPath(actual, candidate) {
+ return true
+ }
+ }
+ return false
+}
+
+func matchSpecPath(pattern, actual string) bool {
+ patternParts := splitPath(pattern)
+ actualParts := splitPath(actual)
+ if len(patternParts) != len(actualParts) {
+ return false
+ }
+ for i := range patternParts {
+ part := patternParts[i]
+ if strings.HasPrefix(part, "{") && strings.HasSuffix(part, "}") {
+ continue
+ }
+ if part != actualParts[i] {
+ return false
+ }
+ }
+ return true
+}
+
+func splitPath(path string) []string {
+ trimmed := strings.Trim(path, "/")
+ if trimmed == "" {
+ return nil
+ }
+ return strings.Split(trimmed, "/")
+}
+
+func sanitizeEnvName(name string) string {
+ name = strings.ToUpper(name)
+ var b strings.Builder
+ lastUnderscore := false
+ for _, r := range name {
+ if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+ b.WriteRune(r)
+ lastUnderscore = false
+ continue
+ }
+ if !lastUnderscore {
+ b.WriteByte('_')
+ lastUnderscore = true
+ }
+ }
+ return strings.Trim(b.String(), "_")
+}
+
+func scoreDomainTables(storeContent string) int {
+ if storeContent == "" {
+ return 0
+ }
+ createTableRe := regexp.MustCompile(`(?is)CREATE TABLE[^()]*\((.*?)\)`)
+ columnTables := 0
+ for _, match := range createTableRe.FindAllStringSubmatch(storeContent, -1) {
+ columnCount := 0
+ for _, line := range strings.Split(match[1], "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || strings.HasPrefix(line, "--") {
+ continue
+ }
+ upper := strings.ToUpper(line)
+ if strings.HasPrefix(upper, "PRIMARY KEY") || strings.HasPrefix(upper, "FOREIGN KEY") || strings.HasPrefix(upper, "UNIQUE") || strings.HasPrefix(upper, "CONSTRAINT") {
+ continue
+ }
+ columnCount++
+ }
+ if columnCount >= 5 {
+ columnTables++
+ }
+ }
+ if columnTables > 0 {
+ return 3
+ }
+ return 0
+}
+
+func hasNonEmptySyncResources(content string) bool {
+ if strings.Contains(content, "[]string{}") || strings.Contains(content, "return nil") {
+ return false
+ }
+ if strings.Contains(content, "defaultSyncResources()") || strings.Contains(content, "syncResources") {
+ listRe := regexp.MustCompile(`\[\]string\{([^}]*)\}`)
+ for _, match := range listRe.FindAllStringSubmatch(content, -1) {
+ if strings.TrimSpace(match[1]) != "" {
+ return true
+ }
+ }
+ if strings.Contains(content, "defaultSyncResources") {
+ return true
+ }
+ }
+ return false
+}
+
+func uniqueMatches(re *regexp.Regexp, content string) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, match := range re.FindAllStringSubmatch(content, -1) {
+ if len(match) < 2 || seen[match[1]] {
+ continue
+ }
+ seen[match[1]] = true
+ out = append(out, match[1])
+ }
+ return out
+}
+
+func readOtherGoFiles(dir string, skip map[string]bool) string {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return ""
+ }
+ var b strings.Builder
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || skip[entry.Name()] {
+ continue
+ }
+ b.WriteString(readFileContent(filepath.Join(dir, entry.Name())))
+ b.WriteByte('\n')
+ }
+ return b.String()
+}
+
+func asString(v any) string {
+ switch value := v.(type) {
+ case string:
+ return value
+ case fmt.Stringer:
+ return value.String()
+ case float64:
+ return strconv.FormatFloat(value, 'f', -1, 64)
+ default:
+ return ""
+ }
+}
+
// hasPlaceholderValues checks if file content contains common placeholder values
// that indicate unpolished examples.
func hasPlaceholderValues(content string) bool {
@@ -847,10 +1337,20 @@ func buildGapReport(s SteinerScore) []string {
{"vision", s.Vision},
{"workflows", s.Workflows},
{"insight", s.Insight},
+ {"path_validity", s.PathValidity},
+ {"auth_protocol", s.AuthProtocol},
+ {"data_pipeline_integrity", s.DataPipelineIntegrity},
+ {"sync_correctness", s.SyncCorrectness},
+ {"type_fidelity", s.TypeFidelity},
+ {"dead_code", s.DeadCode},
}
for _, d := range dimensions {
- if d.score < 5 {
- gaps = append(gaps, fmt.Sprintf("%s scored %d/10 - needs improvement", d.name, d.score))
+ max := 10
+ if d.name == "type_fidelity" || d.name == "dead_code" {
+ max = 5
+ }
+ if d.score < max/2 {
+ gaps = append(gaps, fmt.Sprintf("%s scored %d/%d - needs improvement", d.name, d.score, max))
}
}
return gaps
@@ -906,8 +1406,8 @@ func writeScorecardMD(sc *Scorecard, pipelineDir string) error {
// Steinberger dimensions table
b.WriteString("## Steinberger Bar\n\n")
- b.WriteString("| Dimension | Score | Max |\n")
- b.WriteString("|-----------|-------|-----|\n")
+ b.WriteString("| Dimension | Score |\n")
+ b.WriteString("|-----------|-------|\n")
s := sc.Steinberger
dimensions := []struct {
name string
@@ -925,12 +1425,27 @@ func writeScorecardMD(sc *Scorecard, pipelineDir string) error {
{"Vision", s.Vision},
{"Workflows", s.Workflows},
{"Insight", s.Insight},
+ {"Path Validity", s.PathValidity},
+ {"Auth Protocol", s.AuthProtocol},
+ {"Data Pipeline Integrity", s.DataPipelineIntegrity},
+ {"Sync Correctness", s.SyncCorrectness},
}
for _, d := range dimensions {
bar := strings.Repeat("#", d.score) + strings.Repeat(".", 10-d.score)
b.WriteString(fmt.Sprintf("| %s | %d/10 %s |\n", d.name, d.score, bar))
}
- b.WriteString(fmt.Sprintf("| **Total** | **%d/120** |\n\n", s.Total))
+ typeDimensions := []struct {
+ name string
+ score int
+ }{
+ {"Type Fidelity", s.TypeFidelity},
+ {"Dead Code", s.DeadCode},
+ }
+ for _, d := range typeDimensions {
+ bar := strings.Repeat("#", d.score) + strings.Repeat(".", 5-d.score)
+ b.WriteString(fmt.Sprintf("| %s | %d/5 %s |\n", d.name, d.score, bar))
+ }
+ b.WriteString(fmt.Sprintf("| **Total** | **%d/100** |\n\n", s.Total))
// Competitor comparison
if len(sc.CompetitorScores) > 0 {
diff --git a/internal/pipeline/scorecard_run_test.go b/internal/pipeline/scorecard_run_test.go
index f4b4949f..82796603 100644
--- a/internal/pipeline/scorecard_run_test.go
+++ b/internal/pipeline/scorecard_run_test.go
@@ -16,7 +16,7 @@ func TestScorecardOnRealCLI(t *testing.T) {
pipelineDir = t.TempDir()
}
- sc, err := RunScorecard(outputDir, pipelineDir)
+ sc, err := RunScorecard(outputDir, pipelineDir, "")
if err != nil {
t.Fatalf("RunScorecard: %v", err)
}
diff --git a/internal/pipeline/scorecard_tier2_test.go b/internal/pipeline/scorecard_tier2_test.go
new file mode 100644
index 00000000..ca4870fb
--- /dev/null
+++ b/internal/pipeline/scorecard_tier2_test.go
@@ -0,0 +1,261 @@
+package pipeline
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestScoreDeadCode(t *testing.T) {
+ t.Run("penalizes dead flags and helper functions", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `
+package cli
+
+var flags struct {
+ jsonOutput bool
+ csvOutput bool
+ stdinInput bool
+}
+
+func init() {
+ rootCmd.Flags().BoolVar(&flags.jsonOutput, "json", false, "JSON output")
+ rootCmd.Flags().BoolVar(&flags.csvOutput, "csv", false, "CSV output")
+ rootCmd.Flags().BoolVar(&flags.stdinInput, "stdin", false, "Read stdin")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+func runMessages() {
+ if flags.jsonOutput {
+ println("json")
+ }
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/helpers.go", `
+package cli
+
+func filterFields() {}
+
+func outputCSV() {}
+`)
+
+ assert.Equal(t, 1, scoreDeadCode(dir))
+ })
+
+ t.Run("returns full score when nothing is dead", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/root.go", `
+package cli
+
+var flags struct {
+ jsonOutput bool
+}
+
+func init() {
+ rootCmd.Flags().BoolVar(&flags.jsonOutput, "json", false, "JSON output")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+func runMessages() {
+ if flags.jsonOutput {
+ println("json")
+ }
+}
+`)
+
+ assert.Equal(t, 5, scoreDeadCode(dir))
+ })
+}
+
+func TestScoreDataPipelineIntegrity(t *testing.T) {
+ t.Run("scores generic store methods and tables low", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/sync.go", `
+package cli
+
+import "example.com/project/internal/store"
+
+func runSync(db *store.DB) {
+ db.Upsert("messages", nil)
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/search.go", `
+package cli
+
+func runSearch(db interface{ Search(string) error }) {
+ _ = db.Search("term")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/store/store.go", `
+package store
+
+const schema = "`+`
+CREATE TABLE sync_records (
+ id TEXT,
+ data JSON,
+ synced_at TEXT
+);
+`+`"
+`)
+
+ assert.Equal(t, 1, scoreDataPipelineIntegrity(dir))
+ })
+
+ t.Run("scores domain specific pipelines high", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/sync.go", `
+package cli
+
+func runSync(db interface {
+ UpsertMessage(any) error
+ UpsertChannel(any) error
+}) {
+ _ = db.UpsertMessage(nil)
+ _ = db.UpsertChannel(nil)
+}
+`)
+ writeScorecardFixture(t, dir, "internal/cli/search.go", `
+package cli
+
+func runSearch(db interface{ SearchMessages(string) error }) {
+ _ = db.SearchMessages("hello")
+}
+`)
+ writeScorecardFixture(t, dir, "internal/store/store.go", `
+package store
+
+const schema = "`+`
+CREATE TABLE messages (
+ id TEXT,
+ channel_id TEXT,
+ author_id TEXT,
+ content TEXT,
+ created_at TEXT,
+ updated_at TEXT
+);
+
+CREATE TABLE channels (
+ id TEXT,
+ guild_id TEXT,
+ name TEXT,
+ type TEXT,
+ position INTEGER,
+ synced_at TEXT
+);
+`+`"
+`)
+
+ assert.Equal(t, 9, scoreDataPipelineIntegrity(dir))
+ })
+}
+
+func TestScoreSyncCorrectness(t *testing.T) {
+ t.Run("scores empty resource selection and missing state tracking low", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/sync.go", `
+package cli
+
+func defaultSyncResources() []string {
+ return []string{}
+}
+
+func runSync(resource string) string {
+ path := "/" + resource
+ return path
+}
+`)
+
+ assert.LessOrEqual(t, scoreSyncCorrectness(dir), 3)
+ })
+
+ t.Run("scores resource defaults pagination and sync state high", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/sync.go", `
+package cli
+
+func defaultSyncResources() []string {
+ return []string{"channels", "messages"}
+}
+
+func runSync(store interface {
+ GetSyncState(string) string
+ SaveSyncState(string, string)
+}) {
+ path := "/guilds/{guild_id}/messages"
+ cursor := store.GetSyncState("messages")
+ paginatedGet(path, cursor)
+ store.SaveSyncState("messages", "next")
+}
+`)
+
+ assert.Equal(t, 10, scoreSyncCorrectness(dir))
+ })
+}
+
+func TestScoreTypeFidelity(t *testing.T) {
+ t.Run("scores wrong id flag types and dummy guards low", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+import "strings"
+
+var _ = strings.ReplaceAll
+
+func init() {
+ cmd := messagesCmd
+ cmd.Flags().IntVar(&flagAfterID, "after-id", 0, "After")
+}
+`)
+
+ assert.Equal(t, 0, scoreTypeFidelity(dir))
+ })
+
+ t.Run("scores string id flags required markers and clear descriptions high", func(t *testing.T) {
+ dir := t.TempDir()
+
+ writeScorecardFixture(t, dir, "internal/cli/messages.go", `
+package cli
+
+func init() {
+ cmd := messagesCmd
+ cmd.Flags().StringVar(&flagAfterID, "after-id", "", "Snowflake ID to fetch results after the given message")
+ cmd.Flags().StringVar(&flagChannelID, "channel-id", "", "Channel ID containing the messages to fetch for sync")
+ cmd.Flags().StringVar(&flagGuildID, "guild-id", "", "Guild ID used to scope channel and message syncing")
+ _ = cmd.MarkFlagRequired("after-id")
+ _ = cmd.MarkFlagRequired("channel-id")
+ _ = cmd.MarkFlagRequired("guild-id")
+}
+`)
+
+ assert.GreaterOrEqual(t, scoreTypeFidelity(dir), 4)
+ })
+}
+
+func writeScorecardFixture(t *testing.T, root, relPath, content string) {
+ t.Helper()
+
+ path := filepath.Join(root, relPath)
+ err := os.MkdirAll(filepath.Dir(path), 0o755)
+ if err != nil {
+ t.Fatalf("mkdir %s: %v", relPath, err)
+ }
+
+ err = os.WriteFile(path, []byte(content), 0o644)
+ if err != nil {
+ t.Fatalf("write %s: %v", relPath, err)
+ }
+}
← 86c5d908 feat(generator): Apache 2.0 license on generated CLIs with N
·
back to Cli Printing Press
·
feat(generator): wire BuildSchema to store/sync/search templ eb59816b →