← back to Cli Printing Press
feat(dogfood): add ExampleCheck to validate help example correctness
c3eacf940b6ad493609383a66d86529fcf1be16e · 2026-03-27 14:24:32 -0700 · Trevin Chow
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
M internal/cli/dogfood.goM internal/pipeline/dogfood.goM internal/pipeline/dogfood_test.go
Diff
commit c3eacf940b6ad493609383a66d86529fcf1be16e
Author: Trevin Chow <trevin@trevinchow.com>
Date: Fri Mar 27 14:24:32 2026 -0700
feat(dogfood): add ExampleCheck to validate help example correctness
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/cli/dogfood.go | 22 ++++
internal/pipeline/dogfood.go | 243 +++++++++++++++++++++++++++++++++++++-
internal/pipeline/dogfood_test.go | 106 +++++++++++++++++
3 files changed, 369 insertions(+), 2 deletions(-)
diff --git a/internal/cli/dogfood.go b/internal/cli/dogfood.go
index 7c953f96..be448e57 100644
--- a/internal/cli/dogfood.go
+++ b/internal/cli/dogfood.go
@@ -128,6 +128,28 @@ func printDogfoodReport(report *pipeline.DogfoodReport) {
fmt.Printf(" Domain tables: %d\n", report.PipelineCheck.DomainTables)
fmt.Println()
+ if report.ExampleCheck.Skipped {
+ fmt.Printf("Examples: SKIP (%s)\n", report.ExampleCheck.Detail)
+ } else {
+ exampleStatus := "PASS"
+ if report.ExampleCheck.Tested > 0 && (report.ExampleCheck.WithExamples*100/report.ExampleCheck.Tested) < 50 {
+ exampleStatus = "FAIL"
+ } else if len(report.ExampleCheck.InvalidFlags) > 0 {
+ exampleStatus = "WARN"
+ } else if report.ExampleCheck.Tested == 0 {
+ exampleStatus = "SKIP"
+ }
+ fmt.Printf("Examples: %d/%d commands have examples", report.ExampleCheck.WithExamples, report.ExampleCheck.Tested)
+ if len(report.ExampleCheck.InvalidFlags) > 0 {
+ fmt.Printf(" (%d invalid flags: %s)", len(report.ExampleCheck.InvalidFlags), strings.Join(report.ExampleCheck.InvalidFlags, ", "))
+ }
+ fmt.Printf(" (%s)\n", exampleStatus)
+ for _, cmd := range report.ExampleCheck.Missing {
+ fmt.Printf(" - %s: missing example\n", cmd)
+ }
+ }
+ fmt.Println()
+
fmt.Printf("Verdict: %s\n", report.Verdict)
for _, issue := range report.Issues {
fmt.Printf(" - %s\n", issue)
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
index b3b0ae5e..455efa3a 100644
--- a/internal/pipeline/dogfood.go
+++ b/internal/pipeline/dogfood.go
@@ -1,13 +1,16 @@
package pipeline
import (
+ "context"
"encoding/json"
"fmt"
"os"
+ "os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
+ "time"
)
type DogfoodReport struct {
@@ -18,8 +21,9 @@ type DogfoodReport struct {
AuthCheck AuthCheckResult `json:"auth_check"`
DeadFlags DeadCodeResult `json:"dead_flags"`
DeadFuncs DeadCodeResult `json:"dead_functions"`
- PipelineCheck PipelineResult `json:"pipeline_check"`
- Issues []string `json:"issues"`
+ PipelineCheck PipelineResult `json:"pipeline_check"`
+ ExampleCheck ExampleCheckResult `json:"example_check"`
+ Issues []string `json:"issues"`
}
type PathCheckResult struct {
@@ -49,6 +53,16 @@ type PipelineResult struct {
Detail string `json:"detail"`
}
+type ExampleCheckResult struct {
+ Tested int `json:"tested"`
+ WithExamples int `json:"with_examples"`
+ ValidExamples int `json:"valid_examples"`
+ InvalidFlags []string `json:"invalid_flags,omitempty"`
+ Missing []string `json:"missing,omitempty"`
+ Skipped bool `json:"skipped,omitempty"`
+ Detail string `json:"detail"`
+}
+
type openAPISpec struct {
Paths map[string]json.RawMessage `json:"paths"`
Components struct {
@@ -86,6 +100,7 @@ func RunDogfood(dir, specPath string) (*DogfoodReport, error) {
report.DeadFlags = checkDeadFlags(dir)
report.DeadFuncs = checkDeadFunctions(dir)
report.PipelineCheck = checkPipelineIntegrity(dir)
+ report.ExampleCheck = checkExamples(dir)
report.Issues = collectDogfoodIssues(report, spec != nil)
report.Verdict = deriveDogfoodVerdict(report, spec != nil)
@@ -383,6 +398,9 @@ func deriveDogfoodVerdict(report *DogfoodReport, hasSpec bool) string {
if report.DeadFlags.Dead >= 3 {
return "FAIL"
}
+ if report.ExampleCheck.Tested > 0 && (report.ExampleCheck.WithExamples*100/report.ExampleCheck.Tested) < 50 {
+ return "FAIL"
+ }
if report.DeadFlags.Dead >= 1 && report.DeadFlags.Dead <= 2 {
return "WARN"
}
@@ -392,6 +410,12 @@ func deriveDogfoodVerdict(report *DogfoodReport, hasSpec bool) string {
if !report.PipelineCheck.SyncCallsDomain {
return "WARN"
}
+ if len(report.ExampleCheck.InvalidFlags) > 0 {
+ return "WARN"
+ }
+ if report.ExampleCheck.Skipped {
+ return "WARN"
+ }
return "PASS"
}
@@ -414,9 +438,224 @@ func collectDogfoodIssues(report *DogfoodReport, hasSpec bool) []string {
if !report.PipelineCheck.SyncCallsDomain {
issues = append(issues, "sync uses generic Upsert only")
}
+ if report.ExampleCheck.Tested > 0 && (report.ExampleCheck.WithExamples*100/report.ExampleCheck.Tested) < 50 {
+ issues = append(issues, fmt.Sprintf("%d%% example coverage", report.ExampleCheck.WithExamples*100/report.ExampleCheck.Tested))
+ }
+ if len(report.ExampleCheck.InvalidFlags) > 0 {
+ issues = append(issues, fmt.Sprintf("%d invalid flags in examples", len(report.ExampleCheck.InvalidFlags)))
+ }
+ if report.ExampleCheck.Skipped {
+ issues = append(issues, fmt.Sprintf("example check skipped: %s", report.ExampleCheck.Detail))
+ }
return issues
}
+func checkExamples(dir string) ExampleCheckResult {
+ result := ExampleCheckResult{}
+
+ cliName := findCLIName(dir)
+ if cliName == "" {
+ result.Skipped = true
+ result.Detail = "no CLI command directory found"
+ return result
+ }
+
+ binaryPath, err := buildDogfoodBinary(dir, cliName)
+ if err != nil {
+ result.Skipped = true
+ result.Detail = fmt.Sprintf("could not build CLI binary: %v", err)
+ return result
+ }
+
+ // Get global flags from root --help
+ globalOut, err := runDogfoodCmd(binaryPath, 15*time.Second, "--help")
+ if err != nil {
+ result.Skipped = true
+ result.Detail = fmt.Sprintf("failed to run --help: %v", err)
+ return result
+ }
+ globalFlags := extractFlagNames(globalOut)
+
+ // List command files (same filtering as PathCheck)
+ cliDir := filepath.Join(dir, "internal", "cli")
+ files := listGoFiles(cliDir)
+ var endpointFiles []string
+ for _, file := range files {
+ base := filepath.Base(file)
+ switch base {
+ case "root.go", "helpers.go", "doctor.go", "auth.go", "dogfood.go", "scorecard.go", "vision.go":
+ continue
+ }
+ // Only include endpoint commands (those with RunE)
+ content, err := os.ReadFile(file)
+ if err != nil {
+ continue
+ }
+ if !strings.Contains(string(content), "RunE:") {
+ continue
+ }
+ endpointFiles = append(endpointFiles, file)
+ }
+ sort.Strings(endpointFiles)
+ if len(endpointFiles) > 10 {
+ endpointFiles = sampleEvenly(endpointFiles, 10)
+ }
+
+ for _, file := range endpointFiles {
+ base := strings.TrimSuffix(filepath.Base(file), ".go")
+ parts := strings.Split(base, "_")
+
+ result.Tested++
+ cmdLabel := strings.Join(parts, " ")
+
+ cmdArgs := append(parts, "--help")
+ cmdOut, err := runDogfoodCmd(binaryPath, 15*time.Second, cmdArgs...)
+ if err != nil {
+ result.Missing = append(result.Missing, cmdLabel)
+ continue
+ }
+
+ examples := extractExamplesSection(cmdOut)
+ if examples == "" {
+ result.Missing = append(result.Missing, cmdLabel)
+ continue
+ }
+
+ result.WithExamples++
+
+ // Extract flags used in examples
+ exampleFlags := extractFlagNames(examples)
+ // Extract all valid flags from command help + global flags
+ cmdFlags := extractFlagNames(cmdOut)
+ allValidFlags := make(map[string]struct{})
+ for _, f := range cmdFlags {
+ allValidFlags[f] = struct{}{}
+ }
+ for _, f := range globalFlags {
+ allValidFlags[f] = struct{}{}
+ }
+
+ valid := true
+ for _, f := range exampleFlags {
+ if _, ok := allValidFlags[f]; !ok {
+ result.InvalidFlags = append(result.InvalidFlags, "--"+f)
+ valid = false
+ }
+ }
+ if valid {
+ result.ValidExamples++
+ }
+ }
+
+ result.InvalidFlags = uniqueSorted(result.InvalidFlags)
+ result.Missing = uniqueSorted(result.Missing)
+
+ if result.Tested == 0 {
+ result.Detail = "no endpoint commands found to test"
+ } else {
+ result.Detail = fmt.Sprintf("%d/%d commands have examples", result.WithExamples, result.Tested)
+ if len(result.InvalidFlags) > 0 {
+ result.Detail += fmt.Sprintf(" (%d invalid flags: %s)", len(result.InvalidFlags), strings.Join(result.InvalidFlags, ", "))
+ }
+ }
+
+ return result
+}
+
+func findCLIName(dir string) string {
+ cmdDir := filepath.Join(dir, "cmd")
+ entries, err := os.ReadDir(cmdDir)
+ if err != nil {
+ return ""
+ }
+ for _, entry := range entries {
+ if entry.IsDir() && strings.HasSuffix(entry.Name(), "-cli") {
+ return entry.Name()
+ }
+ }
+ return ""
+}
+
+func buildDogfoodBinary(dir, cliName string) (string, error) {
+ buildPath := filepath.Join(dir, cliName+"-dogfood")
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, "go", "build", "-o", buildPath, "./cmd/"+cliName)
+ cmd.Dir = dir
+ if err := cmd.Run(); err != nil {
+ if ctx.Err() == context.DeadlineExceeded {
+ return "", fmt.Errorf("timed out after 2m")
+ }
+ return "", err
+ }
+ return buildPath, nil
+}
+
+func runDogfoodCmd(binary string, timeout time.Duration, args ...string) (string, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, binary, args...)
+ out, err := cmd.CombinedOutput()
+ if err != nil && ctx.Err() == context.DeadlineExceeded {
+ return "", fmt.Errorf("timed out after %s", timeout)
+ }
+ // --help often returns exit 0, but accept output regardless
+ if len(out) > 0 {
+ return string(out), nil
+ }
+ return "", err
+}
+
+func extractExamplesSection(helpOutput string) string {
+ lines := strings.Split(helpOutput, "\n")
+ var inExamples bool
+ var examples []string
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+ if trimmed == "Examples:" {
+ inExamples = true
+ continue
+ }
+ if inExamples {
+ // Section headers in Cobra help are non-indented and non-empty
+ if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
+ break
+ }
+ examples = append(examples, line)
+ }
+ }
+ return strings.TrimSpace(strings.Join(examples, "\n"))
+}
+
+func extractFlagNames(text string) []string {
+ re := regexp.MustCompile(`--([a-z][-a-z0-9]*)`)
+ matches := re.FindAllStringSubmatch(text, -1)
+ seen := make(map[string]struct{})
+ var flags []string
+ for _, match := range matches {
+ name := match[1]
+ if _, ok := seen[name]; !ok {
+ seen[name] = struct{}{}
+ flags = append(flags, name)
+ }
+ }
+ sort.Strings(flags)
+ return flags
+}
+
+func sampleEvenly(items []string, n int) []string {
+ if len(items) <= n {
+ return items
+ }
+ step := float64(len(items)) / float64(n)
+ result := make([]string, n)
+ for i := 0; i < n; i++ {
+ idx := int(float64(i) * step)
+ result[i] = items[idx]
+ }
+ return result
+}
+
func compileSpecPathPatterns(paths map[string]json.RawMessage) []*regexp.Regexp {
keys := make([]string, 0, len(paths))
for path := range paths {
diff --git a/internal/pipeline/dogfood_test.go b/internal/pipeline/dogfood_test.go
index e7be2a6b..0eefb6ad 100644
--- a/internal/pipeline/dogfood_test.go
+++ b/internal/pipeline/dogfood_test.go
@@ -110,6 +110,9 @@ func authHeader(token string) string {
assert.True(t, report.PipelineCheck.SyncCallsDomain)
assert.True(t, report.PipelineCheck.SearchCallsDomain)
assert.Equal(t, 1, report.PipelineCheck.DomainTables)
+ assert.Equal(t, 0, report.ExampleCheck.Tested)
+ assert.True(t, report.ExampleCheck.Skipped)
+ assert.Equal(t, "no CLI command directory found", report.ExampleCheck.Detail)
loaded, err := LoadDogfoodResults(dir)
require.NoError(t, err)
@@ -154,6 +157,109 @@ func TestDeriveDogfoodVerdict(t *testing.T) {
report.PipelineCheck.SyncCallsDomain = true
assert.Equal(t, "PASS", deriveDogfoodVerdict(report, true))
+
+ // ExampleCheck: FAIL when <50% coverage
+ report.ExampleCheck = ExampleCheckResult{Tested: 10, WithExamples: 4}
+ assert.Equal(t, "FAIL", deriveDogfoodVerdict(report, true))
+
+ // ExampleCheck: not FAIL at exactly 50%
+ report.ExampleCheck = ExampleCheckResult{Tested: 10, WithExamples: 5}
+ assert.Equal(t, "PASS", deriveDogfoodVerdict(report, true))
+
+ // ExampleCheck: WARN when invalid flags present
+ report.ExampleCheck = ExampleCheckResult{Tested: 10, WithExamples: 10, InvalidFlags: []string{"--bogus"}}
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
+
+ // ExampleCheck: WARN when skipped (build failure etc.)
+ report.ExampleCheck = ExampleCheckResult{Skipped: true, Detail: "could not build CLI binary"}
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
+
+ // ExampleCheck: PASS when ran successfully with no issues
+ report.ExampleCheck = ExampleCheckResult{Tested: 10, WithExamples: 10, ValidExamples: 10}
+ assert.Equal(t, "PASS", deriveDogfoodVerdict(report, true))
+}
+
+func TestExtractExamplesSection(t *testing.T) {
+ tests := []struct {
+ name string
+ help string
+ want string
+ }{
+ {
+ name: "standard cobra help",
+ help: "Some command\n\nUsage:\n cli users list [flags]\n\nExamples:\n # List all users\n cli users list --limit 10\n\nFlags:\n --limit int max results\n",
+ want: "# List all users\n cli users list --limit 10",
+ },
+ {
+ name: "no examples section",
+ help: "Some command\n\nUsage:\n cli version\n\nFlags:\n -h, --help help\n",
+ want: "",
+ },
+ {
+ name: "examples before global flags",
+ help: "Examples:\n cli foo --bar baz\n\nGlobal Flags:\n --config string\n",
+ want: "cli foo --bar baz",
+ },
+ {
+ name: "multi-line examples",
+ help: "Examples:\n # First example\n cli do --a 1\n\n # Second example\n cli do --b 2\n\nFlags:\n --a int\n",
+ want: "# First example\n cli do --a 1\n\n # Second example\n cli do --b 2",
+ },
+ {
+ name: "empty help",
+ help: "",
+ want: "",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, extractExamplesSection(tt.help))
+ })
+ }
+}
+
+func TestExtractFlagNames(t *testing.T) {
+ tests := []struct {
+ name string
+ text string
+ want []string
+ }{
+ {
+ name: "multiple flags",
+ text: "cli users list --limit 10 --format json",
+ want: []string{"format", "limit"},
+ },
+ {
+ name: "deduplication",
+ text: "--flag value --flag other",
+ want: []string{"flag"},
+ },
+ {
+ name: "hyphenated flag names",
+ text: "--dry-run --output-format table",
+ want: []string{"dry-run", "output-format"},
+ },
+ {
+ name: "ignores short flags",
+ text: "-h --help -v --verbose",
+ want: []string{"help", "verbose"},
+ },
+ {
+ name: "no flags",
+ text: "just some text with no flags",
+ want: nil,
+ },
+ {
+ name: "ignores uppercase",
+ text: "--OK should not match",
+ want: nil,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, extractFlagNames(tt.text))
+ })
+ }
}
func writeTestFile(t *testing.T, path string, content string) {
← b86384b1 docs: add testing guidance to AGENTS.md
·
back to Cli Printing Press
·
feat(templates): add structured confirmation envelope for mu d91566f9 →