← back to Cli Printing Press
feat(dogfood): add mechanical CLI validation command
82bae3ee664602548f7db63e61639638f9de67b4 · 2026-03-26 15:20:29 -0700 · Matt Van Horn
New `printing-press dogfood` command that mechanically validates generated
CLIs against their source spec. Replaces Claude's subjective PASS/WARN/FAIL
with automated checks:
- Path Validity: extracts URL paths from commands, validates against spec
- Auth Protocol: checks if auth format matches spec's securitySchemes
- Dead Flags: finds flags declared in root.go but never read in any RunE
- Dead Functions: finds helpers.go functions never called by any command
- Data Pipeline: checks if sync calls domain UpsertX or generic Upsert
Results on generated CLIs:
- discord-cli: FAIL (auth mismatch, 8 dead flags, 12 dead functions)
- linear-cli: FAIL (5 dead flags, 8 dead functions - all scorecard gaming)
- discord-cli-v3: FAIL but improved (domain upserts detected, paths valid)
Updated fullrun.go and planner.go to use new DogfoodReport type.
Tests: 3 new test functions, all passing.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
A internal/cli/dogfood.goM internal/cli/root.goM internal/pipeline/dogfood.goM internal/pipeline/dogfood_test.goM internal/pipeline/fullrun.goM internal/pipeline/planner.go
Diff
commit 82bae3ee664602548f7db63e61639638f9de67b4
Author: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Date: Thu Mar 26 15:20:29 2026 -0700
feat(dogfood): add mechanical CLI validation command
New `printing-press dogfood` command that mechanically validates generated
CLIs against their source spec. Replaces Claude's subjective PASS/WARN/FAIL
with automated checks:
- Path Validity: extracts URL paths from commands, validates against spec
- Auth Protocol: checks if auth format matches spec's securitySchemes
- Dead Flags: finds flags declared in root.go but never read in any RunE
- Dead Functions: finds helpers.go functions never called by any command
- Data Pipeline: checks if sync calls domain UpsertX or generic Upsert
Results on generated CLIs:
- discord-cli: FAIL (auth mismatch, 8 dead flags, 12 dead functions)
- linear-cli: FAIL (5 dead flags, 8 dead functions - all scorecard gaming)
- discord-cli-v3: FAIL but improved (domain upserts detected, paths valid)
Updated fullrun.go and planner.go to use new DogfoodReport type.
Tests: 3 new test functions, all passing.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/cli/dogfood.go | 138 +++++++++
internal/cli/root.go | 1 +
internal/pipeline/dogfood.go | 603 ++++++++++++++++++++++++++++----------
internal/pipeline/dogfood_test.go | 234 +++++++++------
internal/pipeline/fullrun.go | 34 +--
internal/pipeline/planner.go | 10 +-
6 files changed, 745 insertions(+), 275 deletions(-)
diff --git a/internal/cli/dogfood.go b/internal/cli/dogfood.go
new file mode 100644
index 00000000..c97d7fdf
--- /dev/null
+++ b/internal/cli/dogfood.go
@@ -0,0 +1,138 @@
+package cli
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/mvanhorn/cli-printing-press/internal/pipeline"
+ "github.com/spf13/cobra"
+)
+
+func newDogfoodCmd() *cobra.Command {
+ var dir string
+ var specPath string
+ var asJSON bool
+
+ cmd := &cobra.Command{
+ Use: "dogfood",
+ Short: "Validate a generated CLI against its source spec",
+ Long: "Mechanically verify that a generated CLI's commands hit valid API paths, auth matches the spec protocol, no dead flags/functions exist, and the data pipeline is wired correctly.",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ report, err := pipeline.RunDogfood(dir, specPath)
+ if err != nil {
+ return fmt.Errorf("running dogfood: %w", err)
+ }
+
+ if asJSON {
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetIndent("", " ")
+ return enc.Encode(report)
+ }
+
+ printDogfoodReport(report)
+ return nil
+ },
+ }
+
+ cmd.Flags().StringVar(&dir, "dir", "", "Path to the generated CLI directory (required)")
+ cmd.Flags().StringVar(&specPath, "spec", "", "Path to the OpenAPI spec file")
+ cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON")
+ _ = cmd.MarkFlagRequired("dir")
+ return cmd
+}
+
+func printDogfoodReport(report *pipeline.DogfoodReport) {
+ name := filepath.Base(report.Dir)
+
+ fmt.Printf("Dogfood Report: %s\n", name)
+ fmt.Println("================================")
+ fmt.Println()
+
+ pathStatus := "SKIP"
+ if report.SpecPath != "" {
+ pathStatus = "PASS"
+ if report.PathCheck.Pct < 70 {
+ pathStatus = "FAIL"
+ }
+ }
+ fmt.Printf("Path Validity: %d/%d valid (%s)\n", report.PathCheck.Valid, report.PathCheck.Tested, pathStatus)
+ for _, path := range report.PathCheck.Invalid {
+ fmt.Printf(" - %s: not in spec\n", path)
+ }
+ fmt.Println()
+
+ authStatus := "SKIP"
+ if report.SpecPath != "" {
+ authStatus = "MATCH"
+ if !report.AuthCheck.Match {
+ authStatus = "MISMATCH"
+ }
+ }
+ fmt.Printf("Auth Protocol: %s\n", authStatus)
+ if report.AuthCheck.SpecScheme != "" {
+ fmt.Printf(" Spec: %s\n", report.AuthCheck.SpecScheme)
+ }
+ if report.AuthCheck.GeneratedFmt != "" {
+ fmt.Printf(" Generated: Uses %q prefix\n", trimSpaceOrUnknown(report.AuthCheck.GeneratedFmt))
+ }
+ if report.AuthCheck.Detail != "" {
+ fmt.Printf(" Detail: %s\n", report.AuthCheck.Detail)
+ }
+ fmt.Println()
+
+ flagsStatus := "PASS"
+ if report.DeadFlags.Dead >= 3 {
+ flagsStatus = "FAIL"
+ } else if report.DeadFlags.Dead > 0 {
+ flagsStatus = "WARN"
+ }
+ fmt.Printf("Dead Flags: %d dead (%s)\n", report.DeadFlags.Dead, flagsStatus)
+ for _, item := range report.DeadFlags.Items {
+ fmt.Printf(" - %s (declared, never read)\n", item)
+ }
+ fmt.Println()
+
+ funcsStatus := "PASS"
+ if report.DeadFuncs.Dead > 0 {
+ funcsStatus = "WARN"
+ }
+ fmt.Printf("Dead Functions: %d dead (%s)\n", report.DeadFuncs.Dead, funcsStatus)
+ for _, item := range report.DeadFuncs.Items {
+ fmt.Printf(" - %s (defined, never called)\n", item)
+ }
+ fmt.Println()
+
+ pipelineStatus := "GOOD"
+ if !report.PipelineCheck.SyncCallsDomain || !report.PipelineCheck.SearchCallsDomain || report.PipelineCheck.DomainTables == 0 {
+ pipelineStatus = "PARTIAL"
+ }
+ fmt.Printf("Data Pipeline: %s\n", pipelineStatus)
+ if report.PipelineCheck.SyncCallsDomain {
+ fmt.Println(" Sync: calls domain-specific Upsert methods (GOOD)")
+ } else {
+ fmt.Println(" Sync: uses generic Upsert only")
+ }
+ if report.PipelineCheck.SearchCallsDomain {
+ fmt.Println(" Search: calls domain-specific Search methods (GOOD)")
+ } else {
+ fmt.Println(" Search: uses generic Search only or direct SQL")
+ }
+ fmt.Printf(" Domain tables: %d\n", report.PipelineCheck.DomainTables)
+ fmt.Println()
+
+ fmt.Printf("Verdict: %s\n", report.Verdict)
+ for _, issue := range report.Issues {
+ fmt.Printf(" - %s\n", issue)
+ }
+}
+
+func trimSpaceOrUnknown(s string) string {
+ trimmed := strings.TrimSpace(s)
+ if trimmed == "" {
+ return "unknown"
+ }
+ return trimmed
+}
diff --git a/internal/cli/root.go b/internal/cli/root.go
index 6d8474cc..aa2cdfab 100644
--- a/internal/cli/root.go
+++ b/internal/cli/root.go
@@ -36,6 +36,7 @@ func Execute() error {
rootCmd.AddCommand(newGenerateCmd())
rootCmd.AddCommand(newScorecardCmd())
+ rootCmd.AddCommand(newDogfoodCmd())
rootCmd.AddCommand(newVisionCmd())
rootCmd.AddCommand(newVersionCmd())
rootCmd.AddCommand(newPrintCmd())
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
index 8f91c3ca..b3b0ae5e 100644
--- a/internal/pipeline/dogfood.go
+++ b/internal/pipeline/dogfood.go
@@ -1,231 +1,522 @@
package pipeline
import (
- "context"
"encoding/json"
"fmt"
"os"
- "os/exec"
"path/filepath"
+ "regexp"
+ "sort"
"strings"
- "time"
)
-// DogfoodResults holds the output of the dogfood phase.
-type DogfoodResults struct {
- Tier int `json:"tier"`
- TotalCommands int `json:"total_commands"`
- PassedCommands int `json:"passed_commands"`
- FailedCommands int `json:"failed_commands"`
- Commands []CommandResult `json:"commands"`
- Score int `json:"score"` // 0-50
+type DogfoodReport struct {
+ Dir string `json:"dir"`
+ SpecPath string `json:"spec_path,omitempty"`
+ Verdict string `json:"verdict"`
+ PathCheck PathCheckResult `json:"path_check"`
+ AuthCheck AuthCheckResult `json:"auth_check"`
+ DeadFlags DeadCodeResult `json:"dead_flags"`
+ DeadFuncs DeadCodeResult `json:"dead_functions"`
+ PipelineCheck PipelineResult `json:"pipeline_check"`
+ Issues []string `json:"issues"`
}
-// CommandResult records a single dogfood command execution.
-type CommandResult struct {
- Tier int `json:"tier"`
- Command string `json:"command"`
- ExitCode int `json:"exit_code"`
- StdoutFile string `json:"stdout_file"`
- Stderr string `json:"stderr"`
- DurationMs int `json:"duration_ms"`
- Pass bool `json:"pass"`
+type PathCheckResult struct {
+ Tested int `json:"tested"`
+ Valid int `json:"valid"`
+ Invalid []string `json:"invalid,omitempty"`
+ Pct int `json:"valid_pct"`
}
-// DogfoodConfig controls what tiers to run.
-type DogfoodConfig struct {
- BinaryPath string
- PipelineDir string
- MaxTier int // 1, 2, or 3
- SandboxSafe bool // from KnownSpecs
- CmdTimeout time.Duration // per-command timeout
- Resources []string // resource names from the generated CLI
- AuthEnvVars []string // env vars that indicate credentials are available
+type AuthCheckResult struct {
+ SpecScheme string `json:"spec_scheme"`
+ GeneratedFmt string `json:"generated_format"`
+ Match bool `json:"match"`
+ Detail string `json:"detail"`
}
-// RunDogfood executes tiered dogfood testing against a generated CLI binary.
-func RunDogfood(cfg DogfoodConfig) (*DogfoodResults, error) {
- if cfg.CmdTimeout == 0 {
- cfg.CmdTimeout = 15 * time.Second
- }
- if cfg.MaxTier == 0 {
- cfg.MaxTier = 1
- }
+type DeadCodeResult struct {
+ Total int `json:"total"`
+ Dead int `json:"dead"`
+ Items []string `json:"items,omitempty"`
+}
- evidenceDir := filepath.Join(cfg.PipelineDir, "dogfood-evidence")
- if err := os.MkdirAll(evidenceDir, 0o755); err != nil {
- return nil, fmt.Errorf("creating evidence dir: %w", err)
- }
+type PipelineResult struct {
+ SyncCallsDomain bool `json:"sync_calls_domain"`
+ SearchCallsDomain bool `json:"search_calls_domain"`
+ DomainTables int `json:"domain_tables"`
+ Detail string `json:"detail"`
+}
- results := &DogfoodResults{Tier: 1}
+type openAPISpec struct {
+ Paths map[string]json.RawMessage `json:"paths"`
+ Components struct {
+ SecuritySchemes map[string]struct {
+ Type string `json:"type"`
+ Scheme string `json:"scheme"`
+ } `json:"securitySchemes"`
+ } `json:"components"`
+}
- // Tier 1: No auth required - always runs
- tier1Commands := buildTier1Commands(cfg.BinaryPath, cfg.Resources)
- for _, cmd := range tier1Commands {
- cr := runCommand(cfg.BinaryPath, cmd, 1, cfg.CmdTimeout, evidenceDir)
- results.Commands = append(results.Commands, cr)
+func RunDogfood(dir, specPath string) (*DogfoodReport, error) {
+ report := &DogfoodReport{
+ Dir: dir,
+ SpecPath: specPath,
+ Verdict: "PASS",
}
- // Tier 2: Read-only with auth - if credentials available and tier >= 2
- if cfg.MaxTier >= 2 && hasCredentials(cfg.AuthEnvVars) {
- results.Tier = 2
- tier2Dir := filepath.Join(evidenceDir, "tier2-reads")
- os.MkdirAll(tier2Dir, 0o755)
- tier2Commands := buildTier2Commands(cfg.BinaryPath, cfg.Resources)
- for _, cmd := range tier2Commands {
- cr := runCommand(cfg.BinaryPath, cmd, 2, cfg.CmdTimeout, tier2Dir)
- results.Commands = append(results.Commands, cr)
+ var spec *openAPISpec
+ if specPath != "" {
+ loaded, err := loadDogfoodOpenAPISpec(specPath)
+ if err != nil {
+ return nil, err
}
- }
+ spec = loaded
- // Tier 3: Sandbox write ops - only if sandbox safe and tier >= 3
- if cfg.MaxTier >= 3 && cfg.SandboxSafe {
- results.Tier = 3
- tier3Dir := filepath.Join(evidenceDir, "tier3-writes")
- os.MkdirAll(tier3Dir, 0o755)
- // Tier 3 is API-specific and would need per-API test plans.
- // For now, just record that tier 3 was available.
- fmt.Fprintf(os.Stderr, "info: Tier 3 sandbox testing available but no test plan defined\n")
- }
-
- // Compute results
- for _, cr := range results.Commands {
- results.TotalCommands++
- if cr.Pass {
- results.PassedCommands++
- } else {
- results.FailedCommands++
+ report.PathCheck = checkPaths(dir, spec.Paths)
+ report.AuthCheck = checkAuth(dir, spec.Components.SecuritySchemes)
+ } else {
+ report.AuthCheck = AuthCheckResult{
+ Match: true,
+ Detail: "spec not provided; auth protocol check skipped",
}
}
- results.Score = computeDogfoodScore(results)
+ report.DeadFlags = checkDeadFlags(dir)
+ report.DeadFuncs = checkDeadFunctions(dir)
+ report.PipelineCheck = checkPipelineIntegrity(dir)
+ report.Issues = collectDogfoodIssues(report, spec != nil)
+ report.Verdict = deriveDogfoodVerdict(report, spec != nil)
- // Write results
- if err := writeDogfoodResults(results, cfg.PipelineDir); err != nil {
- return results, fmt.Errorf("writing results: %w", err)
+ if err := writeDogfoodResults(report, dir); err != nil {
+ return nil, err
}
- return results, nil
+ return report, nil
}
-// LoadDogfoodResults reads dogfood-results.json from a pipeline directory.
-func LoadDogfoodResults(pipelineDir string) (*DogfoodResults, error) {
- data, err := os.ReadFile(filepath.Join(pipelineDir, "dogfood-results.json"))
+func LoadDogfoodResults(dir string) (*DogfoodReport, error) {
+ data, err := os.ReadFile(filepath.Join(dir, "dogfood-results.json"))
if err != nil {
return nil, err
}
- var r DogfoodResults
- if err := json.Unmarshal(data, &r); err != nil {
+
+ var report DogfoodReport
+ if err := json.Unmarshal(data, &report); err != nil {
return nil, err
}
- return &r, nil
+ return &report, nil
+}
+
+func writeDogfoodResults(report *DogfoodReport, dir string) error {
+ data, err := json.MarshalIndent(report, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(dir, "dogfood-results.json"), data, 0o644)
}
-func buildTier1Commands(binary string, resources []string) []dogfoodCmd {
- cmds := []dogfoodCmd{
- {args: []string{"--help"}, evidenceFile: "tier1-help.txt"},
- {args: []string{"version"}, evidenceFile: "tier1-version.txt"},
- {args: []string{"doctor"}, evidenceFile: "tier1-doctor.txt"},
+func loadDogfoodOpenAPISpec(specPath string) (*openAPISpec, error) {
+ data, err := os.ReadFile(specPath)
+ if err != nil {
+ return nil, fmt.Errorf("reading spec: %w", err)
}
- // Add per-resource help commands
- resDir := "tier1-resources"
- for _, r := range resources {
- cmds = append(cmds, dogfoodCmd{
- args: []string{r, "--help"},
- evidenceFile: filepath.Join(resDir, r+"-help.txt"),
- })
+
+ var spec openAPISpec
+ if err := json.Unmarshal(data, &spec); err != nil {
+ return nil, fmt.Errorf("parsing spec JSON: %w", err)
}
- return cmds
+ return &spec, nil
}
-func buildTier2Commands(binary string, resources []string) []dogfoodCmd {
- var cmds []dogfoodCmd
- for _, r := range resources {
- cmds = append(cmds, dogfoodCmd{
- args: []string{r, "list", "--json"},
- evidenceFile: r + "-list.txt",
- })
+func checkPaths(dir string, paths map[string]json.RawMessage) PathCheckResult {
+ result := PathCheckResult{}
+ if len(paths) == 0 {
+ return result
+ }
+
+ cliDir := filepath.Join(dir, "internal", "cli")
+ files := listGoFiles(cliDir)
+ var commandFiles []string
+ for _, file := range files {
+ base := filepath.Base(file)
+ switch base {
+ case "root.go", "helpers.go", "doctor.go", "auth.go", "dogfood.go", "scorecard.go", "vision.go":
+ continue
+ default:
+ commandFiles = append(commandFiles, file)
+ }
+ }
+ sort.Strings(commandFiles)
+ if len(commandFiles) > 10 {
+ commandFiles = commandFiles[:10]
+ }
+
+ specPatterns := compileSpecPathPatterns(paths)
+ pathAssignmentRe := regexp.MustCompile(`(?m)\bpath\s*(?::=|=)\s*"([^"]+)"`)
+
+ for _, file := range commandFiles {
+ content, err := os.ReadFile(file)
+ if err != nil {
+ continue
+ }
+
+ matches := pathAssignmentRe.FindAllStringSubmatch(string(content), -1)
+ for _, match := range matches {
+ path := match[1]
+ result.Tested++
+ if pathMatchesSpec(path, specPatterns) {
+ result.Valid++
+ continue
+ }
+ result.Invalid = append(result.Invalid, path)
+ }
+ }
+
+ if result.Tested > 0 {
+ result.Pct = (result.Valid * 100) / result.Tested
}
- return cmds
+ result.Invalid = uniqueSorted(result.Invalid)
+ return result
}
-type dogfoodCmd struct {
- args []string
- evidenceFile string
+func checkAuth(dir string, schemes map[string]struct {
+ Type string `json:"type"`
+ Scheme string `json:"scheme"`
+}) AuthCheckResult {
+ result := AuthCheckResult{
+ Match: true,
+ Detail: "no recognized auth scheme in spec",
+ }
+
+ expectedPrefix := ""
+ for name, scheme := range schemes {
+ if strings.Contains(strings.ToLower(name), "bot") {
+ result.SpecScheme = name + ` scheme (expects "Bot " prefix)`
+ expectedPrefix = "Bot "
+ break
+ }
+ if strings.EqualFold(scheme.Type, "http") && strings.EqualFold(scheme.Scheme, "bearer") {
+ result.SpecScheme = `http bearer scheme (expects "Bearer " prefix)`
+ expectedPrefix = "Bearer "
+ }
+ }
+
+ clientData, err := os.ReadFile(filepath.Join(dir, "internal", "client", "client.go"))
+ if err != nil {
+ result.Match = false
+ result.Detail = fmt.Sprintf("failed to read client.go: %v", err)
+ return result
+ }
+
+ clientSource := string(clientData)
+ switch {
+ case strings.Contains(clientSource, `"Bot "`):
+ result.GeneratedFmt = "Bot "
+ case strings.Contains(clientSource, `"Bearer "`):
+ result.GeneratedFmt = "Bearer "
+ default:
+ result.GeneratedFmt = "unknown"
+ }
+
+ if expectedPrefix == "" {
+ result.Detail = "spec not provided or no bot/bearer scheme detected"
+ return result
+ }
+
+ result.Match = result.GeneratedFmt == expectedPrefix
+ if result.Match {
+ result.Detail = fmt.Sprintf(`spec and generated client both use %q`, strings.TrimSpace(expectedPrefix))
+ } else {
+ result.Detail = fmt.Sprintf(`spec expects %q but generated client uses %q`, strings.TrimSpace(expectedPrefix), strings.TrimSpace(result.GeneratedFmt))
+ }
+ return result
}
-func runCommand(binaryPath string, dc dogfoodCmd, tier int, timeout time.Duration, evidenceDir string) CommandResult {
- cr := CommandResult{
- Tier: tier,
- Command: strings.Join(dc.args, " "),
+func checkDeadFlags(dir string) DeadCodeResult {
+ rootData, err := os.ReadFile(filepath.Join(dir, "internal", "cli", "root.go"))
+ if err != nil {
+ return DeadCodeResult{}
+ }
+
+ fieldRe := regexp.MustCompile(`&flags\.(\w+)`)
+ matches := fieldRe.FindAllStringSubmatch(string(rootData), -1)
+ if len(matches) == 0 {
+ return DeadCodeResult{}
}
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- defer cancel()
+ fields := make(map[string]struct{})
+ for _, match := range matches {
+ fields[match[1]] = struct{}{}
+ }
- start := time.Now()
- cmd := exec.CommandContext(ctx, binaryPath, dc.args...)
- stdout, err := cmd.CombinedOutput()
- cr.DurationMs = int(time.Since(start).Milliseconds())
+ files := listGoFiles(filepath.Join(dir, "internal", "cli"))
+ var otherSources []string
+ for _, file := range files {
+ if filepath.Base(file) == "root.go" {
+ continue
+ }
+ data, err := os.ReadFile(file)
+ if err != nil {
+ continue
+ }
+ otherSources = append(otherSources, string(data))
+ }
+ result := DeadCodeResult{Total: len(fields)}
+ for _, field := range sortedKeys(fields) {
+ needle := "flags." + field
+ if containsAny(otherSources, needle) {
+ continue
+ }
+ result.Dead++
+ result.Items = append(result.Items, field)
+ }
+ return result
+}
+
+func checkDeadFunctions(dir string) DeadCodeResult {
+ helpersPath := filepath.Join(dir, "internal", "cli", "helpers.go")
+ data, err := os.ReadFile(helpersPath)
if err != nil {
- if exitErr, ok := err.(*exec.ExitError); ok {
- cr.ExitCode = exitErr.ExitCode()
- } else {
- cr.ExitCode = -1
+ return DeadCodeResult{}
+ }
+
+ funcRe := regexp.MustCompile(`(?m)^func\s+([A-Za-z_]\w*)\s*\(`)
+ matches := funcRe.FindAllStringSubmatch(string(data), -1)
+ if len(matches) == 0 {
+ return DeadCodeResult{}
+ }
+
+ names := make(map[string]struct{})
+ for _, match := range matches {
+ names[match[1]] = struct{}{}
+ }
+
+ files := listGoFiles(filepath.Join(dir, "internal", "cli"))
+ var otherSources []string
+ for _, file := range files {
+ if filepath.Base(file) == "helpers.go" {
+ continue
}
- cr.Stderr = err.Error()
- cr.Pass = false
- } else {
- cr.ExitCode = 0
- cr.Pass = true
+ content, err := os.ReadFile(file)
+ if err != nil {
+ continue
+ }
+ otherSources = append(otherSources, string(content))
+ }
+
+ result := DeadCodeResult{Total: len(names)}
+ for _, name := range sortedKeys(names) {
+ callRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(name) + `\s*\(`)
+ used := false
+ for _, source := range otherSources {
+ if callRe.MatchString(source) {
+ used = true
+ break
+ }
+ }
+ if used {
+ continue
+ }
+ result.Dead++
+ result.Items = append(result.Items, name)
+ }
+ return result
+}
+
+func checkPipelineIntegrity(dir string) PipelineResult {
+ result := PipelineResult{
+ Detail: "sync/search/store files not found",
+ }
+
+ syncData, _ := os.ReadFile(filepath.Join(dir, "internal", "cli", "sync.go"))
+ searchData, _ := os.ReadFile(filepath.Join(dir, "internal", "cli", "search.go"))
+ storeData, _ := os.ReadFile(filepath.Join(dir, "internal", "store", "store.go"))
+
+ syncSource := string(syncData)
+ searchSource := string(searchData)
+ storeSource := string(storeData)
+
+ domainUpsertRe := regexp.MustCompile(`\.Upsert[A-Z]\w*\s*\(`)
+ domainSearchRe := regexp.MustCompile(`\.Search[A-Z]\w*\s*\(`)
+
+ result.SyncCallsDomain = domainUpsertRe.MatchString(syncSource)
+ result.SearchCallsDomain = domainSearchRe.MatchString(searchSource)
+ result.DomainTables = countDomainTables(storeSource)
+
+ var parts []string
+ switch {
+ case result.SyncCallsDomain:
+ parts = append(parts, "sync uses domain-specific Upsert methods")
+ case strings.Contains(syncSource, ".Upsert("):
+ parts = append(parts, "sync uses generic Upsert only")
+ default:
+ parts = append(parts, "sync Upsert calls not found")
+ }
+
+ switch {
+ case result.SearchCallsDomain:
+ parts = append(parts, "search uses domain-specific Search methods")
+ case strings.Contains(searchSource, ".Search("):
+ parts = append(parts, "search uses generic Search only")
+ default:
+ parts = append(parts, "search methods not found")
+ }
+
+ if storeSource != "" {
+ parts = append(parts, fmt.Sprintf("%d domain tables found", result.DomainTables))
+ }
+
+ result.Detail = strings.Join(parts, "; ")
+ return result
+}
+
+func deriveDogfoodVerdict(report *DogfoodReport, hasSpec bool) string {
+ if hasSpec && report.PathCheck.Tested > 0 && report.PathCheck.Pct < 70 {
+ return "FAIL"
+ }
+ if hasSpec && !report.AuthCheck.Match {
+ return "FAIL"
+ }
+ if report.DeadFlags.Dead >= 3 {
+ return "FAIL"
+ }
+ if report.DeadFlags.Dead >= 1 && report.DeadFlags.Dead <= 2 {
+ return "WARN"
+ }
+ if report.DeadFuncs.Dead >= 1 {
+ return "WARN"
}
+ if !report.PipelineCheck.SyncCallsDomain {
+ return "WARN"
+ }
+ return "PASS"
+}
- // Write evidence file
- evidencePath := filepath.Join(evidenceDir, dc.evidenceFile)
- os.MkdirAll(filepath.Dir(evidencePath), 0o755)
- os.WriteFile(evidencePath, stdout, 0o644)
- cr.StdoutFile = evidencePath
+func collectDogfoodIssues(report *DogfoodReport, hasSpec bool) []string {
+ var issues []string
+ if hasSpec && report.PathCheck.Tested > 0 && report.PathCheck.Pct < 70 {
+ issues = append(issues, fmt.Sprintf("%d%% path validity against spec", report.PathCheck.Pct))
+ }
+ if hasSpec && !report.AuthCheck.Match {
+ issues = append(issues, "auth protocol mismatch")
+ }
+ if report.DeadFlags.Dead >= 3 {
+ issues = append(issues, fmt.Sprintf("%d dead flags found", report.DeadFlags.Dead))
+ } else if report.DeadFlags.Dead > 0 {
+ issues = append(issues, fmt.Sprintf("%d dead flags found", report.DeadFlags.Dead))
+ }
+ if report.DeadFuncs.Dead > 0 {
+ issues = append(issues, fmt.Sprintf("%d dead helper functions found", report.DeadFuncs.Dead))
+ }
+ if !report.PipelineCheck.SyncCallsDomain {
+ issues = append(issues, "sync uses generic Upsert only")
+ }
+ return issues
+}
- return cr
+func compileSpecPathPatterns(paths map[string]json.RawMessage) []*regexp.Regexp {
+ keys := make([]string, 0, len(paths))
+ for path := range paths {
+ keys = append(keys, path)
+ }
+ sort.Strings(keys)
+
+ paramRe := regexp.MustCompile(`\\\{[^/]+\\\}`)
+ var patterns []*regexp.Regexp
+ for _, path := range keys {
+ quoted := regexp.QuoteMeta(path)
+ regex := "^" + paramRe.ReplaceAllString(quoted, `[^/]+`) + "$"
+ patterns = append(patterns, regexp.MustCompile(regex))
+ }
+ return patterns
}
-func hasCredentials(envVars []string) bool {
- for _, v := range envVars {
- if os.Getenv(v) != "" {
+func pathMatchesSpec(path string, patterns []*regexp.Regexp) bool {
+ for _, pattern := range patterns {
+ if pattern.MatchString(path) {
return true
}
}
return false
}
-func computeDogfoodScore(results *DogfoodResults) int {
- if results.TotalCommands == 0 {
+func listGoFiles(dir string) []string {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return nil
+ }
+
+ var files []string
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+ name := entry.Name()
+ if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ files = append(files, filepath.Join(dir, name))
+ }
+ sort.Strings(files)
+ return files
+}
+
+func countDomainTables(storeSource string) int {
+ if storeSource == "" {
return 0
}
- // Base score: percentage of passed commands, scaled to 0-40
- passRate := float64(results.PassedCommands) / float64(results.TotalCommands)
- score := int(passRate * 40)
+ tableRe := regexp.MustCompile(`(?is)CREATE TABLE IF NOT EXISTS\s+\w+\s*\((.*?)\);`)
+ matches := tableRe.FindAllStringSubmatch(storeSource, -1)
+ count := 0
+ for _, match := range matches {
+ columns := 0
+ for _, line := range strings.Split(match[1], "\n") {
+ line = strings.TrimSpace(strings.TrimSuffix(line, ","))
+ if line == "" {
+ continue
+ }
+ upper := strings.ToUpper(line)
+ if strings.HasPrefix(upper, "PRIMARY KEY") || strings.HasPrefix(upper, "FOREIGN KEY") || strings.HasPrefix(upper, "UNIQUE") || strings.HasPrefix(upper, "CONSTRAINT") || strings.HasPrefix(upper, "CHECK") {
+ continue
+ }
+ columns++
+ }
+ if columns > 3 {
+ count++
+ }
+ }
+ return count
+}
- // Bonus for higher tiers
- switch results.Tier {
- case 2:
- score += 5
- case 3:
- score += 10
+func uniqueSorted(items []string) []string {
+ if len(items) == 0 {
+ return nil
}
+ set := make(map[string]struct{}, len(items))
+ for _, item := range items {
+ set[item] = struct{}{}
+ }
+ return sortedKeys(set)
+}
- if score > 50 {
- score = 50
+func sortedKeys(set map[string]struct{}) []string {
+ keys := make([]string, 0, len(set))
+ for key := range set {
+ keys = append(keys, key)
}
- return score
+ sort.Strings(keys)
+ return keys
}
-func writeDogfoodResults(results *DogfoodResults, pipelineDir string) error {
- data, err := json.MarshalIndent(results, "", " ")
- if err != nil {
- return err
+func containsAny(sources []string, needle string) bool {
+ for _, source := range sources {
+ if strings.Contains(source, needle) {
+ return true
+ }
}
- return os.WriteFile(filepath.Join(pipelineDir, "dogfood-results.json"), data, 0o644)
+ return false
}
diff --git a/internal/pipeline/dogfood_test.go b/internal/pipeline/dogfood_test.go
index 2abc66f3..e7be2a6b 100644
--- a/internal/pipeline/dogfood_test.go
+++ b/internal/pipeline/dogfood_test.go
@@ -1,116 +1,162 @@
package pipeline
import (
+ "os"
+ "path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
-func TestBuildTier1Commands(t *testing.T) {
- t.Run("zero resources returns 3 commands", func(t *testing.T) {
- cmds := buildTier1Commands("/bin/fake", nil)
- assert.Len(t, cmds, 3)
- })
+func TestRunDogfood(t *testing.T) {
+ dir := t.TempDir()
+
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "client"), 0o755))
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "store"), 0o755))
- t.Run("two resources returns 5 commands", func(t *testing.T) {
- cmds := buildTier1Commands("/bin/fake", []string{"users", "projects"})
- assert.Len(t, cmds, 5)
- })
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "root.go"), `package cli
+type rootFlags struct {
+ jsonOutput bool
+ csvOutput bool
+ stdinInput bool
+}
+func initFlags(flags *rootFlags) {
+ _ = &flags.jsonOutput
+ _ = &flags.csvOutput
+ _ = &flags.stdinInput
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "helpers.go"), `package cli
+func usedHelper() {}
+func deadHelper() {}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "users_list.go"), `package cli
+func usersList() {
+ path := "/users/123"
+ flags.jsonOutput = true
+ usedHelper()
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "projects_get.go"), `package cli
+func projectsGet() {
+ path := "/bogus"
}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "sync.go"), `package cli
+func runSync(s interface{ UpsertUsers() error }) error {
+ return s.UpsertUsers()
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "search.go"), `package cli
+func runSearch(s interface{ SearchUsers() error }) error {
+ return s.SearchUsers()
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "client", "client.go"), `package client
+func authHeader(token string) string {
+ return "Bearer " + token
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "store", "store.go"), "package store\n"+
+ "func schema() string {\n"+
+ "\treturn `\n"+
+ "\t\tCREATE TABLE IF NOT EXISTS users (\n"+
+ "\t\t\tid TEXT PRIMARY KEY,\n"+
+ "\t\t\tname TEXT NOT NULL,\n"+
+ "\t\t\temail TEXT,\n"+
+ "\t\t\tdata JSON NOT NULL\n"+
+ "\t\t);\n"+
+ "\t\tCREATE TABLE IF NOT EXISTS sync_state (\n"+
+ "\t\t\tentity_type TEXT PRIMARY KEY,\n"+
+ "\t\t\tlast_sync_at TEXT NOT NULL,\n"+
+ "\t\t\tcursor TEXT\n"+
+ "\t\t);\n"+
+ "\t`\n"+
+ "}\n")
-func TestHasCredentials(t *testing.T) {
- t.Run("empty slice returns false", func(t *testing.T) {
- assert.False(t, hasCredentials(nil))
- assert.False(t, hasCredentials([]string{}))
- })
+ specPath := filepath.Join(dir, "spec.json")
+ writeTestFile(t, specPath, `{
+ "paths": {
+ "/users/{id}": {},
+ "/projects/{id}": {}
+ },
+ "components": {
+ "securitySchemes": {
+ "BotToken": {
+ "type": "http",
+ "scheme": "bearer"
+ }
+ }
+ }
+}`)
+
+ report, err := RunDogfood(dir, specPath)
+ require.NoError(t, err)
- t.Run("unset vars return false", func(t *testing.T) {
- assert.False(t, hasCredentials([]string{"TOTALLY_UNSET_VAR_12345"}))
- })
+ assert.Equal(t, "FAIL", report.Verdict)
+ assert.Equal(t, 2, report.PathCheck.Tested)
+ assert.Equal(t, 1, report.PathCheck.Valid)
+ assert.Equal(t, 50, report.PathCheck.Pct)
+ assert.Equal(t, []string{"/bogus"}, report.PathCheck.Invalid)
+ assert.False(t, report.AuthCheck.Match)
+ assert.Equal(t, 3, report.DeadFlags.Total)
+ assert.Equal(t, 2, report.DeadFlags.Dead)
+ assert.Equal(t, []string{"csvOutput", "stdinInput"}, report.DeadFlags.Items)
+ assert.Equal(t, 2, report.DeadFuncs.Total)
+ assert.Equal(t, 1, report.DeadFuncs.Dead)
+ assert.Equal(t, []string{"deadHelper"}, report.DeadFuncs.Items)
+ assert.True(t, report.PipelineCheck.SyncCallsDomain)
+ assert.True(t, report.PipelineCheck.SearchCallsDomain)
+ assert.Equal(t, 1, report.PipelineCheck.DomainTables)
- t.Run("set var returns true", func(t *testing.T) {
- t.Setenv("TEST_CRED_ABC", "secret123")
- assert.True(t, hasCredentials([]string{"TEST_CRED_ABC"}))
- })
+ loaded, err := LoadDogfoodResults(dir)
+ require.NoError(t, err)
+ assert.Equal(t, report.Verdict, loaded.Verdict)
}
-func TestComputeDogfoodScore(t *testing.T) {
- t.Run("all pass returns 40+", func(t *testing.T) {
- r := &DogfoodResults{
- Tier: 1,
- TotalCommands: 5,
- PassedCommands: 5,
- }
- score := computeDogfoodScore(r)
- assert.GreaterOrEqual(t, score, 40)
- })
-
- t.Run("half pass returns ~20", func(t *testing.T) {
- r := &DogfoodResults{
- Tier: 1,
- TotalCommands: 10,
- PassedCommands: 5,
- }
- score := computeDogfoodScore(r)
- assert.Equal(t, 20, score)
- })
-
- t.Run("zero pass returns 0", func(t *testing.T) {
- r := &DogfoodResults{
- Tier: 1,
- TotalCommands: 5,
- PassedCommands: 0,
- }
- score := computeDogfoodScore(r)
- assert.Equal(t, 0, score)
- })
-
- t.Run("zero total returns 0", func(t *testing.T) {
- r := &DogfoodResults{
- Tier: 1,
- TotalCommands: 0,
- }
- score := computeDogfoodScore(r)
- assert.Equal(t, 0, score)
- })
-
- t.Run("tier bonus adds points", func(t *testing.T) {
- r := &DogfoodResults{
- Tier: 2,
- TotalCommands: 5,
- PassedCommands: 5,
- }
- score := computeDogfoodScore(r)
- assert.Equal(t, 45, score) // 40 base + 5 tier2 bonus
- })
+func TestCountDomainTables(t *testing.T) {
+ storeSource := `
+CREATE TABLE IF NOT EXISTS users (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ email TEXT,
+ data JSON NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS sync_state (
+ entity_type TEXT PRIMARY KEY,
+ last_sync_at TEXT NOT NULL,
+ cursor TEXT
+);
+`
+ assert.Equal(t, 1, countDomainTables(storeSource))
}
-func TestWriteAndLoadDogfoodResults(t *testing.T) {
- dir := t.TempDir()
- original := &DogfoodResults{
- Tier: 2,
- TotalCommands: 4,
- PassedCommands: 3,
- FailedCommands: 1,
- Score: 35,
- Commands: []CommandResult{
- {Tier: 1, Command: "--help", ExitCode: 0, Pass: true},
- {Tier: 1, Command: "version", ExitCode: 0, Pass: true},
- {Tier: 1, Command: "doctor", ExitCode: 0, Pass: true},
- {Tier: 2, Command: "users list --json", ExitCode: 1, Pass: false},
- },
+func TestDeriveDogfoodVerdict(t *testing.T) {
+ report := &DogfoodReport{
+ PathCheck: PathCheckResult{Tested: 10, Valid: 10, Pct: 100},
+ AuthCheck: AuthCheckResult{Match: true},
+ DeadFlags: DeadCodeResult{Dead: 1},
+ DeadFuncs: DeadCodeResult{Dead: 0},
+ PipelineCheck: PipelineResult{SyncCallsDomain: true},
}
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
- err := writeDogfoodResults(original, dir)
- require.NoError(t, err)
+ report.DeadFlags.Dead = 0
+ report.DeadFuncs.Dead = 1
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
- loaded, err := LoadDogfoodResults(dir)
- require.NoError(t, err)
- assert.Equal(t, 2, loaded.Tier)
- assert.Equal(t, 4, loaded.TotalCommands)
- assert.Equal(t, 3, loaded.PassedCommands)
- assert.Equal(t, 1, loaded.FailedCommands)
- assert.Equal(t, 35, loaded.Score)
- assert.Len(t, loaded.Commands, 4)
+ report.DeadFuncs.Dead = 0
+ report.PipelineCheck.SyncCallsDomain = false
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
+
+ report.PipelineCheck.SyncCallsDomain = true
+ assert.Equal(t, "PASS", deriveDogfoodVerdict(report, true))
+}
+
+func writeTestFile(t *testing.T, path string, content string) {
+ t.Helper()
+ require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
}
diff --git a/internal/pipeline/fullrun.go b/internal/pipeline/fullrun.go
index d4a530cb..8f452b4d 100644
--- a/internal/pipeline/fullrun.go
+++ b/internal/pipeline/fullrun.go
@@ -36,7 +36,7 @@ type FullRunResult struct {
PolishResult *llmpolish.PolishResult
// Step 4: Dogfood
- Dogfood *DogfoodResults
+ Dogfood *DogfoodReport
DogfoodError string
// Step 5: Scorecard
@@ -132,17 +132,12 @@ func MakeBestCLI(apiName, level, specFlag, specURL, outputDir, pressBinary strin
result.DogfoodError = fmt.Sprintf("build failed: %v", buildErr)
result.Errors = append(result.Errors, fmt.Sprintf("dogfood build: %v", buildErr))
} else {
- cfg := DogfoodConfig{
- BinaryPath: cliBinaryPath,
- PipelineDir: pipelineDir,
- MaxTier: 1,
- CmdTimeout: 15 * time.Second,
- Resources: resources,
- }
- dogfood, dogErr := RunDogfood(cfg)
+ dogfood, dogErr := RunDogfood(outputDir, "")
if dogErr != nil {
result.DogfoodError = dogErr.Error()
result.Errors = append(result.Errors, fmt.Sprintf("dogfood: %v", dogErr))
+ } else if err := writeDogfoodResults(dogfood, pipelineDir); err != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("dogfood write: %v", err))
}
result.Dogfood = dogfood
}
@@ -326,16 +321,15 @@ func PrintComparisonTable(results []*FullRunResult) string {
return fmt.Sprintf("%d/%d", wins, len(r.Scorecard.CompetitorScores))
})
- // Dogfood pass rate
- writeRow(&b, "Dogfood Pass Rate", results, func(r *FullRunResult) string {
+ // Dogfood result
+ writeRow(&b, "Dogfood", results, func(r *FullRunResult) string {
if r.Dogfood == nil {
return "n/a"
}
- if r.Dogfood.TotalCommands == 0 {
- return "0/0"
+ if r.Dogfood.PathCheck.Tested > 0 {
+ return fmt.Sprintf("%s %d%%", r.Dogfood.Verdict, r.Dogfood.PathCheck.Pct)
}
- pct := (r.Dogfood.PassedCommands * 100) / r.Dogfood.TotalCommands
- return fmt.Sprintf("%d/%d (%d%%)", r.Dogfood.PassedCommands, r.Dogfood.TotalCommands, pct)
+ return r.Dogfood.Verdict
})
// LLM Polish
@@ -500,12 +494,12 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
b.WriteString("## Dogfood Summary\n\n")
for _, r := range results {
if r.Dogfood != nil {
- pct := 0
- if r.Dogfood.TotalCommands > 0 {
- pct = (r.Dogfood.PassedCommands * 100) / r.Dogfood.TotalCommands
+ if r.Dogfood.PathCheck.Tested > 0 {
+ b.WriteString(fmt.Sprintf("- **%s** - %s, path validity %d/%d (%d%%)\n",
+ r.APIName, r.Dogfood.Verdict, r.Dogfood.PathCheck.Valid, r.Dogfood.PathCheck.Tested, r.Dogfood.PathCheck.Pct))
+ } else {
+ b.WriteString(fmt.Sprintf("- **%s** - %s\n", r.APIName, r.Dogfood.Verdict))
}
- b.WriteString(fmt.Sprintf("- **%s** - %d/%d passed (%d%%)\n",
- r.APIName, r.Dogfood.PassedCommands, r.Dogfood.TotalCommands, pct))
} else {
b.WriteString(fmt.Sprintf("- **%s** - dogfood not run (%s)\n", r.APIName, r.DogfoodError))
}
diff --git a/internal/pipeline/planner.go b/internal/pipeline/planner.go
index d2dafd2a..339f29b9 100644
--- a/internal/pipeline/planner.go
+++ b/internal/pipeline/planner.go
@@ -8,11 +8,11 @@ import (
// PlanContext aggregates outputs from completed phases for dynamic plan generation.
type PlanContext struct {
- SeedData SeedData
- Research *ResearchResult
- Dogfood *DogfoodResults
- Scorecard *Scorecard
- Learnings *LearningsDB
+ SeedData SeedData
+ Research *ResearchResult
+ Dogfood *DogfoodReport
+ Scorecard *Scorecard
+ Learnings *LearningsDB
}
// GenerateNextPlan writes a dynamic plan for the next phase, informed by
← 7bcacea3 feat(skill): add Phase 4.6 hallucination audit and anti-gami
·
back to Cli Printing Press
·
feat(graphql): add GraphQL SDL parser for CLI generation 8c92c19d →