[object Object]

← back to Cli Printing Press

fix(cli): honor documented typed exit codes (#504)

31e63be4587d1b497f409b55d73ad414e73b6dfa · 2026-05-02 10:18:06 -0700 · Trevin Chow

Files touched

Diff

commit 31e63be4587d1b497f409b55d73ad414e73b6dfa
Author: Trevin Chow <trevin@trevinchow.com>
Date:   Sat May 2 10:18:06 2026 -0700

    fix(cli): honor documented typed exit codes (#504)
---
 AGENTS.md                                  |  10 +++
 internal/generator/generator_test.go       |   1 +
 internal/generator/templates/which.go.tmpl |   3 +
 internal/pipeline/dogfood.go               |  14 +--
 internal/pipeline/runtime.go               |   8 +-
 internal/pipeline/runtime_annotations.go   | 135 ++++++++++++++++++++++++++++
 internal/pipeline/runtime_commands.go      |  11 +--
 internal/pipeline/runtime_exec.go          |   2 +-
 internal/pipeline/runtime_exitcodes.go     | 114 ++++++++++++++++++++++++
 internal/pipeline/runtime_test.go          | 138 +++++++++++++++++++++++++++++
 skills/printing-press/SKILL.md             |   2 +
 11 files changed, 416 insertions(+), 22 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 6f99325c..96185ed7 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -66,6 +66,16 @@ When adding a generator template that emits a novel command whose only effect is
 
 Wrong annotations are worse than missing ones: the host trusts the claim and stops asking. A `readOnlyHint: true` on a mutating tool is a real bug; a missing annotation is just a permission prompt.
 
+### Typed exit-code verification
+
+`printing-press verify` treats exit `0` as success by default. For commands where a non-zero code is an intentional, non-error control-flow result (for example, "no confident match"), declare that contract in the Cobra command:
+
+```go
+Annotations: map[string]string{"pp:typed-exit-codes": "0,2"},
+```
+
+The verifier reads this annotation first, then falls back to parsing a command-level `Exit codes:` help block when the annotation is absent. Do **not** put the whole global failure palette (`3` not found, `4` auth, `5` API error, etc.) in a command-level `Exit codes:` block unless those codes should count as a verify pass for that specific command. General troubleshooting exit-code docs belong in README/SKILL prose, not command help.
+
 ## Build, Test & Lint
 
 ```bash
diff --git a/internal/generator/generator_test.go b/internal/generator/generator_test.go
index 22528382..62557981 100644
--- a/internal/generator/generator_test.go
+++ b/internal/generator/generator_test.go
@@ -3579,6 +3579,7 @@ func TestGenerateWhichFallsBackToCommandTree(t *testing.T) {
 	assert.Contains(t, whichSrc, `Command: "products list"`)
 	assert.Contains(t, whichSrc, `Description: "List products"`)
 	assert.Contains(t, whichSrc, `Command: "products reviews list"`)
+	assert.Contains(t, whichSrc, `"pp:typed-exit-codes": "0,2"`)
 
 	runGoCommand(t, outputDir, "mod", "tidy")
 	binaryPath := filepath.Join(outputDir, "whichfallback-pp-cli")
diff --git a/internal/generator/templates/which.go.tmpl b/internal/generator/templates/which.go.tmpl
index 3d4f5469..f3940862 100644
--- a/internal/generator/templates/which.go.tmpl
+++ b/internal/generator/templates/which.go.tmpl
@@ -137,6 +137,9 @@ func newWhichCmd(flags *rootFlags) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "which [query]",
 		Short: "Find the command that implements a capability",
+		Annotations: map[string]string{
+			"pp:typed-exit-codes": "0,2",
+		},
 		Long: `which resolves a natural-language capability query (for example, "search messages" or "stale tickets") to the best matching command from this CLI's curated feature index.
 
 Exit codes:
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
index bd28ebd9..fdc380bb 100644
--- a/internal/pipeline/dogfood.go
+++ b/internal/pipeline/dogfood.go
@@ -1734,18 +1734,6 @@ func runDogfoodCmd(binary string, timeout time.Duration, args ...string) (string
 // continuation of the Examples block — losing-by-default is safer than
 // misclassifying real example content as a section boundary.
 func extractExamplesSection(helpOutput string) string {
-	// Canonical Cobra section header set. Match on the trimmed line being
-	// exactly equal to one of these (case-sensitive — Cobra's emission is).
-	cobraSectionHeaders := map[string]struct{}{
-		"Usage:":                  {},
-		"Aliases:":                {},
-		"Available Commands:":     {},
-		"Examples:":               {},
-		"Flags:":                  {},
-		"Global Flags:":           {},
-		"Additional help topics:": {},
-	}
-
 	lines := strings.Split(helpOutput, "\n")
 	var inExamples bool
 	var examples []string
@@ -1759,7 +1747,7 @@ func extractExamplesSection(helpOutput string) string {
 			continue
 		}
 		// Section boundary: a known Cobra section header.
-		if _, ok := cobraSectionHeaders[trimmed]; ok {
+		if isCobraHelpSectionHeader(trimmed) {
 			break
 		}
 		// Cobra emits a "Use \"<root> <subcommand> [command] --help\" for
diff --git a/internal/pipeline/runtime.go b/internal/pipeline/runtime.go
index 78f26361..ba5e6eeb 100644
--- a/internal/pipeline/runtime.go
+++ b/internal/pipeline/runtime.go
@@ -311,7 +311,9 @@ func runCommandTests(binary string, cmd discoveredCommand, mode string, env []st
 	}
 
 	// Test 1: --help
-	result.Help = runCLI(binary, []string{cmd.Name, "--help"}, env, 10*time.Second) == nil
+	helpOutput, helpErr := runCLIWithOutput(binary, []string{cmd.Name, "--help"}, env, 10*time.Second)
+	result.Help = helpErr == nil
+	typedCodes := typedSuccessCodes(cmd, string(helpOutput))
 
 	// Get any required flags/args for this command.
 	// First, probe the binary for cobra-declared required flags (generic, spec-agnostic).
@@ -334,7 +336,7 @@ func runCommandTests(binary string, cmd discoveredCommand, mode string, env []st
 	if cmd.Kind != "local" && cmd.Kind != "data-layer" {
 		args := buildTestArgs(cmd.Name, cmd.Args, extraFlags, "--dry-run")
 		err := runCLI(binary, args, env, 10*time.Second)
-		result.DryRun = err == nil || isIntentionalStubExit(err)
+		result.DryRun = err == nil || isIntentionalStubExit(err) || isDocumentedSuccessExit(err, typedCodes)
 	} else {
 		result.DryRun = true // skip = pass
 	}
@@ -347,7 +349,7 @@ func runCommandTests(binary string, cmd discoveredCommand, mode string, env []st
 	} else {
 		args := buildTestArgs(cmd.Name, cmd.Args, extraFlags, "--json")
 		err := runCLI(binary, args, env, 15*time.Second)
-		result.Execute = err == nil || isIntentionalStubExit(err)
+		result.Execute = err == nil || isIntentionalStubExit(err) || isDocumentedSuccessExit(err, typedCodes)
 	}
 
 	// Score
diff --git a/internal/pipeline/runtime_annotations.go b/internal/pipeline/runtime_annotations.go
new file mode 100644
index 00000000..658e404b
--- /dev/null
+++ b/internal/pipeline/runtime_annotations.go
@@ -0,0 +1,135 @@
+package pipeline
+
+import (
+	"bytes"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func enrichCommandAnnotationsFromSource(dir string, commands []discoveredCommand) []discoveredCommand {
+	if len(commands) == 0 {
+		return commands
+	}
+	annotations := sourceCommandAnnotations(dir)
+	if len(annotations) == 0 {
+		return commands
+	}
+	for i := range commands {
+		if found := annotations[commands[i].Name]; len(found) > 0 {
+			commands[i].Annotations = found
+		}
+	}
+	return commands
+}
+
+func sourceCommandAnnotations(dir string) map[string]map[string]string {
+	if dir == "" {
+		return nil
+	}
+	cliDir := filepath.Join(dir, "internal", "cli")
+	files := listGoFiles(cliDir)
+	if len(files) == 0 {
+		return nil
+	}
+
+	fset := token.NewFileSet()
+	out := map[string]map[string]string{}
+	for _, path := range files {
+		data, err := os.ReadFile(path)
+		if err != nil || !bytes.Contains(data, []byte(typedExitCodesAnnotation)) {
+			continue
+		}
+		file, err := parser.ParseFile(fset, path, data, parser.SkipObjectResolution)
+		if err != nil {
+			continue
+		}
+		ast.Inspect(file, func(n ast.Node) bool {
+			lit, ok := n.(*ast.CompositeLit)
+			if !ok || !isCobraCommandLiteral(lit) {
+				return true
+			}
+			name, annotations := commandLiteralMetadata(lit)
+			if name != "" && len(annotations) > 0 {
+				out[name] = annotations
+			}
+			return true
+		})
+	}
+	return out
+}
+
+func isCobraCommandLiteral(lit *ast.CompositeLit) bool {
+	switch typ := lit.Type.(type) {
+	case *ast.SelectorExpr:
+		return typ.Sel.Name == "Command"
+	case *ast.Ident:
+		return typ.Name == "Command"
+	default:
+		return false
+	}
+}
+
+func commandLiteralMetadata(lit *ast.CompositeLit) (string, map[string]string) {
+	var name string
+	var annotations map[string]string
+	for _, elt := range lit.Elts {
+		kv, ok := elt.(*ast.KeyValueExpr)
+		if !ok {
+			continue
+		}
+		key, ok := kv.Key.(*ast.Ident)
+		if !ok {
+			continue
+		}
+		switch key.Name {
+		case "Use":
+			name = commandNameFromUse(stringLiteralValue(kv.Value))
+		case "Annotations":
+			annotations = stringMapLiteral(kv.Value)
+		}
+	}
+	return name, annotations
+}
+
+func commandNameFromUse(use string) string {
+	if match := cobraUseLeafRe.FindStringSubmatch(`Use: "` + use + `"`); match != nil {
+		return match[1]
+	}
+	return ""
+}
+
+func stringLiteralValue(expr ast.Expr) string {
+	lit, ok := expr.(*ast.BasicLit)
+	if !ok || lit.Kind != token.STRING {
+		return ""
+	}
+	value, err := strconv.Unquote(lit.Value)
+	if err != nil {
+		return ""
+	}
+	return value
+}
+
+func stringMapLiteral(expr ast.Expr) map[string]string {
+	lit, ok := expr.(*ast.CompositeLit)
+	if !ok {
+		return nil
+	}
+	out := map[string]string{}
+	for _, elt := range lit.Elts {
+		kv, ok := elt.(*ast.KeyValueExpr)
+		if !ok {
+			continue
+		}
+		key := stringLiteralValue(kv.Key)
+		value := stringLiteralValue(kv.Value)
+		if key != "" {
+			out[key] = value
+		}
+	}
+	return out
+}
diff --git a/internal/pipeline/runtime_commands.go b/internal/pipeline/runtime_commands.go
index 517dc6f9..8abae637 100644
--- a/internal/pipeline/runtime_commands.go
+++ b/internal/pipeline/runtime_commands.go
@@ -16,11 +16,11 @@ import (
 func discoverCommands(dir string, binaryPath string) []discoveredCommand {
 	if binaryPath != "" {
 		if cmds := discoverCommandsFromHelp(binaryPath); len(cmds) > 0 {
-			return cmds
+			return enrichCommandAnnotationsFromSource(dir, cmds)
 		}
 	}
 
-	return discoverCommandsFromSource(dir)
+	return enrichCommandAnnotationsFromSource(dir, discoverCommandsFromSource(dir))
 }
 
 func discoverCommandsFromHelp(binaryPath string) []discoveredCommand {
@@ -107,9 +107,10 @@ func discoverCommandsFromSource(dir string) []discoveredCommand {
 }
 
 type discoveredCommand struct {
-	Name string
-	Kind string // read, write, local, data-layer
-	Args []string
+	Name        string
+	Kind        string // read, write, local, data-layer
+	Args        []string
+	Annotations map[string]string
 }
 
 // inferPositionalArgs runs `<binary> <cmd> --help`, parses the Usage line for
diff --git a/internal/pipeline/runtime_exec.go b/internal/pipeline/runtime_exec.go
index 14b33706..d85539cc 100644
--- a/internal/pipeline/runtime_exec.go
+++ b/internal/pipeline/runtime_exec.go
@@ -95,7 +95,7 @@ func runCLIWithOutput(binary string, args []string, env []string, timeout time.D
 	cmd.Env = env
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return out, fmt.Errorf("exit %v: %s", err, string(out))
+		return out, fmt.Errorf("exit %w: %s", err, string(out))
 	}
 	return out, nil
 }
diff --git a/internal/pipeline/runtime_exitcodes.go b/internal/pipeline/runtime_exitcodes.go
new file mode 100644
index 00000000..2ebb27ac
--- /dev/null
+++ b/internal/pipeline/runtime_exitcodes.go
@@ -0,0 +1,114 @@
+package pipeline
+
+import (
+	"errors"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+const typedExitCodesAnnotation = "pp:typed-exit-codes"
+
+var (
+	exitCodeLineRe   = regexp.MustCompile(`^\s+(\d+)\s+.+$`)
+	exitStatusCodeRe = regexp.MustCompile(`exit status (\d+)`)
+	cobraHelpHeaders = map[string]struct{}{
+		"Usage:":                  {},
+		"Aliases:":                {},
+		"Available Commands:":     {},
+		"Examples:":               {},
+		"Flags:":                  {},
+		"Global Flags:":           {},
+		"Additional help topics:": {},
+	}
+)
+
+func typedSuccessCodes(cmd discoveredCommand, helpOutput string) map[int]bool {
+	if cmd.Annotations != nil {
+		if raw := strings.TrimSpace(cmd.Annotations[typedExitCodesAnnotation]); raw != "" {
+			if codes, ok := parseTypedExitCodesAnnotation(raw); ok {
+				return codes
+			}
+		}
+	}
+	if codes, ok := parseExitCodesFromHelp(helpOutput); ok {
+		return codes
+	}
+	return map[int]bool{0: true}
+}
+
+func parseTypedExitCodesAnnotation(raw string) (map[int]bool, bool) {
+	codes := map[int]bool{}
+	for part := range strings.SplitSeq(raw, ",") {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		code, err := strconv.Atoi(part)
+		if err != nil || code < 0 {
+			return nil, false
+		}
+		codes[code] = true
+	}
+	return codes, len(codes) > 0
+}
+
+func parseExitCodesFromHelp(helpOutput string) (map[int]bool, bool) {
+	codes := map[int]bool{}
+	inBlock := false
+
+	for line := range strings.SplitSeq(helpOutput, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if !inBlock {
+			if strings.EqualFold(trimmed, "Exit codes:") {
+				inBlock = true
+			}
+			continue
+		}
+
+		if trimmed == "" || isCobraHelpSectionHeader(trimmed) || trimmed == "Use:" {
+			break
+		}
+
+		match := exitCodeLineRe.FindStringSubmatch(line)
+		if match == nil {
+			if len(codes) > 0 {
+				break
+			}
+			continue
+		}
+		code, err := strconv.Atoi(match[1])
+		if err == nil {
+			codes[code] = true
+		}
+	}
+
+	return codes, len(codes) > 0
+}
+
+func isCobraHelpSectionHeader(line string) bool {
+	_, ok := cobraHelpHeaders[line]
+	return ok
+}
+
+func isDocumentedSuccessExit(err error, codes map[int]bool) bool {
+	code, ok := extractExitCode(err)
+	return ok && codes[code]
+}
+
+func extractExitCode(err error) (int, bool) {
+	if err == nil {
+		return 0, false
+	}
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		return exitErr.ExitCode(), true
+	}
+	match := exitStatusCodeRe.FindStringSubmatch(err.Error())
+	if match == nil {
+		return 0, false
+	}
+	code, convErr := strconv.Atoi(match[1])
+	return code, convErr == nil
+}
diff --git a/internal/pipeline/runtime_test.go b/internal/pipeline/runtime_test.go
index 5561f274..24ef75c0 100644
--- a/internal/pipeline/runtime_test.go
+++ b/internal/pipeline/runtime_test.go
@@ -591,6 +591,144 @@ func TestIsIntentionalStubExit(t *testing.T) {
 	assert.False(t, isIntentionalStubExit(nil))
 }
 
+func TestParseExitCodesFromHelp(t *testing.T) {
+	tests := []struct {
+		name string
+		help string
+		want map[int]bool
+		ok   bool
+	}{
+		{
+			name: "command help block",
+			help: `Resolve capabilities.
+
+Exit codes:
+  0  at least one match found
+  2  no confident match - fall back to help
+
+Flags:
+  -h, --help   help for which`,
+			want: map[int]bool{0: true, 2: true},
+			ok:   true,
+		},
+		{
+			name: "next section without blank",
+			help: `Resolve capabilities.
+
+Exit codes:
+  0  at least one match found
+  2  no confident match
+Examples:
+  cli which search`,
+			want: map[int]bool{0: true, 2: true},
+			ok:   true,
+		},
+		{
+			name: "malformed block",
+			help: `Resolve capabilities.
+
+Exit codes:
+success only`,
+			want: nil,
+			ok:   false,
+		},
+		{
+			name: "no block",
+			help: `Resolve capabilities.
+
+Flags:
+  -h, --help   help for which`,
+			want: nil,
+			ok:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := parseExitCodesFromHelp(tt.help)
+			assert.Equal(t, tt.ok, ok)
+			if tt.ok {
+				assert.Equal(t, tt.want, got)
+			}
+		})
+	}
+}
+
+func TestParseTypedExitCodesAnnotation(t *testing.T) {
+	got, ok := parseTypedExitCodesAnnotation("0,2")
+	assert.True(t, ok)
+	assert.Equal(t, map[int]bool{0: true, 2: true}, got)
+
+	got, ok = parseTypedExitCodesAnnotation("0, 2, 3")
+	assert.True(t, ok)
+	assert.Equal(t, map[int]bool{0: true, 2: true, 3: true}, got)
+
+	_, ok = parseTypedExitCodesAnnotation("")
+	assert.False(t, ok)
+	_, ok = parseTypedExitCodesAnnotation("abc")
+	assert.False(t, ok)
+	_, ok = parseTypedExitCodesAnnotation("-1")
+	assert.False(t, ok)
+}
+
+func TestTypedSuccessCodesAnnotationWinsOverHelp(t *testing.T) {
+	cmd := discoveredCommand{
+		Name: "which",
+		Annotations: map[string]string{
+			typedExitCodesAnnotation: "0,2",
+		},
+	}
+	help := `Exit codes:
+  0  success
+  3  not found`
+
+	assert.Equal(t, map[int]bool{0: true, 2: true}, typedSuccessCodes(cmd, help))
+}
+
+func TestTypedSuccessCodesMalformedAnnotationFallsBackToHelp(t *testing.T) {
+	cmd := discoveredCommand{
+		Name: "which",
+		Annotations: map[string]string{
+			typedExitCodesAnnotation: "0,nope",
+		},
+	}
+	help := `Exit codes:
+  0  success
+  2  no confident match`
+
+	assert.Equal(t, map[int]bool{0: true, 2: true}, typedSuccessCodes(cmd, help))
+}
+
+func TestIsDocumentedSuccessExit(t *testing.T) {
+	err := exec.Command("sh", "-c", "exit 2").Run()
+	require.Error(t, err)
+
+	wrapped := fmt.Errorf("exit %w: no confident match", err)
+	assert.True(t, isDocumentedSuccessExit(wrapped, map[int]bool{0: true, 2: true}))
+	assert.False(t, isDocumentedSuccessExit(wrapped, map[int]bool{0: true}))
+	assert.False(t, isDocumentedSuccessExit(nil, map[int]bool{0: true, 2: true}))
+}
+
+func TestEnrichCommandAnnotationsFromSource(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+	writeTestFile(t, filepath.Join(dir, "internal", "cli", "which.go"), `package cli
+
+import "github.com/spf13/cobra"
+
+func newWhichCmd() *cobra.Command {
+	return &cobra.Command{
+		Use: "which [query]",
+		Annotations: map[string]string{"pp:typed-exit-codes": "0,2"},
+	}
+}
+`)
+
+	commands := enrichCommandAnnotationsFromSource(dir, []discoveredCommand{{Name: "which"}})
+	require.Len(t, commands, 1)
+	assert.Equal(t, "0,2", commands[0].Annotations[typedExitCodesAnnotation])
+}
+
 func TestSyntheticFlagValue(t *testing.T) {
 	tests := []struct {
 		name     string
diff --git a/skills/printing-press/SKILL.md b/skills/printing-press/SKILL.md
index 6a23133b..1463dd4c 100644
--- a/skills/printing-press/SKILL.md
+++ b/skills/printing-press/SKILL.md
@@ -1999,6 +1999,8 @@ For features that combine both (cache an API response in the store, or fall thro
 
 **Shared helpers available to novel code:** The generator emits `internal/cliutil/` in every CLI. When authoring novel commands, prefer `cliutil.FanoutRun` for any aggregation command (any `--site`/`--source`/`--region` CSV fan-out) and `cliutil.CleanText` for any text extracted from HTML or schema.org JSON-LD. Re-implementing these inline is how recipe-goat's trending silent-drop and `&#39;` entity bugs shipped.
 
+**Typed exit-code verification:** If a novel command intentionally returns a non-zero code for a non-error control-flow result, add `cmd.Annotations["pp:typed-exit-codes"] = "0,<code>"` (or the equivalent `Annotations: map[string]string{...}` literal) and document the same command-specific codes in its help. Do not list the global failure palette in command help unless those exits should count as a verify pass for that command; keep general exit-code troubleshooting in README/SKILL prose.
+
 **MCP exposure:** The generator emits `internal/mcp/cobratree/`, and the MCP binary mirrors the Cobra tree at startup. When you add, rename, or remove a user-facing Cobra command, the MCP surface follows automatically. Two annotations control how each command appears as an MCP tool:
 
 - `cmd.Annotations["mcp:hidden"] = "true"` — exclude the command from the MCP surface entirely. Use only for debug/internal commands that should not become agent tools.

← 97180c03 fix(cli): route discriminator sync to typed tables (#503)  ·  back to Cli Printing Press  ·  docs(cli): bump README "Go 1.22+" prerequisite to "Go 1.26+" 6e2562d8 →