← back to Cli Printing Press
feat(cli): add live dogfood matrix runner (#559)
e973d5c505e6cab5a5ed163ed736112698d95325 · 2026-05-03 20:32:21 -0700 · Trevin Chow
* feat(cli): add live dogfood matrix runner
* fix(cli): simplify live dogfood safeguards
Files touched
M internal/cli/catalog_test.goM internal/cli/dogfood.goM internal/cli/dogfood_test.goA internal/pipeline/live_dogfood.goA internal/pipeline/live_dogfood_test.goM skills/printing-press/SKILL.md
Diff
commit e973d5c505e6cab5a5ed163ed736112698d95325
Author: Trevin Chow <trevin@trevinchow.com>
Date: Sun May 3 20:32:21 2026 -0700
feat(cli): add live dogfood matrix runner (#559)
* feat(cli): add live dogfood matrix runner
* fix(cli): simplify live dogfood safeguards
---
internal/cli/catalog_test.go | 10 +
internal/cli/dogfood.go | 56 ++++
internal/cli/dogfood_test.go | 24 +-
internal/pipeline/live_dogfood.go | 481 +++++++++++++++++++++++++++++++++
internal/pipeline/live_dogfood_test.go | 270 ++++++++++++++++++
skills/printing-press/SKILL.md | 45 ++-
6 files changed, 845 insertions(+), 41 deletions(-)
diff --git a/internal/cli/catalog_test.go b/internal/cli/catalog_test.go
index c8393acd..71fb390b 100644
--- a/internal/cli/catalog_test.go
+++ b/internal/cli/catalog_test.go
@@ -28,6 +28,16 @@ func runWithCapturedStdout(t *testing.T, fn func() error) (string, error) {
return string(out), execErr
}
+func captureStdout(t *testing.T, fn func()) string {
+ t.Helper()
+ out, err := runWithCapturedStdout(t, func() error {
+ fn()
+ return nil
+ })
+ require.NoError(t, err)
+ return out
+}
+
func TestCatalogListJSON(t *testing.T) {
cmd := newCatalogCmd()
cmd.SetArgs([]string{"list", "--json"})
diff --git a/internal/cli/dogfood.go b/internal/cli/dogfood.go
index 61164cba..845d703a 100644
--- a/internal/cli/dogfood.go
+++ b/internal/cli/dogfood.go
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strings"
+ "time"
"github.com/mvanhorn/cli-printing-press/v3/internal/pipeline"
"github.com/spf13/cobra"
@@ -16,6 +17,11 @@ func newDogfoodCmd() *cobra.Command {
var specPath string
var researchDir string
var asJSON bool
+ var live bool
+ var level string
+ var timeout time.Duration
+ var writeAcceptance string
+ var authEnv string
cmd := &cobra.Command{
Use: "dogfood",
@@ -27,6 +33,32 @@ func newDogfoodCmd() *cobra.Command {
# Output as JSON for programmatic use
printing-press dogfood --dir ./generated/stripe-pp-cli --json`,
RunE: func(cmd *cobra.Command, args []string) error {
+ if live {
+ report, err := pipeline.RunLiveDogfood(pipeline.LiveDogfoodOptions{
+ CLIDir: dir,
+ Level: level,
+ Timeout: timeout,
+ WriteAcceptancePath: writeAcceptance,
+ AuthEnv: authEnv,
+ })
+ if err != nil {
+ return &ExitError{Code: ExitGenerationError, Err: fmt.Errorf("running live dogfood: %w", err)}
+ }
+ if asJSON {
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetIndent("", " ")
+ if err := enc.Encode(report); err != nil {
+ return err
+ }
+ } else {
+ printLiveDogfoodReport(report)
+ }
+ if report.Verdict != "PASS" {
+ return &ExitError{Code: ExitGenerationError, Err: fmt.Errorf("live dogfood failed: %d/%d tests failed", report.Failed, report.MatrixSize)}
+ }
+ return nil
+ }
+
var opts []pipeline.DogfoodOption
if researchDir != "" {
opts = append(opts, pipeline.WithResearchDir(researchDir))
@@ -51,10 +83,34 @@ func newDogfoodCmd() *cobra.Command {
cmd.Flags().StringVar(&specPath, "spec", "", "Path to the OpenAPI spec file")
cmd.Flags().StringVar(&researchDir, "research-dir", "", "Pipeline directory containing research.json for novel features validation")
cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON")
+ cmd.Flags().BoolVar(&live, "live", false, "Run the Phase 5 live command-tree dogfood matrix")
+ cmd.Flags().StringVar(&level, "level", "full", "Live dogfood depth: quick or full")
+ cmd.Flags().DurationVar(&timeout, "timeout", 30*time.Second, "Timeout for each live dogfood test")
+ cmd.Flags().StringVar(&writeAcceptance, "write-acceptance", "", "Write phase5-acceptance.json to this path when live dogfood passes")
+ cmd.Flags().StringVar(&authEnv, "auth-env", "", "Environment variable that proves an API credential was available for the acceptance marker")
_ = cmd.MarkFlagRequired("dir")
return cmd
}
+func printLiveDogfoodReport(report *pipeline.LiveDogfoodReport) {
+ fmt.Printf("Live Dogfood Report: %s\n", filepath.Base(report.Dir))
+ fmt.Println("================================")
+ fmt.Println()
+ fmt.Printf("Level: %s\n", report.Level)
+ fmt.Printf("Verdict: %s\n", report.Verdict)
+ fmt.Printf("Commands: %d\n", len(report.Commands))
+ fmt.Printf("Tests: %d passed, %d failed, %d skipped\n", report.Passed, report.Failed, report.Skipped)
+ fmt.Println()
+ for _, result := range report.Tests {
+ status := strings.ToUpper(string(result.Status))
+ fmt.Printf("[%s] %s %s", status, result.Command, result.Kind)
+ if result.Reason != "" {
+ fmt.Printf(": %s", result.Reason)
+ }
+ fmt.Println()
+ }
+}
+
func printDogfoodReport(report *pipeline.DogfoodReport) {
name := filepath.Base(report.Dir)
diff --git a/internal/cli/dogfood_test.go b/internal/cli/dogfood_test.go
index 60529157..d808c4a9 100644
--- a/internal/cli/dogfood_test.go
+++ b/internal/cli/dogfood_test.go
@@ -1,9 +1,6 @@
package cli
import (
- "bytes"
- "io"
- "os"
"testing"
"github.com/mvanhorn/cli-printing-press/v3/internal/pipeline"
@@ -30,21 +27,14 @@ func TestPrintDogfoodReportRespectsSkippedPathCheck(t *testing.T) {
assert.NotContains(t, out, "Path Validity: 0/0 valid (FAIL)")
}
-func captureStdout(t *testing.T, fn func()) string {
- t.Helper()
+func TestDogfoodHelpIncludesLiveFlags(t *testing.T) {
+ cmd := newDogfoodCmd()
+ cmd.SetArgs([]string{"--help"})
- orig := os.Stdout
- r, w, err := os.Pipe()
+ output, err := runWithCapturedStdout(t, cmd.Execute)
require.NoError(t, err)
- os.Stdout = w
- defer func() { os.Stdout = orig }()
- fn()
- require.NoError(t, w.Close())
-
- var buf bytes.Buffer
- _, err = io.Copy(&buf, r)
- require.NoError(t, err)
- require.NoError(t, r.Close())
- return buf.String()
+ assert.Contains(t, output, "--live")
+ assert.Contains(t, output, "--level")
+ assert.Contains(t, output, "--write-acceptance")
}
diff --git a/internal/pipeline/live_dogfood.go b/internal/pipeline/live_dogfood.go
new file mode 100644
index 00000000..9d231b45
--- /dev/null
+++ b/internal/pipeline/live_dogfood.go
@@ -0,0 +1,481 @@
+package pipeline
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "slices"
+ "sort"
+ "strings"
+ "time"
+)
+
+type LiveDogfoodStatus string
+
+const (
+ LiveDogfoodStatusPass LiveDogfoodStatus = "pass"
+ LiveDogfoodStatusFail LiveDogfoodStatus = "fail"
+ LiveDogfoodStatusSkip LiveDogfoodStatus = "skip"
+)
+
+type LiveDogfoodTestKind string
+
+const (
+ LiveDogfoodTestHelp LiveDogfoodTestKind = "help"
+ LiveDogfoodTestHappy LiveDogfoodTestKind = "happy_path"
+ LiveDogfoodTestJSON LiveDogfoodTestKind = "json_fidelity"
+ LiveDogfoodTestError LiveDogfoodTestKind = "error_path"
+)
+
+type LiveDogfoodOptions struct {
+ CLIDir string
+ BinaryName string
+ Level string
+ Timeout time.Duration
+ WriteAcceptancePath string
+ AuthEnv string
+}
+
+type LiveDogfoodReport struct {
+ Dir string `json:"dir"`
+ Binary string `json:"binary"`
+ Level string `json:"level"`
+ Verdict string `json:"verdict"`
+ MatrixSize int `json:"matrix_size"`
+ Passed int `json:"passed"`
+ Failed int `json:"failed"`
+ Skipped int `json:"skipped"`
+ Commands []string `json:"commands"`
+ Tests []LiveDogfoodTestResult `json:"tests"`
+ RanAt time.Time `json:"ran_at"`
+}
+
+type LiveDogfoodTestResult struct {
+ Command string `json:"command"`
+ Kind LiveDogfoodTestKind `json:"kind"`
+ Args []string `json:"args"`
+ Status LiveDogfoodStatus `json:"status"`
+ ExitCode int `json:"exit_code,omitempty"`
+ Reason string `json:"reason,omitempty"`
+ OutputSample string `json:"output_sample,omitempty"`
+}
+
+type liveDogfoodCommand struct {
+ Path []string
+ Help string
+}
+
+type liveDogfoodRun struct {
+ stdout string
+ stderr string
+ exitCode int
+ err error
+}
+
+func RunLiveDogfood(opts LiveDogfoodOptions) (*LiveDogfoodReport, error) {
+ if strings.TrimSpace(opts.CLIDir) == "" {
+ return nil, fmt.Errorf("CLIDir is required")
+ }
+ level, err := normalizeLiveDogfoodLevel(opts.Level)
+ if err != nil {
+ return nil, err
+ }
+
+ timeout := opts.Timeout
+ if timeout <= 0 {
+ timeout = 30 * time.Second
+ }
+
+ binaryPath, err := liveDogfoodBinaryPath(opts.CLIDir, opts.BinaryName)
+ if err != nil {
+ return nil, err
+ }
+
+ commands, err := discoverLiveDogfoodCommands(binaryPath)
+ if err != nil {
+ return nil, err
+ }
+ if level == "quick" {
+ commands = liveDogfoodQuickCommands(commands)
+ }
+ if len(commands) == 0 {
+ return nil, fmt.Errorf("no live dogfood command leaves discovered")
+ }
+
+ report := &LiveDogfoodReport{
+ Dir: opts.CLIDir,
+ Binary: binaryPath,
+ Level: level,
+ Verdict: "PASS",
+ RanAt: time.Now().UTC(),
+ }
+
+ for _, command := range commands {
+ commandName := strings.Join(command.Path, " ")
+ report.Commands = append(report.Commands, commandName)
+ report.Tests = append(report.Tests, runLiveDogfoodCommand(binaryPath, opts.CLIDir, command, timeout)...)
+ }
+
+ finalizeLiveDogfoodReport(report)
+ if report.Verdict == "PASS" && opts.WriteAcceptancePath != "" {
+ if err := writeLiveDogfoodAcceptance(opts, report); err != nil {
+ return nil, err
+ }
+ }
+ return report, nil
+}
+
+func liveDogfoodBinaryPath(dir, name string) (string, error) {
+ if path, err := resolveBinaryPath(dir, name); err == nil {
+ return path, nil
+ } else if strings.TrimSpace(name) != "" {
+ return "", err
+ }
+
+ cliName := findCLIName(dir)
+ if cliName == "" {
+ return "", fmt.Errorf("no runnable binary found in %q and no cmd/<cli-name> package to build", dir)
+ }
+ return buildDogfoodBinary(dir, cliName)
+}
+
+func discoverLiveDogfoodCommands(binaryPath string) ([]liveDogfoodCommand, error) {
+ out, err := runStdoutOnly(binaryPath, 15*time.Second, "agent-context")
+ if err != nil {
+ return nil, fmt.Errorf("agent-context failed: %w", err)
+ }
+
+ var ctx dogfoodAgentContext
+ if err := json.Unmarshal(out, &ctx); err != nil {
+ return nil, fmt.Errorf("parsing agent-context: %w", err)
+ }
+
+ var paths [][]string
+ for _, command := range ctx.Commands {
+ collectLiveDogfoodCommandPaths(nil, command, &paths)
+ }
+ sort.Slice(paths, func(i, j int) bool {
+ return strings.Join(paths[i], " ") < strings.Join(paths[j], " ")
+ })
+
+ commands := make([]liveDogfoodCommand, 0, len(paths))
+ for _, path := range paths {
+ commands = append(commands, liveDogfoodCommand{Path: path})
+ }
+ return commands, nil
+}
+
+var liveDogfoodFrameworkSkip = map[string]bool{
+ "agent-context": true,
+ "completion": true,
+ "help": true,
+ "version": true,
+}
+
+func collectLiveDogfoodCommandPaths(prefix []string, command dogfoodAgentCommand, paths *[][]string) {
+ if command.Name == "" || liveDogfoodFrameworkSkip[command.Name] {
+ return
+ }
+
+ next := append(append([]string{}, prefix...), command.Name)
+ if len(command.Subcommands) == 0 {
+ *paths = append(*paths, next)
+ return
+ }
+ for _, sub := range command.Subcommands {
+ collectLiveDogfoodCommandPaths(next, sub, paths)
+ }
+}
+
+func runLiveDogfoodCommand(binaryPath, cliDir string, command liveDogfoodCommand, timeout time.Duration) []LiveDogfoodTestResult {
+ commandName := strings.Join(command.Path, " ")
+
+ helpArgs := append(append([]string{}, command.Path...), "--help")
+ helpRun := runLiveDogfoodProcess(binaryPath, cliDir, helpArgs, timeout)
+ helpResult := liveDogfoodResult(commandName, LiveDogfoodTestHelp, helpArgs, helpRun)
+ helpPassed := helpRun.exitCode == 0
+ help := helpRun.stdout + helpRun.stderr
+ if helpPassed && extractExamplesSection(help) == "" {
+ helpPassed = false
+ helpResult.Status = LiveDogfoodStatusFail
+ helpResult.Reason = "missing Examples section"
+ }
+ if helpPassed {
+ helpResult.Status = LiveDogfoodStatusPass
+ helpResult.Reason = ""
+ }
+
+ results := []LiveDogfoodTestResult{helpResult}
+ if !helpPassed {
+ results = append(results,
+ skippedLiveDogfoodResult(commandName, LiveDogfoodTestHappy, "help check failed"),
+ skippedLiveDogfoodResult(commandName, LiveDogfoodTestJSON, "help check failed"),
+ skippedLiveDogfoodResult(commandName, LiveDogfoodTestError, "help check failed"),
+ )
+ return results
+ }
+
+ command.Help = help
+ happyArgs, ok := liveDogfoodHappyArgs(command)
+ if !ok {
+ results = append(results,
+ failedLiveDogfoodResult(commandName, LiveDogfoodTestHappy, command.Path, "missing runnable example"),
+ skippedLiveDogfoodResult(commandName, LiveDogfoodTestJSON, "missing runnable example"),
+ skippedLiveDogfoodResult(commandName, LiveDogfoodTestError, "missing runnable example"),
+ )
+ return results
+ }
+
+ happyRun := runLiveDogfoodProcess(binaryPath, cliDir, happyArgs, timeout)
+ happyResult := liveDogfoodResult(commandName, LiveDogfoodTestHappy, happyArgs, happyRun)
+ if happyRun.exitCode == 0 {
+ happyResult.Status = LiveDogfoodStatusPass
+ happyResult.Reason = ""
+ }
+ results = append(results, happyResult)
+
+ if commandSupportsJSON(command.Help) {
+ jsonArgs := appendJSONArg(happyArgs)
+ jsonRun := runLiveDogfoodProcess(binaryPath, cliDir, jsonArgs, timeout)
+ jsonResult := liveDogfoodResult(commandName, LiveDogfoodTestJSON, jsonArgs, jsonRun)
+ if jsonRun.exitCode == 0 {
+ if !json.Valid([]byte(jsonRun.stdout)) {
+ jsonResult.Status = LiveDogfoodStatusFail
+ jsonResult.Reason = "invalid JSON"
+ } else {
+ jsonResult.Status = LiveDogfoodStatusPass
+ jsonResult.Reason = ""
+ }
+ }
+ results = append(results, jsonResult)
+ } else {
+ results = append(results, skippedLiveDogfoodResult(commandName, LiveDogfoodTestJSON, "--json not supported"))
+ }
+
+ if liveDogfoodCommandTakesArg(command.Help) {
+ errorArgs := append(append([]string{}, command.Path...), "__printing_press_invalid__")
+ errorRun := runLiveDogfoodProcess(binaryPath, cliDir, errorArgs, timeout)
+ errorResult := liveDogfoodResult(commandName, LiveDogfoodTestError, errorArgs, errorRun)
+ if errorRun.exitCode != 0 {
+ errorResult.Status = LiveDogfoodStatusPass
+ errorResult.Reason = ""
+ } else {
+ errorResult.Status = LiveDogfoodStatusFail
+ errorResult.Reason = "expected non-zero exit for invalid argument"
+ }
+ results = append(results, errorResult)
+ } else {
+ results = append(results, skippedLiveDogfoodResult(commandName, LiveDogfoodTestError, "no positional argument"))
+ }
+
+ return results
+}
+
+func runLiveDogfoodProcess(binaryPath, cliDir string, args []string, timeout time.Duration) liveDogfoodRun {
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ cmd := exec.CommandContext(ctx, binaryPath, args...)
+ cmd.Dir = cliDir
+ stdout := &bytes.Buffer{}
+ stderr := &bytes.Buffer{}
+ cmd.Stdout = &limitedWriter{w: stdout, remaining: MaxOutputBytes}
+ cmd.Stderr = &limitedWriter{w: stderr, remaining: MaxOutputBytes}
+
+ err := cmd.Run()
+ result := liveDogfoodRun{
+ stdout: stdout.String(),
+ stderr: stderr.String(),
+ exitCode: 0,
+ err: err,
+ }
+ if ctx.Err() == context.DeadlineExceeded {
+ result.exitCode = -1
+ result.err = fmt.Errorf("timed out after %s", timeout)
+ return result
+ }
+ if err != nil {
+ var exitErr *exec.ExitError
+ if errors.As(err, &exitErr) {
+ result.exitCode = exitErr.ExitCode()
+ } else {
+ result.exitCode = -1
+ }
+ }
+ return result
+}
+
+func liveDogfoodResult(command string, kind LiveDogfoodTestKind, args []string, run liveDogfoodRun) LiveDogfoodTestResult {
+ result := LiveDogfoodTestResult{
+ Command: command,
+ Kind: kind,
+ Args: append([]string{}, args...),
+ Status: LiveDogfoodStatusFail,
+ ExitCode: run.exitCode,
+ OutputSample: sampleOutput(run.stdout + run.stderr),
+ }
+ if run.exitCode != 0 {
+ result.Reason = fmt.Sprintf("exit %d", run.exitCode)
+ }
+ if run.err != nil && result.Reason == "" {
+ result.Reason = run.err.Error()
+ }
+ return result
+}
+
+func failedLiveDogfoodResult(command string, kind LiveDogfoodTestKind, args []string, reason string) LiveDogfoodTestResult {
+ return LiveDogfoodTestResult{
+ Command: command,
+ Kind: kind,
+ Args: append([]string{}, args...),
+ Status: LiveDogfoodStatusFail,
+ Reason: reason,
+ }
+}
+
+func skippedLiveDogfoodResult(command string, kind LiveDogfoodTestKind, reason string) LiveDogfoodTestResult {
+ return LiveDogfoodTestResult{
+ Command: command,
+ Kind: kind,
+ Status: LiveDogfoodStatusSkip,
+ Reason: reason,
+ }
+}
+
+func liveDogfoodHappyArgs(command liveDogfoodCommand) ([]string, bool) {
+ examples := extractExamplesSection(command.Help)
+ for line := range strings.SplitSeq(examples, "\n") {
+ candidate := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "$"))
+ if candidate == "" || strings.HasPrefix(candidate, "#") {
+ continue
+ }
+ args, err := parseExampleArgs(candidate)
+ if err == nil && len(args) > 0 && slices.Equal(args[:min(len(command.Path), len(args))], command.Path) {
+ return args, true
+ }
+ }
+ return nil, false
+}
+
+func commandSupportsJSON(help string) bool {
+ return slices.Contains(extractFlagNames(help), "json")
+}
+
+func appendJSONArg(args []string) []string {
+ out := append([]string{}, args...)
+ for _, arg := range out {
+ if arg == "--json" || strings.HasPrefix(arg, "--json=") {
+ return out
+ }
+ }
+ return append(out, "--json")
+}
+
+func liveDogfoodCommandTakesArg(help string) bool {
+ usage := liveDogfoodUsageSuffix(help)
+ return len(extractPositionalPlaceholders(usage)) > 0
+}
+
+func liveDogfoodUsageSuffix(help string) string {
+ lines := strings.Split(help, "\n")
+ for i, line := range lines {
+ if strings.TrimSpace(line) != "Usage:" {
+ continue
+ }
+ if i+1 < len(lines) {
+ return lines[i+1]
+ }
+ }
+ return ""
+}
+
+func finalizeLiveDogfoodReport(report *LiveDogfoodReport) {
+ for _, result := range report.Tests {
+ switch result.Status {
+ case LiveDogfoodStatusPass:
+ report.Passed++
+ report.MatrixSize++
+ case LiveDogfoodStatusFail:
+ report.Failed++
+ report.MatrixSize++
+ default:
+ report.Skipped++
+ }
+ }
+ switch {
+ case report.Level == "quick" && report.MatrixSize == 6 && report.Passed >= 5:
+ report.Verdict = "PASS"
+ case report.Failed > 0 || report.MatrixSize == 0:
+ report.Verdict = "FAIL"
+ case report.Level == "quick" && report.MatrixSize != 6:
+ report.Verdict = "FAIL"
+ }
+}
+
+func writeLiveDogfoodAcceptance(opts LiveDogfoodOptions, report *LiveDogfoodReport) error {
+ manifest, err := ReadCLIManifest(opts.CLIDir)
+ if err != nil {
+ return fmt.Errorf("reading CLI manifest for phase5 acceptance: %w", err)
+ }
+ if manifest.APIName == "" {
+ return fmt.Errorf("CLI manifest missing api_name; cannot write phase5 acceptance")
+ }
+ if manifest.RunID == "" {
+ return fmt.Errorf("CLI manifest missing run_id; cannot write phase5 acceptance")
+ }
+ authType := manifest.AuthType
+ if authType == "" {
+ authType = "none"
+ }
+
+ marker := Phase5GateMarker{
+ SchemaVersion: 1,
+ APIName: manifest.APIName,
+ RunID: manifest.RunID,
+ Status: "pass",
+ Level: report.Level,
+ MatrixSize: report.MatrixSize,
+ TestsPassed: report.Passed,
+ TestsFailed: report.Failed,
+ AuthContext: Phase5AuthContext{
+ Type: authType,
+ APIKeyAvailable: opts.AuthEnv != "" && os.Getenv(opts.AuthEnv) != "",
+ },
+ }
+ data, err := json.MarshalIndent(marker, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshaling phase5 acceptance marker: %w", err)
+ }
+ if err := os.MkdirAll(filepath.Dir(opts.WriteAcceptancePath), 0o755); err != nil {
+ return fmt.Errorf("creating phase5 acceptance directory: %w", err)
+ }
+ if err := os.WriteFile(opts.WriteAcceptancePath, data, 0o644); err != nil {
+ return fmt.Errorf("writing phase5 acceptance marker: %w", err)
+ }
+ return nil
+}
+
+func liveDogfoodQuickCommands(commands []liveDogfoodCommand) []liveDogfoodCommand {
+ if len(commands) <= 2 {
+ return commands
+ }
+ return commands[:2]
+}
+
+func normalizeLiveDogfoodLevel(level string) (string, error) {
+ level = strings.ToLower(strings.TrimSpace(level))
+ if level == "" {
+ return "full", nil
+ }
+ switch level {
+ case "quick", "full":
+ return level, nil
+ default:
+ return "", fmt.Errorf("invalid live dogfood level %q (expected quick or full)", level)
+ }
+}
diff --git a/internal/pipeline/live_dogfood_test.go b/internal/pipeline/live_dogfood_test.go
new file mode 100644
index 00000000..398e5cec
--- /dev/null
+++ b/internal/pipeline/live_dogfood_test.go
@@ -0,0 +1,270 @@
+package pipeline
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRunLiveDogfoodDetectsJSONParseFailure(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir, binaryName := writeLiveDogfoodFixture(t, false)
+ report, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: binaryName,
+ Level: "full",
+ Timeout: 2 * time.Second,
+ })
+ require.NoError(t, err)
+
+ assert.Equal(t, "FAIL", report.Verdict)
+ assert.Greater(t, report.MatrixSize, 0)
+ assert.Greater(t, report.Failed, 0)
+
+ var jsonFailure *LiveDogfoodTestResult
+ for i := range report.Tests {
+ if report.Tests[i].Command == "widgets broken" && report.Tests[i].Kind == LiveDogfoodTestJSON {
+ jsonFailure = &report.Tests[i]
+ break
+ }
+ }
+ require.NotNil(t, jsonFailure)
+ assert.Equal(t, LiveDogfoodStatusFail, jsonFailure.Status)
+ assert.Contains(t, jsonFailure.Reason, "invalid JSON")
+}
+
+func TestRunLiveDogfoodWritesAcceptanceMarkerOnPass(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir, binaryName := writeLiveDogfoodFixture(t, true)
+ markerPath := filepath.Join(t.TempDir(), Phase5AcceptanceFilename)
+ report, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: binaryName,
+ Level: "full",
+ Timeout: 2 * time.Second,
+ WriteAcceptancePath: markerPath,
+ })
+ require.NoError(t, err)
+ require.Equal(t, "PASS", report.Verdict, report.Tests)
+
+ data, err := os.ReadFile(markerPath)
+ require.NoError(t, err)
+ var marker Phase5GateMarker
+ require.NoError(t, json.Unmarshal(data, &marker))
+ assert.Equal(t, "pass", marker.Status)
+ assert.Equal(t, "full", marker.Level)
+ assert.Equal(t, report.MatrixSize, marker.MatrixSize)
+ assert.Equal(t, report.Passed, marker.TestsPassed)
+ assert.Equal(t, 0, marker.TestsFailed)
+
+ validation := ValidatePhase5Gate(filepath.Dir(markerPath), CLIManifest{APIName: marker.APIName, RunID: marker.RunID, AuthType: "none"})
+ assert.True(t, validation.Passed, validation.Detail)
+}
+
+func TestRunLiveDogfoodErrorPathAcceptsExpectedNonZeroExit(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir, binaryName := writeLiveDogfoodFixture(t, true)
+ report, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: binaryName,
+ Level: "full",
+ Timeout: 2 * time.Second,
+ })
+ require.NoError(t, err)
+
+ var errorPath *LiveDogfoodTestResult
+ for i := range report.Tests {
+ if report.Tests[i].Command == "widgets get" && report.Tests[i].Kind == LiveDogfoodTestError {
+ errorPath = &report.Tests[i]
+ break
+ }
+ }
+ require.NotNil(t, errorPath)
+ assert.Equal(t, LiveDogfoodStatusPass, errorPath.Status)
+ assert.Equal(t, 2, errorPath.ExitCode)
+}
+
+func TestRunLiveDogfoodExplicitBinaryNameMustExist(t *testing.T) {
+ dir := t.TempDir()
+
+ _, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: "missing-pp-cli",
+ })
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "missing-pp-cli")
+}
+
+func TestRunLiveDogfoodAcceptanceRequiresManifestIdentity(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir, binaryName := writeLiveDogfoodFixture(t, true)
+ require.NoError(t, os.Remove(filepath.Join(dir, CLIManifestFilename)))
+
+ _, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: binaryName,
+ Level: "full",
+ Timeout: 2 * time.Second,
+ WriteAcceptancePath: filepath.Join(t.TempDir(), Phase5AcceptanceFilename),
+ })
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "CLI manifest")
+}
+
+func TestRunLiveDogfoodJSONFlagDetectionIsExact(t *testing.T) {
+ help := `Usage:
+ fixture-pp-cli widgets list [flags]
+
+Flags:
+ --json-output string Write JSON to a file
+`
+
+ assert.False(t, commandSupportsJSON(help))
+ assert.True(t, commandSupportsJSON(help+"\n --json Output JSON\n"))
+}
+
+func writeLiveDogfoodFixture(t *testing.T, brokenJSONFixed bool) (dir string, binaryName string) {
+ t.Helper()
+
+ dir = t.TempDir()
+ binaryName = "fixture-pp-cli"
+ writeTestManifestForLiveDogfood(t, dir)
+
+ binPath := filepath.Join(dir, binaryName)
+ brokenJSON := "{not-json"
+ if brokenJSONFixed {
+ brokenJSON = `{"ok":true}`
+ }
+ script := `#!/bin/sh
+set -u
+
+if [ "$1" = "agent-context" ]; then
+ cat <<'JSON'
+{
+ "commands": [
+ {"name":"widgets","subcommands":[
+ {"name":"list"},
+ {"name":"get"},
+ {"name":"broken"}
+ ]},
+ {"name":"completion","subcommands":[{"name":"bash"}]}
+ ]
+}
+JSON
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "list" ] && [ "${3:-}" = "--help" ]; then
+ cat <<'HELP'
+List widgets.
+
+Usage:
+ fixture-pp-cli widgets list [flags]
+
+Examples:
+ fixture-pp-cli widgets list --limit 2
+
+Flags:
+ --json Output JSON
+HELP
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "get" ] && [ "${3:-}" = "--help" ]; then
+ cat <<'HELP'
+Get a widget.
+
+Usage:
+ fixture-pp-cli widgets get <id> [flags]
+
+Examples:
+ fixture-pp-cli widgets get 123
+
+Flags:
+ --json Output JSON
+HELP
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "broken" ] && [ "${3:-}" = "--help" ]; then
+ cat <<'HELP'
+Return malformed JSON.
+
+Usage:
+ fixture-pp-cli widgets broken [flags]
+
+Examples:
+ fixture-pp-cli widgets broken
+
+Flags:
+ --json Output JSON
+HELP
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "list" ]; then
+ if [ "${3:-}" = "--limit" ] && [ "${4:-}" = "2" ] && [ "${5:-}" = "--json" ]; then
+ echo '{"widgets":[{"id":"1"}]}'
+ exit 0
+ fi
+ echo 'widget 1'
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "get" ]; then
+ if [ "${3:-}" = "__printing_press_invalid__" ]; then
+ echo 'not found' >&2
+ exit 2
+ fi
+ if [ "${4:-}" = "--json" ]; then
+ echo '{"id":"123"}'
+ exit 0
+ fi
+ echo 'widget 123'
+ exit 0
+fi
+
+if [ "$1" = "widgets" ] && [ "$2" = "broken" ]; then
+ if [ "${3:-}" = "--json" ]; then
+ echo '` + brokenJSON + `'
+ exit 0
+ fi
+ echo 'broken'
+ exit 0
+fi
+
+echo "unexpected args: $*" >&2
+exit 99
+`
+ require.NoError(t, os.WriteFile(binPath, []byte(script), 0o755))
+ return dir, binaryName
+}
+
+func writeTestManifestForLiveDogfood(t *testing.T, dir string) {
+ t.Helper()
+ require.NoError(t, WriteCLIManifest(dir, CLIManifest{
+ SchemaVersion: 1,
+ APIName: "fixture",
+ CLIName: "fixture-pp-cli",
+ RunID: "run-live-dogfood",
+ AuthType: "none",
+ }))
+}
diff --git a/skills/printing-press/SKILL.md b/skills/printing-press/SKILL.md
index 3d4b8244..cb990ecd 100644
--- a/skills/printing-press/SKILL.md
+++ b/skills/printing-press/SKILL.md
@@ -2329,33 +2329,32 @@ easiest to test and the most embarrassing to ship untested.
Do NOT proceed without asking. Do NOT substitute an ad-hoc smoke test. If some commands cannot be exercised because fixture values are missing, classify them as `BLOCKED_FIXTURE` and file/fix the machine gap; do not use that as a reason to recommend Quick.
-### Step 2: Build the test matrix mechanically
+### Step 2: Run the binary-owned test matrix
-**Full dogfood is not a judgment call about "enough."** Build the test matrix from the CLI's actual command tree:
-
-1. Parse `<cli> --help` recursively until every leaf subcommand is enumerated.
-2. Write the full command list to `$PROOFS_DIR/<stamp>-dogfood-matrix.txt` before running any tests.
-3. For each leaf subcommand, generate at minimum these tests:
- - **Help check**: `<cli> <subcmd> --help` returns exit 0 and produces an Examples section.
- - **Happy path**: one invocation with realistic args. Exit 0 expected.
- - **JSON fidelity**: append `--json` to the happy path; pipe through `python3 -c "import sys,json; json.load(sys.stdin)"` to assert valid JSON.
- - **Error path** (when the command takes an arg): one invocation with a deliberately bad arg (invalid ID, malformed date, non-existent URL). Exit non-zero expected.
-4. Render a live progress line at start: `Dogfood matrix: N leaves × 3-4 tests = M tests total. Running...`
-5. Report pass/fail per test, accumulate to a final tally, and write `$PROOFS_DIR/<stamp>-dogfood-results.md`.
-
-**Critical: pipe-free exit-code checks.** A shell command like `"$BIN" foo | tail -2` captures `tail`'s exit code, not the binary's. Always run as:
+**Full dogfood is not a judgment call about "enough."** Run the Printing
+Press-owned live matrix so command enumeration, exit-code capture, JSON parsing,
+and acceptance-marker writing are deterministic:
```bash
-DOGFOOD_TMP_DIR="/tmp/printing-press/dogfood"
-mkdir -p "$DOGFOOD_TMP_DIR"
-OUT_FILE="$(mktemp "$DOGFOOD_TMP_DIR/<api>-out-XXXXXX")"
-"$BIN" <subcmd> <args> > "$OUT_FILE" 2>&1
-code=$?
-# then check $code directly
-rm -f "$OUT_FILE"
+printing-press dogfood --live \
+ --dir "$CLI_WORK_DIR" \
+ --level full \
+ --json \
+ --write-acceptance "$PROOFS_DIR/phase5-acceptance.json"
```
-Never use `"$BIN" ... && echo ok || echo fail` for exit-code testing — short-circuit and unpredictable piping masks real failures.
+Use `--level quick` only when the user selected Quick Check in Step 1.
+
+The live dogfood runner enumerates the CLI's `agent-context` command tree,
+runs help, happy-path, JSON-fidelity, and error-path checks where applicable,
+captures subprocess exit codes directly without shell pipes, and emits a
+structured report with pass/fail/skipped counts. Save the JSON report to:
+
+`$PROOFS_DIR/<stamp>-dogfood-results.json`
+
+If the command exits non-zero, inspect the structured failures, fix the CLI, and
+rerun live dogfood. Do not hand-edit `phase5-acceptance.json`; it must come from
+the runner.
**Quick check (auto-selected test subset):**
1. `doctor` — auth valid, API reachable.
@@ -2371,8 +2370,6 @@ Never use `"$BIN" ... && echo ok || echo fail` for exit-code testing — short-c
- For every command that supports `--json`, one JSON parse validation.
- For write-side commands (when API key + user consent): create test entity with obviously-test data, verify in subsequent list/get, test one mutation, verify change.
-**Binary support:** future versions of `printing-press dogfood --live` will run this matrix as a single command — see issue #198. Until that ships, the agent must construct the matrix manually from `--help` and run it.
-
### Step 3: Fix issues inline
When a test fails, fix it immediately — do not accumulate failures. Tag each fix:
← 7a143818 fix(cli): gate publishing on Phase 5 proof (#558)
·
back to Cli Printing Press
·
fix(cli): separate sampled probes from live verification (#5 b39a23d0 →