← back to Cli Printing Press
fix(cli): return failure exit from verify json (#1693)
5261ae735caee1b3ccf541d400636491c9a6730a · 2026-05-19 16:09:14 -0700 · Trevin Chow
* fix(cli): return failure exit from verify json
* fix(cli): preserve verify text failure exit
* test(cli): assert verify json failure exit
* fix(cli): silence verify json failures at root
Files touched
M internal/cli/verify.goM internal/cli/verify_test.goM testdata/golden/cases/verify-runtime-matrix/command.txt
Diff
commit 5261ae735caee1b3ccf541d400636491c9a6730a
Author: Trevin Chow <trevin@trevinchow.com>
Date: Tue May 19 16:09:14 2026 -0700
fix(cli): return failure exit from verify json (#1693)
* fix(cli): return failure exit from verify json
* fix(cli): preserve verify text failure exit
* test(cli): assert verify json failure exit
* fix(cli): silence verify json failures at root
---
internal/cli/verify.go | 46 ++++++++++-
internal/cli/verify_test.go | 95 ++++++++++++++++++++++
.../golden/cases/verify-runtime-matrix/command.txt | 4 +-
3 files changed, 140 insertions(+), 5 deletions(-)
diff --git a/internal/cli/verify.go b/internal/cli/verify.go
index 861ab769..24af66f9 100644
--- a/internal/cli/verify.go
+++ b/internal/cli/verify.go
@@ -12,6 +12,29 @@ import (
)
func newVerifyCmd() *cobra.Command {
+ return newVerifyCmdWithOptions(verifyCmdOptions{
+ runVerify: pipeline.RunVerify,
+ runFixLoop: pipeline.RunFixLoop,
+ })
+}
+
+type verifyCmdOptions struct {
+ runVerify func(pipeline.VerifyConfig) (*pipeline.VerifyReport, error)
+ runFixLoop func(pipeline.VerifyConfig, *pipeline.VerifyReport, int) (*pipeline.FixLoopReport, error)
+ exitProcess func(int)
+}
+
+func newVerifyCmdWithOptions(opts verifyCmdOptions) *cobra.Command {
+ if opts.runVerify == nil {
+ opts.runVerify = pipeline.RunVerify
+ }
+ if opts.runFixLoop == nil {
+ opts.runFixLoop = pipeline.RunFixLoop
+ }
+ if opts.exitProcess == nil {
+ opts.exitProcess = os.Exit
+ }
+
var dir string
var specPath string
var apiKey string
@@ -61,7 +84,7 @@ Use --fix to auto-patch common failures and re-test (max 3 iterations).`,
NoSpec: noSpec,
}
- report, err := pipeline.RunVerify(cfg)
+ report, err := opts.runVerify(cfg)
if err != nil {
return fmt.Errorf("running verify: %w", err)
}
@@ -71,7 +94,7 @@ Use --fix to auto-patch common failures and re-test (max 3 iterations).`,
if fix && shouldRunFixLoop(report) {
fmt.Printf("\nVerification verdict %s (pass rate %.0f%%, threshold %d%%). Running fix loop (max %d iterations)...\n\n",
report.Verdict, report.PassRate, threshold, maxIterations)
- fixReport, err = pipeline.RunFixLoop(cfg, report, maxIterations)
+ fixReport, err = opts.runFixLoop(cfg, report, maxIterations)
if err != nil {
fmt.Fprintf(os.Stderr, "Fix loop error: %v\n", err)
} else if fixReport.FinalReport != nil {
@@ -90,7 +113,11 @@ Use --fix to auto-patch common failures and re-test (max 3 iterations).`,
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
- return enc.Encode(output)
+ if err := enc.Encode(output); err != nil {
+ return err
+ }
+ cmd.Root().SilenceErrors = true
+ return verifyVerdictError(report, true)
}
printVerifyReport(report)
@@ -104,7 +131,7 @@ Use --fix to auto-patch common failures and re-test (max 3 iterations).`,
}
if report.Verdict == "FAIL" {
- os.Exit(1)
+ opts.exitProcess(1)
}
return nil
},
@@ -124,6 +151,17 @@ Use --fix to auto-patch common failures and re-test (max 3 iterations).`,
return cmd
}
+func verifyVerdictError(report *pipeline.VerifyReport, silent bool) error {
+ if report != nil && report.Verdict == "FAIL" {
+ return &ExitError{
+ Code: ExitGenerationError,
+ Err: fmt.Errorf("verification failed: verdict %s", report.Verdict),
+ Silent: silent,
+ }
+ }
+ return nil
+}
+
func shouldRunFixLoop(report *pipeline.VerifyReport) bool {
if report == nil {
return false
diff --git a/internal/cli/verify_test.go b/internal/cli/verify_test.go
index ba477863..e50bbc5b 100644
--- a/internal/cli/verify_test.go
+++ b/internal/cli/verify_test.go
@@ -1,10 +1,15 @@
package cli
import (
+ "bytes"
+ "encoding/json"
+ "errors"
"os"
"path/filepath"
"testing"
+ "github.com/mvanhorn/cli-printing-press/v4/internal/pipeline"
+ "github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -37,3 +42,93 @@ func TestCleanupVerifyArtifacts_NoOpWhenDisabled(t *testing.T) {
assert.FileExists(t, filepath.Join(dir, "sample-cli"))
}
+
+func TestVerifyCmdJSONFailReturnsExitErrorAfterWritingReport(t *testing.T) {
+ cmd := newVerifyCmdWithOptions(verifyCmdOptions{
+ runVerify: func(cfg pipeline.VerifyConfig) (*pipeline.VerifyReport, error) {
+ return &pipeline.VerifyReport{
+ Mode: "mock",
+ Total: 1,
+ Failed: 1,
+ PassRate: 0,
+ Verdict: "FAIL",
+ Binary: filepath.Join(cfg.Dir, "sample-cli"),
+ }, nil
+ },
+ })
+ cmd.SetArgs([]string{"--dir", t.TempDir(), "--json"})
+
+ output, err := runWithCapturedStdout(t, cmd.Execute)
+ require.Error(t, err)
+
+ var exitErr *ExitError
+ require.True(t, errors.As(err, &exitErr))
+ assert.Equal(t, ExitGenerationError, exitErr.Code)
+ assert.True(t, exitErr.Silent)
+
+ var payload struct {
+ Verify pipeline.VerifyReport `json:"verify"`
+ }
+ require.NoError(t, json.Unmarshal([]byte(output), &payload))
+ assert.Equal(t, "FAIL", payload.Verify.Verdict)
+}
+
+func TestVerifyCmdJSONFailSilencesRootCobraError(t *testing.T) {
+ verifyCmd := newVerifyCmdWithOptions(verifyCmdOptions{
+ runVerify: func(cfg pipeline.VerifyConfig) (*pipeline.VerifyReport, error) {
+ return &pipeline.VerifyReport{
+ Mode: "mock",
+ Total: 1,
+ Failed: 1,
+ PassRate: 0,
+ Verdict: "FAIL",
+ Binary: filepath.Join(cfg.Dir, "sample-cli"),
+ }, nil
+ },
+ })
+ root := &cobra.Command{Use: "printing-press", SilenceUsage: true}
+ var stderr bytes.Buffer
+ root.SetErr(&stderr)
+ root.AddCommand(verifyCmd)
+ root.SetArgs([]string{"verify", "--dir", t.TempDir(), "--json"})
+
+ output, err := runWithCapturedStdout(t, root.Execute)
+ require.Error(t, err)
+ assert.Empty(t, stderr.String())
+
+ var exitErr *ExitError
+ require.True(t, errors.As(err, &exitErr))
+ assert.True(t, exitErr.Silent)
+
+ var payload struct {
+ Verify pipeline.VerifyReport `json:"verify"`
+ }
+ require.NoError(t, json.Unmarshal([]byte(output), &payload))
+ assert.Equal(t, "FAIL", payload.Verify.Verdict)
+}
+
+func TestVerifyCmdTextFailExitsWithLegacyCode(t *testing.T) {
+ var exitCode *int
+ cmd := newVerifyCmdWithOptions(verifyCmdOptions{
+ runVerify: func(cfg pipeline.VerifyConfig) (*pipeline.VerifyReport, error) {
+ return &pipeline.VerifyReport{
+ Mode: "mock",
+ Total: 1,
+ Failed: 1,
+ PassRate: 0,
+ Verdict: "FAIL",
+ Binary: filepath.Join(cfg.Dir, "sample-cli"),
+ }, nil
+ },
+ exitProcess: func(code int) {
+ exitCode = &code
+ },
+ })
+ cmd.SetArgs([]string{"--dir", t.TempDir()})
+
+ output, err := runWithCapturedStdout(t, cmd.Execute)
+ require.NoError(t, err)
+ require.NotNil(t, exitCode)
+ assert.Equal(t, 1, *exitCode)
+ assert.Contains(t, output, "Verdict: FAIL")
+}
diff --git a/testdata/golden/cases/verify-runtime-matrix/command.txt b/testdata/golden/cases/verify-runtime-matrix/command.txt
index 577a5ddf..93e593dc 100644
--- a/testdata/golden/cases/verify-runtime-matrix/command.txt
+++ b/testdata/golden/cases/verify-runtime-matrix/command.txt
@@ -1,5 +1,7 @@
set -euo pipefail
cp -R testdata/golden/fixtures/verify-runtime-matrix "$CASE_ACTUAL_DIR/fixtures"
"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/structural-pass-pp-cli" --no-spec --json > "$CASE_ACTUAL_DIR/structural-pass.json"
-"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/structural-fail-pp-cli" --no-spec --json > "$CASE_ACTUAL_DIR/structural-fail.json"
+rc=0
+"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/structural-fail-pp-cli" --no-spec --json > "$CASE_ACTUAL_DIR/structural-fail.json" || rc=$?
+test "$rc" -eq 3
"$BINARY" verify --dir "$CASE_ACTUAL_DIR/fixtures/mock-pass-pp-cli" --spec "$CASE_ACTUAL_DIR/fixtures/mock-pass-pp-cli/spec.yaml" --json > "$CASE_ACTUAL_DIR/mock-pass.json"
← 4093ef3e fix(cli): redact shipcheck api key in json (#1673)
·
back to Cli Printing Press
·
feat(cli): GraphQL credential probe in doctor + actionable c b534884f →