← back to Cli Printing Press
fix(dogfood): accept quick live passes with skips (#646)
c7574bd478eb14e1e26d2421a224bb2be8f0a214 · 2026-05-06 10:03:47 -0700 · Dinakar Sarbada
Co-authored-by: Dinakar Sarbada <dinakars777@users.noreply.github.com>
Files touched
M internal/cli/dogfood.goM internal/cli/dogfood_test.goM internal/pipeline/live_dogfood.goM internal/pipeline/live_dogfood_test.goM internal/pipeline/phase5_gate.goM internal/pipeline/phase5_gate_test.go
Diff
commit c7574bd478eb14e1e26d2421a224bb2be8f0a214
Author: Dinakar Sarbada <sarbadadinu@gmail.com>
Date: Wed May 6 10:03:47 2026 -0700
fix(dogfood): accept quick live passes with skips (#646)
Co-authored-by: Dinakar Sarbada <dinakars777@users.noreply.github.com>
---
internal/cli/dogfood.go | 12 ++-
internal/cli/dogfood_test.go | 149 +++++++++++++++++++++++++++++++++
internal/pipeline/live_dogfood.go | 13 +--
internal/pipeline/live_dogfood_test.go | 26 +++++-
internal/pipeline/phase5_gate.go | 8 +-
internal/pipeline/phase5_gate_test.go | 54 ++++++++++--
6 files changed, 242 insertions(+), 20 deletions(-)
diff --git a/internal/cli/dogfood.go b/internal/cli/dogfood.go
index 6154c0bf..0a38de88 100644
--- a/internal/cli/dogfood.go
+++ b/internal/cli/dogfood.go
@@ -100,7 +100,7 @@ func printLiveDogfoodReport(report *pipeline.LiveDogfoodReport) {
fmt.Println("================================")
fmt.Println()
fmt.Printf("Level: %s\n", report.Level)
- fmt.Printf("Verdict: %s\n", report.Verdict)
+ fmt.Printf("Verdict: %s%s\n", report.Verdict, liveDogfoodVerdictQualifier(report))
fmt.Printf("Commands: %d\n", len(report.Commands))
fmt.Printf("Tests: %d passed, %d failed, %d skipped\n", report.Passed, report.Failed, report.Skipped)
fmt.Println()
@@ -114,6 +114,16 @@ func printLiveDogfoodReport(report *pipeline.LiveDogfoodReport) {
}
}
+func liveDogfoodVerdictQualifier(report *pipeline.LiveDogfoodReport) string {
+ if report == nil || report.Verdict != "PASS" {
+ return ""
+ }
+ if report.Skipped > 0 {
+ return " (with skips)"
+ }
+ return " (all tests run)"
+}
+
func printDogfoodReport(report *pipeline.DogfoodReport) {
name := filepath.Base(report.Dir)
diff --git a/internal/cli/dogfood_test.go b/internal/cli/dogfood_test.go
index d808c4a9..9e316435 100644
--- a/internal/cli/dogfood_test.go
+++ b/internal/cli/dogfood_test.go
@@ -1,6 +1,10 @@
package cli
import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "runtime"
"testing"
"github.com/mvanhorn/cli-printing-press/v3/internal/pipeline"
@@ -38,3 +42,148 @@ func TestDogfoodHelpIncludesLiveFlags(t *testing.T) {
assert.Contains(t, output, "--level")
assert.Contains(t, output, "--write-acceptance")
}
+
+func TestPrintLiveDogfoodReportDistinguishesPassWithSkips(t *testing.T) {
+ clean := captureStdout(t, func() {
+ printLiveDogfoodReport(&pipeline.LiveDogfoodReport{
+ Dir: t.TempDir(),
+ Level: "quick",
+ Verdict: "PASS",
+ Passed: 4,
+ Failed: 0,
+ Skipped: 0,
+ Commands: []string{"widgets list"},
+ })
+ })
+ assert.Contains(t, clean, "Verdict: PASS (all tests run)")
+ assert.NotContains(t, clean, "PASS (with skips)")
+
+ withSkips := captureStdout(t, func() {
+ printLiveDogfoodReport(&pipeline.LiveDogfoodReport{
+ Dir: t.TempDir(),
+ Level: "quick",
+ Verdict: "PASS",
+ Passed: 2,
+ Failed: 0,
+ Skipped: 2,
+ Commands: []string{"widgets list"},
+ })
+ })
+ assert.Contains(t, withSkips, "Verdict: PASS (with skips)")
+ assert.NotContains(t, withSkips, "PASS (all tests run)")
+}
+
+func TestDogfoodLiveQuickPassWithSkipsWritesAcceptance(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir := writeDogfoodQuickSkipFixture(t)
+ markerPath := filepath.Join(t.TempDir(), pipeline.Phase5AcceptanceFilename)
+ cmd := newDogfoodCmd()
+ cmd.SetArgs([]string{
+ "--dir", dir,
+ "--live",
+ "--level", "quick",
+ "--timeout", "2s",
+ "--write-acceptance", markerPath,
+ })
+
+ output, err := runWithCapturedStdout(t, cmd.Execute)
+ require.NoError(t, err)
+ assert.Contains(t, output, "Verdict: PASS (with skips)")
+ assert.Contains(t, output, "Tests: 4 passed, 0 failed, 4 skipped")
+
+ data, err := os.ReadFile(markerPath)
+ require.NoError(t, err)
+ var marker pipeline.Phase5GateMarker
+ require.NoError(t, json.Unmarshal(data, &marker))
+ assert.Equal(t, "pass", marker.Status)
+ assert.Equal(t, "quick", marker.Level)
+ assert.Equal(t, 4, marker.TestsPassed)
+ assert.Equal(t, 4, marker.TestsSkipped)
+ assert.Equal(t, 0, marker.TestsFailed)
+
+ validation := pipeline.ValidatePhase5Gate(filepath.Dir(markerPath), pipeline.CLIManifest{
+ APIName: marker.APIName,
+ RunID: marker.RunID,
+ AuthType: "none",
+ })
+ assert.True(t, validation.Passed, validation.Detail)
+}
+
+func writeDogfoodQuickSkipFixture(t *testing.T) string {
+ t.Helper()
+
+ dir := filepath.Join(t.TempDir(), "fixture")
+ require.NoError(t, os.MkdirAll(dir, 0o755))
+ require.NoError(t, pipeline.WriteCLIManifest(dir, pipeline.CLIManifest{
+ SchemaVersion: 1,
+ APIName: "fixture",
+ CLIName: "fixture-pp-cli",
+ RunID: "run-live-dogfood",
+ AuthType: "none",
+ }))
+
+ binPath := filepath.Join(dir, "fixture-pp-cli")
+ script := `#!/bin/sh
+set -u
+
+if [ "${1:-}" = "agent-context" ]; then
+ cat <<'JSON'
+{
+ "commands": [
+ {"name":"alpha","subcommands":[
+ {"name":"list"}
+ ]},
+ {"name":"widgets","subcommands":[
+ {"name":"list"}
+ ]}
+ ]
+}
+JSON
+ exit 0
+fi
+
+if [ "${1:-}" = "widgets" ] && [ "${2:-}" = "list" ] && [ "${3:-}" = "--help" ]; then
+ cat <<'HELP'
+List widgets.
+
+Usage:
+ fixture-pp-cli widgets list [flags]
+
+Examples:
+ fixture-pp-cli widgets list
+HELP
+ exit 0
+fi
+
+if [ "${1:-}" = "widgets" ] && [ "${2:-}" = "list" ]; then
+ echo 'widgets'
+ exit 0
+fi
+
+if [ "${1:-}" = "alpha" ] && [ "${2:-}" = "list" ] && [ "${3:-}" = "--help" ]; then
+ cat <<'HELP'
+List alpha records.
+
+Usage:
+ fixture-pp-cli alpha list [flags]
+
+Examples:
+ fixture-pp-cli alpha list
+HELP
+ exit 0
+fi
+
+if [ "${1:-}" = "alpha" ] && [ "${2:-}" = "list" ]; then
+ echo 'alpha'
+ exit 0
+fi
+
+echo "unexpected args: $*" >&2
+exit 99
+`
+ require.NoError(t, os.WriteFile(binPath, []byte(script), 0o755))
+ return dir
+}
diff --git a/internal/pipeline/live_dogfood.go b/internal/pipeline/live_dogfood.go
index 028cf24c..a504b300 100644
--- a/internal/pipeline/live_dogfood.go
+++ b/internal/pipeline/live_dogfood.go
@@ -1019,18 +1019,11 @@ func finalizeLiveDogfoodReport(report *LiveDogfoodReport) {
report.Skipped++
}
}
- // Failed-or-empty wins: any real failure or a fully empty matrix is FAIL.
- // Quick runs whose whole slice was explicitly skipped are no-signal, not
- // a failing matrix: the classifier did its job and avoided live mutation.
- // The quick-level PASS arm uses min(5, MatrixSize) for the threshold so
- // single-command quick runs (~4 entries when all probes succeed) can
- // PASS, while keeping a MatrixSize >= 4 floor that blocks pathological
- // matrices where most entries skipped.
+ // Failed-or-empty wins. Skips are non-failures, but quick acceptance still
+ // needs enough counted signal before it can write an acceptance marker.
switch {
- case report.Failed > 0 || (report.MatrixSize == 0 && (report.Level != "quick" || report.Skipped == 0)):
+ case report.Failed > 0 || report.MatrixSize == 0:
report.Verdict = "FAIL"
- case report.Level == "quick" && report.MatrixSize == 0 && report.Skipped > 0:
- report.Verdict = "PASS"
case report.Level == "quick" && report.MatrixSize >= 4 && report.Passed+report.Skipped >= min(5, report.MatrixSize):
report.Verdict = "PASS"
case report.Level == "quick":
diff --git a/internal/pipeline/live_dogfood_test.go b/internal/pipeline/live_dogfood_test.go
index 419dff3c..ab95e016 100644
--- a/internal/pipeline/live_dogfood_test.go
+++ b/internal/pipeline/live_dogfood_test.go
@@ -82,6 +82,28 @@ func TestRunLiveDogfoodWritesAcceptanceMarkerOnPass(t *testing.T) {
assert.True(t, validation.Passed, validation.Detail)
}
+func TestRunLiveDogfoodDoesNotWriteAcceptanceMarkerOnFail(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a shell script as the fake binary; skip on Windows")
+ }
+
+ dir, binaryName := writeLiveDogfoodFixture(t, false)
+ markerPath := filepath.Join(t.TempDir(), Phase5AcceptanceFilename)
+ report, err := RunLiveDogfood(LiveDogfoodOptions{
+ CLIDir: dir,
+ BinaryName: binaryName,
+ Level: "full",
+ Timeout: 2 * time.Second,
+ WriteAcceptancePath: markerPath,
+ })
+ require.NoError(t, err)
+ require.Equal(t, "FAIL", report.Verdict, report.Tests)
+ assert.Greater(t, report.Failed, 0)
+
+ _, statErr := os.Stat(markerPath)
+ assert.True(t, os.IsNotExist(statErr), "failed live dogfood must not write an acceptance marker")
+}
+
func TestRunLiveDogfoodErrorPathAcceptsExpectedNonZeroExit(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a shell script as the fake binary; skip on Windows")
@@ -236,7 +258,7 @@ func TestFinalizeLiveDogfoodReportVerdictGate(t *testing.T) {
want: "PASS",
},
{
- name: "quick 3 pass + 3 skip — MatrixSize floor (3) below 4",
+ name: "quick 3 pass + 3 skip - skips do not satisfy the signal floor",
level: "quick",
results: []LiveDogfoodTestResult{
mkResult(LiveDogfoodStatusPass), mkResult(LiveDogfoodStatusPass),
@@ -268,7 +290,7 @@ func TestFinalizeLiveDogfoodReportVerdictGate(t *testing.T) {
name: "quick all skip — MatrixSize 0",
level: "quick",
results: []LiveDogfoodTestResult{mkResult(LiveDogfoodStatusSkip), mkResult(LiveDogfoodStatusSkip)},
- want: "PASS",
+ want: "FAIL",
},
{
name: "full all skip — MatrixSize 0 still blocks acceptance",
diff --git a/internal/pipeline/phase5_gate.go b/internal/pipeline/phase5_gate.go
index 1a4714fb..e9844fad 100644
--- a/internal/pipeline/phase5_gate.go
+++ b/internal/pipeline/phase5_gate.go
@@ -154,7 +154,13 @@ func phase5AcceptancePassed(marker Phase5GateMarker) (bool, string) {
level := phase5Level(marker)
switch level {
case "quick":
- // Mirror finalizeLiveDogfoodReport's quick PASS condition exactly:
+ if marker.TestsFailed != 0 {
+ return false, fmt.Sprintf("phase5 quick acceptance has %d failed tests", marker.TestsFailed)
+ }
+ if marker.TestsPassed != marker.MatrixSize {
+ return false, fmt.Sprintf("phase5 quick acceptance requires all %d counted tests passed, got %d", marker.MatrixSize, marker.TestsPassed)
+ }
+ // Mirror finalizeLiveDogfoodReport's quick PASS condition:
// MatrixSize >= 4 AND Passed+Skipped >= min(5, MatrixSize). The runner
// is the source of truth; this gate must accept any marker the runner
// would have accepted. Drift here was the original bug (#589/#590).
diff --git a/internal/pipeline/phase5_gate_test.go b/internal/pipeline/phase5_gate_test.go
index cb52d6b7..a7af3c51 100644
--- a/internal/pipeline/phase5_gate_test.go
+++ b/internal/pipeline/phase5_gate_test.go
@@ -39,7 +39,7 @@ func TestValidatePhase5Gate_PassMarker(t *testing.T) {
assert.Equal(t, filepath.Join(proofsDir, Phase5AcceptanceFilename), result.MarkerPath)
}
-func TestValidatePhase5Gate_QuickPassAllowsOneNonBlockingMiss(t *testing.T) {
+func TestValidatePhase5Gate_QuickPassRejectsFailures(t *testing.T) {
proofsDir := t.TempDir()
manifest := CLIManifest{APIName: "test", CLIName: "test-pp-cli", RunID: "run-1", AuthType: "none"}
writePhase5GateMarker(t, proofsDir, Phase5AcceptanceFilename, Phase5GateMarker{
@@ -55,11 +55,11 @@ func TestValidatePhase5Gate_QuickPassAllowsOneNonBlockingMiss(t *testing.T) {
})
result := ValidatePhase5Gate(proofsDir, manifest)
- require.True(t, result.Passed, result.Detail)
- assert.Equal(t, "pass", result.Status)
+ require.False(t, result.Passed)
+ assert.Contains(t, result.Detail, "failed tests")
}
-func TestValidatePhase5Gate_QuickPassRequiresFiveOfSix(t *testing.T) {
+func TestValidatePhase5Gate_QuickRejectsFailuresBeforeThreshold(t *testing.T) {
proofsDir := t.TempDir()
manifest := CLIManifest{APIName: "test", CLIName: "test-pp-cli", RunID: "run-1", AuthType: "none"}
writePhase5GateMarker(t, proofsDir, Phase5AcceptanceFilename, Phase5GateMarker{
@@ -76,7 +76,7 @@ func TestValidatePhase5Gate_QuickPassRequiresFiveOfSix(t *testing.T) {
result := ValidatePhase5Gate(proofsDir, manifest)
require.False(t, result.Passed)
- assert.Contains(t, result.Detail, "5/6")
+ assert.Contains(t, result.Detail, "failed tests")
}
func TestValidatePhase5Gate_QuickPassFiveOfFive(t *testing.T) {
@@ -140,6 +140,48 @@ func TestValidatePhase5Gate_QuickPassCountsSkippedTowardThreshold(t *testing.T)
assert.Equal(t, "pass", result.Status)
}
+func TestValidatePhase5Gate_QuickFailWithSkipsBelowMatrixFloor(t *testing.T) {
+ proofsDir := t.TempDir()
+ manifest := CLIManifest{APIName: "test", CLIName: "test-pp-cli", RunID: "run-1", AuthType: "none"}
+ writePhase5GateMarker(t, proofsDir, Phase5AcceptanceFilename, Phase5GateMarker{
+ SchemaVersion: 1,
+ APIName: "test",
+ RunID: "run-1",
+ Status: "pass",
+ Level: "quick",
+ MatrixSize: 2,
+ TestsPassed: 2,
+ TestsSkipped: 2,
+ TestsFailed: 0,
+ AuthContext: Phase5AuthContext{Type: "none"},
+ })
+
+ result := ValidatePhase5Gate(proofsDir, manifest)
+ require.False(t, result.Passed)
+ assert.Contains(t, result.Detail, "matrix_size >= 4")
+}
+
+func TestValidatePhase5Gate_QuickPassWithSkipsRequiresCountedTestsPassed(t *testing.T) {
+ proofsDir := t.TempDir()
+ manifest := CLIManifest{APIName: "test", CLIName: "test-pp-cli", RunID: "run-1", AuthType: "none"}
+ writePhase5GateMarker(t, proofsDir, Phase5AcceptanceFilename, Phase5GateMarker{
+ SchemaVersion: 1,
+ APIName: "test",
+ RunID: "run-1",
+ Status: "pass",
+ Level: "quick",
+ MatrixSize: 4,
+ TestsPassed: 3,
+ TestsSkipped: 1,
+ TestsFailed: 0,
+ AuthContext: Phase5AuthContext{Type: "none"},
+ })
+
+ result := ValidatePhase5Gate(proofsDir, manifest)
+ require.False(t, result.Passed)
+ assert.Contains(t, result.Detail, "counted tests passed")
+}
+
func TestValidatePhase5Gate_QuickFailMatrixBelowFloor(t *testing.T) {
proofsDir := t.TempDir()
manifest := CLIManifest{APIName: "test", CLIName: "test-pp-cli", RunID: "run-1", AuthType: "none"}
@@ -175,7 +217,7 @@ func TestValidatePhase5Gate_QuickFailBelowThresholdAtMatrixFour(t *testing.T) {
result := ValidatePhase5Gate(proofsDir, manifest)
require.False(t, result.Passed)
- assert.Contains(t, result.Detail, "4/4")
+ assert.Contains(t, result.Detail, "counted tests passed")
}
func TestValidatePhase5Gate_FullPassRejectsFailures(t *testing.T) {
← f486e4d2 chore(main): release 3.10.0 (#616)
·
back to Cli Printing Press
·
feat(cli): Hermes/OpenClaw frontmatter alignment for printed fd7fa6ea →