← back to Cli Printing Press
fix(cli): separate sampled probes from live verification (#560)
b39a23d08620dc871f581742487f9168c468e996 · 2026-05-03 20:58:29 -0700 · Trevin Chow
Files touched
M internal/cli/scorecard.goM internal/cli/scorecard_test.goM internal/cli/shipcheck.goM internal/pipeline/live_check.goM internal/pipeline/live_check_test.goM internal/pipeline/scorecard.goM internal/pipeline/scorecard_live_api_test.go
Diff
commit b39a23d08620dc871f581742487f9168c468e996
Author: Trevin Chow <trevin@trevinchow.com>
Date: Sun May 3 20:58:29 2026 -0700
fix(cli): separate sampled probes from live verification (#560)
---
internal/cli/scorecard.go | 5 +--
internal/cli/scorecard_test.go | 32 ++++++++++++++++
internal/cli/shipcheck.go | 4 +-
internal/pipeline/live_check.go | 31 ++++++++++++++++
internal/pipeline/live_check_test.go | 28 ++++++++++++++
internal/pipeline/scorecard.go | 35 ++++--------------
internal/pipeline/scorecard_live_api_test.go | 55 +++++++++++++---------------
7 files changed, 126 insertions(+), 64 deletions(-)
diff --git a/internal/cli/scorecard.go b/internal/cli/scorecard.go
index dfb9f72c..1a46ef9b 100644
--- a/internal/cli/scorecard.go
+++ b/internal/cli/scorecard.go
@@ -58,9 +58,6 @@ func newScorecardCmd() *cobra.Command {
ResearchDir: researchDir,
Timeout: liveCheckTimeout,
})
- if insightCap := pipeline.InsightCapFromLiveCheck(live); insightCap != nil && sc.Steinberger.Insight > *insightCap {
- sc.Steinberger.Insight = *insightCap
- }
pipeline.ApplyLiveCheckToScorecard(sc, live)
}
@@ -78,7 +75,7 @@ func newScorecardCmd() *cobra.Command {
renderHumanScorecard(os.Stdout, sc)
if live != nil {
- fmt.Printf("\nLive Check (behavioral sample)\n")
+ fmt.Printf("\nSample Output Probe (live command sample)\n")
if live.Unable {
fmt.Printf(" Unable to run: %s\n", live.Reason)
} else {
diff --git a/internal/cli/scorecard_test.go b/internal/cli/scorecard_test.go
index c7c82267..9e121086 100644
--- a/internal/cli/scorecard_test.go
+++ b/internal/cli/scorecard_test.go
@@ -2,6 +2,7 @@ package cli
import (
"bytes"
+ "encoding/json"
"strings"
"testing"
@@ -108,3 +109,34 @@ func TestRenderHumanScorecardShowsScoreForScoredDimensions(t *testing.T) {
assert.False(t, strings.Contains(got, "omitted from denominator"),
"with no unscored dimensions, the composite-note line must not appear")
}
+
+func TestScorecardJSONKeepsLiveCheckSeparateFromLiveVerification(t *testing.T) {
+ t.Parallel()
+ sc := &pipeline.Scorecard{
+ APIName: "test-api",
+ Steinberger: pipeline.SteinerScore{
+ Total: 80,
+ Percentage: 80,
+ },
+ UnscoredDimensions: []string{pipeline.DimLiveAPIVerification},
+ }
+ live := &pipeline.LiveCheckResult{
+ Passed: 1,
+ PassRate: 1.0,
+ Features: []pipeline.LiveFeatureResult{{
+ Name: "sample",
+ Status: pipeline.StatusPass,
+ OutputSample: "Found 3 brownie recipes",
+ }},
+ }
+
+ payload := map[string]any{"scorecard": sc, "live_check": live}
+ data, err := json.Marshal(payload)
+ assert.NoError(t, err)
+ got := string(data)
+
+ assert.Contains(t, got, `"live_check"`)
+ assert.Contains(t, got, `"output_sample":"Found 3 brownie recipes"`)
+ assert.Contains(t, got, `"unscored_dimensions":["live_api_verification"]`)
+ assert.Contains(t, got, `"live_api_verification":0`)
+}
diff --git a/internal/cli/shipcheck.go b/internal/cli/shipcheck.go
index 35af8b8b..21ed5052 100644
--- a/internal/cli/shipcheck.go
+++ b/internal/cli/shipcheck.go
@@ -351,7 +351,7 @@ Legs (in canonical order):
verify — runtime command testing (with --fix to auto-repair common breakage)
workflow-verify — primary workflow end-to-end against the verification manifest
verify-skill — SKILL.md flag/positional/command consistency with the shipped CLI
- scorecard — Steinberger quality bar (with --live-check sampling novel features)
+ scorecard — Steinberger quality bar (with --live-check sampled output probes)
In default mode, every leg streams its full output to the terminal as it runs
and a per-leg verdict table prints at the end. In --json mode, leg output is
@@ -435,7 +435,7 @@ Each leg remains callable standalone — this command is additive orchestration.
cmd.Flags().StringVar(&opts.researchDir, "research-dir", "", "Pipeline directory containing research.json (passed to dogfood and scorecard)")
cmd.Flags().BoolVar(&opts.asJSON, "json", false, "Emit a structured JSON envelope at end-of-run (suppresses per-leg stdout for clean piping; run legs standalone with --json for per-leg detail)")
cmd.Flags().BoolVar(&opts.noFix, "no-fix", false, "Disable verify's --fix auto-repair loop (read-only verify)")
- cmd.Flags().BoolVar(&opts.noLiveCheck, "no-live-check", false, "Disable scorecard's --live-check live-API sampling")
+ cmd.Flags().BoolVar(&opts.noLiveCheck, "no-live-check", false, "Disable scorecard's --live-check sampled output probe")
cmd.Flags().StringVar(&opts.apiKey, "api-key", "", "API key for verify's live testing (read-only GETs only)")
cmd.Flags().StringVar(&opts.envVar, "env-var", "", "Environment variable name verify should read for the API key (e.g., GITHUB_TOKEN)")
cmd.Flags().BoolVar(&opts.strict, "strict", false, "Pass --strict to verify-skill (treat likely-false-positive findings as failures)")
diff --git a/internal/pipeline/live_check.go b/internal/pipeline/live_check.go
index 98e2af54..d2e25ecd 100644
--- a/internal/pipeline/live_check.go
+++ b/internal/pipeline/live_check.go
@@ -364,6 +364,9 @@ func runOneFeatureCheck(cliDir, binaryPath string, f NovelFeature, timeout time.
if !outputMentionsQuery(stdoutCap.String(), query) {
return fail(fmt.Sprintf("output does not contain any token from query %q", query))
}
+ if outputIsBareQueryEcho(stdoutCap.String(), query) {
+ return fail(fmt.Sprintf("output only echoes query %q without result structure", query))
+ }
}
stdout := stdoutCap.String()
@@ -583,6 +586,34 @@ func outputMentionsQuery(output, query string) bool {
return false
}
+func outputIsBareQueryEcho(output, query string) bool {
+ trimmed := strings.TrimSpace(output)
+ if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") {
+ return false
+ }
+ outputWords := normalizedOutputWords(output)
+ queryWords := normalizedOutputWords(query)
+ if len(outputWords) == 0 || len(queryWords) == 0 {
+ return false
+ }
+ if len(outputWords) != len(queryWords) {
+ return false
+ }
+ for i := range outputWords {
+ if outputWords[i] != queryWords[i] {
+ return false
+ }
+ }
+ return true
+}
+
+func normalizedOutputWords(s string) []string {
+ fields := strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
+ return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+ })
+ return fields
+}
+
func trimOutput(s string) string {
s = strings.TrimSpace(s)
if len(s) > 300 {
diff --git a/internal/pipeline/live_check_test.go b/internal/pipeline/live_check_test.go
index dae67d40..ec231353 100644
--- a/internal/pipeline/live_check_test.go
+++ b/internal/pipeline/live_check_test.go
@@ -151,6 +151,34 @@ func TestLiveCheck_PassOnHappyPath(t *testing.T) {
require.Equal(t, 1.0, result.PassRate)
}
+// TestLiveCheck_FailOnTokenEchoOutput guards against commands that satisfy
+// relevance by printing the input token back without returning any result
+// structure. The sampled output probe should feed reviewers, not award
+// behavioral credit for an echo.
+func TestLiveCheck_FailOnTokenEchoOutput(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo "brownies"`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Best ranker", Command: "goat", Example: `stub goat "brownies" --limit 5`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.False(t, result.Unable)
+ require.Equal(t, 1, result.Failed, "expected token-only echo output to fail")
+ require.Contains(t, result.Features[0].Reason, "echo")
+}
+
+func TestLiveCheck_PassOnQueryOnlyJSONOutput(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo '["pikachu","charizard","blastoise"]'`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Pokemon search", Command: "pokemon search", Example: `stub pokemon search "pikachu,charizard,blastoise" --json`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.False(t, result.Unable)
+ require.Equal(t, 1, result.Passed, "structured JSON containing only query values is still a valid result shape")
+ require.Zero(t, result.Failed)
+}
+
// TestLiveCheck_FailOnIrrelevantOutput verifies the relevance check catches
// the Recipe GOAT pattern: command runs successfully but returns results that
// don't match the query (e.g., "brownies" → Texas Chili).
diff --git a/internal/pipeline/scorecard.go b/internal/pipeline/scorecard.go
index c775bd61..40d8b0fb 100644
--- a/internal/pipeline/scorecard.go
+++ b/internal/pipeline/scorecard.go
@@ -888,30 +888,19 @@ func scoreLiveAPIVerification(verifyReport *VerifyReport) (int, bool) {
return score, true
}
-func scoreLiveAPIVerificationFromLiveCheck(live *LiveCheckResult) (int, bool) {
- if live == nil || live.Unable || live.Checked() == 0 {
- return 0, false
- }
- switch {
- case live.Passed >= 3:
- return 10, true
- case live.Passed >= 1:
- return 5, true
- default:
- return 0, true
- }
-}
-
+// ApplyLiveCheckToScorecard lets sampled command output affect only the
+// scorecard dimensions it can honestly support. A weak or failing sample can
+// cap Insight, but it must not populate LiveAPIVerification; that dimension
+// is reserved for VerifyReport evidence from real live verify runs.
func ApplyLiveCheckToScorecard(sc *Scorecard, live *LiveCheckResult) {
if sc == nil {
return
}
- score, scored := scoreLiveAPIVerificationFromLiveCheck(live)
- if !scored {
+ insightCap := InsightCapFromLiveCheck(live)
+ if insightCap == nil || sc.Steinberger.Insight <= *insightCap {
return
}
- sc.Steinberger.LiveAPIVerification = score
- sc.UnscoredDimensions = removeUnscoredDimension(sc.UnscoredDimensions, DimLiveAPIVerification)
+ sc.Steinberger.Insight = *insightCap
recomputeScorecardTotals(sc)
applyScorecardCalibration(sc)
sc.OverallGrade = computeGrade(sc.Steinberger.Percentage)
@@ -938,16 +927,6 @@ func applyScorecardCalibration(sc *Scorecard) {
sc.Steinberger.CalibrationNote = strings.Join(notes, "; ")
}
-func removeUnscoredDimension(dimensions []string, name string) []string {
- out := dimensions[:0]
- for _, dimension := range dimensions {
- if dimension != name {
- out = append(out, dimension)
- }
- }
- return out
-}
-
func recomputeScorecardTotals(sc *Scorecard) {
tier1Raw := sumScorecardDimensions(
sc.Steinberger.OutputModes,
diff --git a/internal/pipeline/scorecard_live_api_test.go b/internal/pipeline/scorecard_live_api_test.go
index 9c4e6345..3c083aeb 100644
--- a/internal/pipeline/scorecard_live_api_test.go
+++ b/internal/pipeline/scorecard_live_api_test.go
@@ -68,31 +68,7 @@ func TestScoreLiveAPIVerification(t *testing.T) {
})
}
-func TestScoreLiveAPIVerificationFromLiveCheck(t *testing.T) {
- tests := []struct {
- name string
- live *LiveCheckResult
- wantScore int
- wantScored bool
- }{
- {name: "nil", live: nil, wantScored: false},
- {name: "unable", live: &LiveCheckResult{Unable: true}, wantScored: false},
- {name: "zero checked", live: &LiveCheckResult{}, wantScored: false},
- {name: "three pass scores ten", live: &LiveCheckResult{Passed: 3, Failed: 2}, wantScore: 10, wantScored: true},
- {name: "two pass scores five", live: &LiveCheckResult{Passed: 2, Failed: 3}, wantScore: 5, wantScored: true},
- {name: "one pass scores five", live: &LiveCheckResult{Passed: 1, Failed: 4}, wantScore: 5, wantScored: true},
- {name: "zero pass scored zero", live: &LiveCheckResult{Failed: 5}, wantScore: 0, wantScored: true},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- score, scored := scoreLiveAPIVerificationFromLiveCheck(tt.live)
- assert.Equal(t, tt.wantScore, score)
- assert.Equal(t, tt.wantScored, scored)
- })
- }
-}
-
-func TestApplyLiveCheckToScorecardCreditsLiveAPIVerification(t *testing.T) {
+func TestApplyLiveCheckToScorecardDoesNotCreditLiveAPIVerification(t *testing.T) {
dir := t.TempDir()
pipelineDir := t.TempDir()
sc, err := RunScorecard(dir, pipelineDir, "", nil)
@@ -100,14 +76,33 @@ func TestApplyLiveCheckToScorecardCreditsLiveAPIVerification(t *testing.T) {
assert.Contains(t, sc.UnscoredDimensions, "live_api_verification")
beforeTotal := sc.Steinberger.Total
- ApplyLiveCheckToScorecard(sc, &LiveCheckResult{Passed: 3, Failed: 2})
+ ApplyLiveCheckToScorecard(sc, &LiveCheckResult{Passed: 3, PassRate: 1.0})
- assert.NotContains(t, sc.UnscoredDimensions, "live_api_verification")
- assert.Equal(t, 10, sc.Steinberger.LiveAPIVerification)
- assert.GreaterOrEqual(t, sc.Steinberger.Total, beforeTotal)
+ assert.Contains(t, sc.UnscoredDimensions, "live_api_verification",
+ "sampled output probes must not masquerade as full live API verification")
+ assert.Equal(t, 0, sc.Steinberger.LiveAPIVerification)
+ assert.Equal(t, beforeTotal, sc.Steinberger.Total,
+ "a passing sampled output probe should not change scorecard totals")
assert.Equal(t, sc.Steinberger.Total, sc.Steinberger.Percentage)
}
+func TestApplyLiveCheckToScorecardCapsInsightOnly(t *testing.T) {
+ dir := t.TempDir()
+ pipelineDir := t.TempDir()
+ sc, err := RunScorecard(dir, pipelineDir, "", nil)
+ assert.NoError(t, err)
+ sc.Steinberger.Insight = 10
+ recomputeScorecardTotals(sc)
+ beforeTotal := sc.Steinberger.Total
+
+ ApplyLiveCheckToScorecard(sc, &LiveCheckResult{Passed: 3, Failed: 7, PassRate: 0.3})
+
+ assert.Equal(t, 4, sc.Steinberger.Insight)
+ assert.Less(t, sc.Steinberger.Total, beforeTotal)
+ assert.Contains(t, sc.UnscoredDimensions, "live_api_verification",
+ "live-check may cap Insight but must leave live_api_verification unscored")
+}
+
func TestApplyLiveCheckToScorecardPreservesBrowserSessionCap(t *testing.T) {
dir := t.TempDir()
pipelineDir := t.TempDir()
@@ -121,7 +116,7 @@ func TestApplyLiveCheckToScorecardPreservesBrowserSessionCap(t *testing.T) {
assert.NoError(t, err)
assert.LessOrEqual(t, sc.Steinberger.Total, 69)
- ApplyLiveCheckToScorecard(sc, &LiveCheckResult{Passed: 3})
+ ApplyLiveCheckToScorecard(sc, &LiveCheckResult{Passed: 3, PassRate: 1.0})
assert.LessOrEqual(t, sc.Steinberger.Total, 69)
assert.Equal(t, sc.Steinberger.Total, sc.Steinberger.Percentage)
← e973d5c5 feat(cli): add live dogfood matrix runner (#559)
·
back to Cli Printing Press
·
fix(cli): support OpenAPI auth metadata overrides (#561) 35fa7afe →