← back to Cli Printing Press
feat(cli): scorecard --live-check samples novel-feature examples against real targets — closes #200 (#210)
df242da414371f480d394435ad933b9d77b26aec · 2026-04-13 13:56:25 -0700 · Trevin Chow
Files touched
M internal/cli/scorecard.goA internal/pipeline/live_check.goA internal/pipeline/live_check_test.go
Diff
commit df242da414371f480d394435ad933b9d77b26aec
Author: Trevin Chow <trevin@trevinchow.com>
Date: Mon Apr 13 13:56:25 2026 -0700
feat(cli): scorecard --live-check samples novel-feature examples against real targets — closes #200 (#210)
---
internal/cli/scorecard.go | 42 ++-
internal/pipeline/live_check.go | 491 +++++++++++++++++++++++++++++++++++
internal/pipeline/live_check_test.go | 355 +++++++++++++++++++++++++
3 files changed, 887 insertions(+), 1 deletion(-)
diff --git a/internal/cli/scorecard.go b/internal/cli/scorecard.go
index ab64a3f0..7f087da0 100644
--- a/internal/cli/scorecard.go
+++ b/internal/cli/scorecard.go
@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"strings"
+ "time"
"github.com/mvanhorn/cli-printing-press/internal/pipeline"
"github.com/spf13/cobra"
@@ -14,6 +15,8 @@ func newScorecardCmd() *cobra.Command {
var dir string
var specPath string
var asJSON bool
+ var liveCheck bool
+ var liveCheckTimeout time.Duration
cmd := &cobra.Command{
Use: "scorecard",
@@ -21,6 +24,9 @@ func newScorecardCmd() *cobra.Command {
Example: ` # Score a generated CLI directory
printing-press scorecard --dir ./generated/stripe-pp-cli
+ # Include a live behavioral sample (runs novel-feature examples against real targets)
+ printing-press scorecard --dir ./generated/stripe-pp-cli --live-check
+
# Output as JSON
printing-press scorecard --dir ./generated/stripe-pp-cli --json`,
RunE: func(cmd *cobra.Command, args []string) error {
@@ -40,10 +46,25 @@ func newScorecardCmd() *cobra.Command {
return &ExitError{Code: ExitGenerationError, Err: fmt.Errorf("running scorecard: %w", err)}
}
+ var live *pipeline.LiveCheckResult
+ if liveCheck {
+ live = pipeline.RunLiveCheck(pipeline.LiveCheckOptions{
+ CLIDir: dir,
+ Timeout: liveCheckTimeout,
+ })
+ if insightCap := pipeline.InsightCapFromLiveCheck(live); insightCap != nil && sc.Steinberger.Insight > *insightCap {
+ sc.Steinberger.Insight = *insightCap
+ }
+ }
+
if asJSON {
+ payload := map[string]any{"scorecard": sc}
+ if live != nil {
+ payload["live_check"] = live
+ }
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
- return enc.Encode(sc)
+ return enc.Encode(payload)
}
// Human-readable output
@@ -80,6 +101,23 @@ func newScorecardCmd() *cobra.Command {
fmt.Printf(" Note: omitted from denominator: %s\n", strings.Join(sc.UnscoredDimensions, ", "))
}
+ if live != nil {
+ fmt.Printf("\nLive Check (behavioral sample)\n")
+ if live.Unable {
+ fmt.Printf(" Unable to run: %s\n", live.Reason)
+ } else {
+ fmt.Printf(" Passed: %d/%d (%d%% pass rate)\n", live.Passed, live.Checked(), int(live.PassRate*100+0.5))
+ if live.Failed > 0 {
+ fmt.Println(" Failures:")
+ for _, f := range live.Features {
+ if f.Status == "fail" {
+ fmt.Printf(" - %s: %s\n", f.Name, f.Reason)
+ }
+ }
+ }
+ }
+ }
+
if len(sc.GapReport) > 0 {
fmt.Printf("\nGaps:\n")
for _, g := range sc.GapReport {
@@ -94,6 +132,8 @@ func newScorecardCmd() *cobra.Command {
cmd.Flags().StringVar(&dir, "dir", "", "Path to generated CLI directory")
cmd.Flags().StringVar(&specPath, "spec", "", "Path to OpenAPI spec JSON for semantic validation")
cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON")
+ cmd.Flags().BoolVar(&liveCheck, "live-check", false, "Sample novel-feature examples against real targets and cap Insight when flagships return broken output")
+ cmd.Flags().DurationVar(&liveCheckTimeout, "live-check-timeout", 10*time.Second, "Per-feature timeout for live check invocations")
return cmd
}
diff --git a/internal/pipeline/live_check.go b/internal/pipeline/live_check.go
new file mode 100644
index 00000000..6319297d
--- /dev/null
+++ b/internal/pipeline/live_check.go
@@ -0,0 +1,491 @@
+package pipeline
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+)
+
+// LiveStatus is the outcome of one feature's live check.
+type LiveStatus string
+
+const (
+ StatusPass LiveStatus = "pass"
+ StatusFail LiveStatus = "fail"
+ StatusSkip LiveStatus = "skip"
+)
+
+// Default bounds for RunLiveCheck. Exported so callers can override via
+// LiveCheckOptions without hard-coding magic numbers.
+const (
+ DefaultLiveCheckTimeout = 10 * time.Second
+ DefaultLiveCheckConcurrency = 4
+ // MaxOutputBytes caps the stdout captured from each feature invocation.
+ // Relevance matching only needs a few hundred bytes; a 1 MiB cap keeps a
+ // misbehaving feature from exhausting the scorecard process's memory.
+ MaxOutputBytes = 1 << 20
+)
+
+// LiveCheckResult summarizes a live-behavior sampling of a printed CLI's
+// novel features. For each novel feature with an Example command, the check
+// runs the command against real targets (the CLI's actual configured API or
+// sites) and asserts the output has the shape a working feature would produce:
+// non-empty, query-relevant when the command encodes a query.
+//
+// Produced by RunLiveCheck. Consumed by the scorecard to apply a behavioral
+// correctness cap on the Insight dimension — a Grade A scorecard with a
+// flagship feature returning wrong data shouldn't be possible.
+type LiveCheckResult struct {
+ Passed int `json:"passed"`
+ Failed int `json:"failed"`
+ Skipped int `json:"skipped"`
+ PassRate float64 `json:"-"` // exposed via pass_rate_pct in MarshalJSON
+ Features []LiveFeatureResult `json:"features"`
+ Unable bool `json:"unable,omitempty"`
+ Reason string `json:"reason,omitempty"`
+ RanAt time.Time `json:"ran_at"`
+}
+
+// Checked returns the total number of features that were sampled.
+// Derived; not persisted to avoid the three-counters-for-three-states
+// redundancy that makes invariants easy to drift.
+func (r *LiveCheckResult) Checked() int {
+ if r == nil {
+ return 0
+ }
+ return r.Passed + r.Failed + r.Skipped
+}
+
+// LiveFeatureResult is one feature's outcome.
+type LiveFeatureResult struct {
+ Name string `json:"name"`
+ Command string `json:"command"`
+ Example string `json:"example"`
+ Status LiveStatus `json:"status"`
+ Reason string `json:"reason,omitempty"`
+}
+
+// LiveCheckOptions bundles the optional knobs for RunLiveCheck. CLIDir is
+// required; every other field has a sensible zero-value default.
+type LiveCheckOptions struct {
+ // CLIDir is the printed CLI's root (containing research.json and the
+ // built binary).
+ CLIDir string
+ // BinaryName, when non-empty, names the executable to run. Leave blank
+ // to let RunLiveCheck derive it from CLIDir (tries `<base>-pp-cli`,
+ // falls back to `<base>`).
+ BinaryName string
+ // Timeout bounds each feature invocation. Zero uses DefaultLiveCheckTimeout.
+ Timeout time.Duration
+ // Concurrency sets the parallel-feature worker count. Zero uses
+ // DefaultLiveCheckConcurrency. Set to 1 to force serial execution.
+ Concurrency int
+}
+
+// RunLiveCheck samples each novel feature's Example command against the real
+// CLI. Returns an Unable=true result (not an error) when research.json or the
+// binary is missing — the scorecard treats those as "could not run" rather
+// than failure, so an absent check doesn't penalize the CLI.
+func RunLiveCheck(opts LiveCheckOptions) *LiveCheckResult {
+ out := &LiveCheckResult{RanAt: time.Now().UTC()}
+
+ if opts.CLIDir == "" {
+ out.Unable = true
+ out.Reason = "CLIDir is required"
+ return out
+ }
+
+ research, err := LoadResearch(opts.CLIDir)
+ if err != nil {
+ out.Unable = true
+ out.Reason = "no research.json: " + err.Error()
+ return out
+ }
+
+ features := pickFeatures(research)
+ if len(features) == 0 {
+ out.Unable = true
+ out.Reason = "no novel features with Example commands to sample"
+ return out
+ }
+
+ binaryPath, binErr := resolveBinaryPath(opts.CLIDir, opts.BinaryName)
+ if binErr != nil {
+ out.Unable = true
+ out.Reason = binErr.Error()
+ return out
+ }
+
+ timeout := opts.Timeout
+ if timeout <= 0 {
+ timeout = DefaultLiveCheckTimeout
+ }
+ concurrency := opts.Concurrency
+ if concurrency <= 0 {
+ concurrency = DefaultLiveCheckConcurrency
+ }
+ if concurrency > len(features) {
+ concurrency = len(features)
+ }
+
+ results := runFeaturesConcurrent(opts.CLIDir, binaryPath, features, timeout, concurrency)
+ out.Features = results
+ for _, r := range results {
+ switch r.Status {
+ case StatusPass:
+ out.Passed++
+ case StatusFail:
+ out.Failed++
+ default:
+ out.Skipped++
+ }
+ }
+ if total := out.Checked(); total > 0 {
+ out.PassRate = float64(out.Passed) / float64(total)
+ }
+ return out
+}
+
+// resolveBinaryPath returns the absolute path to the CLI binary. When name
+// is non-empty it's used verbatim; otherwise RunLiveCheck tries the common
+// `<base>-pp-cli` naming convention and falls back to `<base>`.
+func resolveBinaryPath(cliDir, name string) (string, error) {
+ candidates := []string{name}
+ if name == "" {
+ base := filepath.Base(cliDir)
+ candidates = []string{base + "-pp-cli", base}
+ }
+ for _, candidate := range candidates {
+ if candidate == "" {
+ continue
+ }
+ path := filepath.Join(cliDir, candidate)
+ info, err := os.Stat(path)
+ if err != nil {
+ continue
+ }
+ if info.Mode()&0o111 == 0 {
+ return "", fmt.Errorf("binary %q is not executable", path)
+ }
+ return path, nil
+ }
+ return "", fmt.Errorf("no runnable binary found in %q (tried %v)", cliDir, candidates)
+}
+
+// runFeaturesConcurrent distributes the per-feature checks across a worker
+// pool. Results are collected in-order so LiveCheckResult.Features stays
+// stable across runs.
+func runFeaturesConcurrent(cliDir, binaryPath string, features []NovelFeature, timeout time.Duration, concurrency int) []LiveFeatureResult {
+ results := make([]LiveFeatureResult, len(features))
+ type job struct{ idx int }
+ jobs := make(chan job, len(features))
+ for i := range features {
+ jobs <- job{idx: i}
+ }
+ close(jobs)
+
+ var wg sync.WaitGroup
+ for w := 0; w < concurrency; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for j := range jobs {
+ results[j.idx] = runOneFeatureCheck(cliDir, binaryPath, features[j.idx], timeout)
+ }
+ }()
+ }
+ wg.Wait()
+ return results
+}
+
+// pickFeatures returns the novel features to sample. Prefers NovelFeaturesBuilt
+// (the verified subset) when dogfood has run; falls back to NovelFeatures.
+// Only features with a non-empty Example are usable — the rest are skipped
+// silently since we have no invocation to run.
+func pickFeatures(r *ResearchResult) []NovelFeature {
+ source := r.NovelFeatures
+ if r.NovelFeaturesBuilt != nil && len(*r.NovelFeaturesBuilt) > 0 {
+ source = *r.NovelFeaturesBuilt
+ }
+ var out []NovelFeature
+ for _, f := range source {
+ if strings.TrimSpace(f.Example) != "" {
+ out = append(out, f)
+ }
+ }
+ return out
+}
+
+// runOneFeatureCheck parses the Example invocation, runs it against the real
+// binary, and evaluates the output shape. The Example is expected to start
+// with the binary name (e.g., "recipe-goat-pp-cli goat \"brownies\" --limit 5");
+// we drop that prefix and replace it with the absolute binary path so the
+// check works regardless of the caller's PATH.
+//
+// Note: runCLIWithOutput in runtime.go uses CombinedOutput (stdout+stderr
+// merged) and wraps non-zero exits as a generic error. This check instead
+// separates stdout (for the relevance pass) from stderr (for failure
+// messaging) and needs structured access to *exec.ExitError +
+// DeadlineExceeded, so it runs exec inline.
+func runOneFeatureCheck(cliDir, binaryPath string, f NovelFeature, timeout time.Duration) LiveFeatureResult {
+ result := LiveFeatureResult{Name: f.Name, Command: f.Command, Example: f.Example}
+ fail := func(reason string) LiveFeatureResult {
+ result.Status = StatusFail
+ result.Reason = reason
+ return result
+ }
+
+ args, err := parseExampleArgs(f.Example)
+ if err != nil {
+ result.Status = StatusSkip
+ result.Reason = "could not parse example: " + err.Error()
+ return result
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ cmd := exec.CommandContext(ctx, binaryPath, args...)
+ cmd.Dir = cliDir
+ // Capture stdout into a bounded buffer. An unbounded `cmd.Output()` call
+ // would let a misbehaving feature exhaust the scorecard's memory.
+ stdoutCap := &bytes.Buffer{}
+ stderrCap := &bytes.Buffer{}
+ cmd.Stdout = &limitedWriter{w: stdoutCap, remaining: MaxOutputBytes}
+ cmd.Stderr = &limitedWriter{w: stderrCap, remaining: MaxOutputBytes}
+ runErr := cmd.Run()
+ if ctx.Err() == context.DeadlineExceeded {
+ return fail(fmt.Sprintf("timed out after %s", timeout))
+ }
+
+ if runErr != nil {
+ var exitErr *exec.ExitError
+ if errors.As(runErr, &exitErr) {
+ return fail(fmt.Sprintf("exit %d: %s", exitErr.ExitCode(), trimOutput(stderrCap.String())))
+ }
+ return fail("run error: " + runErr.Error())
+ }
+ if strings.TrimSpace(stdoutCap.String()) == "" {
+ return fail("empty output")
+ }
+
+ if query := extractQueryToken(args); query != "" {
+ if !outputMentionsQuery(stdoutCap.String(), query) {
+ return fail(fmt.Sprintf("output does not contain any token from query %q", query))
+ }
+ }
+
+ result.Status = StatusPass
+ return result
+}
+
+// limitedWriter caps the bytes forwarded to w at `remaining`; further writes
+// are discarded (but still report as successful, so the subprocess doesn't
+// SIGPIPE). Intentionally tolerant of truncation — the live check only needs
+// enough output to run a relevance match.
+type limitedWriter struct {
+ w io.Writer
+ remaining int
+}
+
+func (lw *limitedWriter) Write(p []byte) (int, error) {
+ if lw.remaining <= 0 {
+ return len(p), nil
+ }
+ n := len(p)
+ if n > lw.remaining {
+ n = lw.remaining
+ }
+ if _, err := lw.w.Write(p[:n]); err != nil {
+ return 0, err
+ }
+ lw.remaining -= n
+ return len(p), nil
+}
+
+// parseExampleArgs takes an Example like:
+//
+// recipe-goat-pp-cli goat "chicken tikka masala" --limit 5
+//
+// and returns the subcommand arguments (everything after the binary name).
+// Respects double-quoted tokens so queries with spaces stay intact.
+func parseExampleArgs(example string) ([]string, error) {
+ tokens, err := shellSplit(example)
+ if err != nil {
+ return nil, err
+ }
+ if len(tokens) < 2 {
+ return nil, fmt.Errorf("example has no subcommand: %q", example)
+ }
+ return tokens[1:], nil
+}
+
+// shellSplit handles double-quoted tokens — enough for the Example formats
+// the absorb phase produces. No shell metacharacter interpolation is done.
+// Single quotes, escaped characters, and backslashes are not recognized;
+// Examples using those will need updating.
+func shellSplit(s string) ([]string, error) {
+ var tokens []string
+ var current strings.Builder
+ inQuote := false
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ switch {
+ case c == '"':
+ inQuote = !inQuote
+ case c == ' ' && !inQuote:
+ if current.Len() > 0 {
+ tokens = append(tokens, current.String())
+ current.Reset()
+ }
+ default:
+ current.WriteByte(c)
+ }
+ }
+ if inQuote {
+ return nil, fmt.Errorf("unclosed quote in %q", s)
+ }
+ if current.Len() > 0 {
+ tokens = append(tokens, current.String())
+ }
+ return tokens, nil
+}
+
+// extractQueryToken returns a positional argument that looks like a human-
+// readable search query — the kind of token a search/filter command would
+// mention in its output. Returns "" when no such argument exists, in which
+// case no relevance check is performed.
+//
+// URLs and IDs are intentionally excluded: the CLI's output for a URL-based
+// command (recipe get <url>) wouldn't contain the URL itself, so matching
+// against it produces false negatives.
+//
+// Examples:
+//
+// ["goat", "brownies", "--limit", "5"] → "brownies"
+// ["sub", "buttermilk"] → "buttermilk"
+// ["recipe", "get", "https://foo/bar"] → "" (URL, skip relevance check)
+// ["cookbook", "list", "--json"] → "" (no query)
+//
+// TODO: commands like `list pending` where "pending" is a status keyword
+// won't have the status in their rendered output, producing a spurious
+// relevance failure. If this starts biting, consider a denylist of common
+// non-content positionals or reading a dedicated "relevance arg" pointer
+// from NovelFeature metadata.
+func extractQueryToken(args []string) string {
+ var positionals []string
+ for _, arg := range args {
+ if strings.HasPrefix(arg, "-") {
+ break
+ }
+ positionals = append(positionals, arg)
+ }
+ if len(positionals) < 2 {
+ return ""
+ }
+ candidate := positionals[len(positionals)-1]
+ if looksLikeURLOrID(candidate) {
+ return ""
+ }
+ if len(candidate) < 3 {
+ return ""
+ }
+ return candidate
+}
+
+// looksLikeURLOrID returns true for tokens that shouldn't be used as search-
+// relevance queries: URLs, numeric IDs, UUIDs. The CLI output for a
+// get-by-id command won't contain the ID as text.
+func looksLikeURLOrID(s string) bool {
+ if strings.Contains(s, "://") || strings.HasPrefix(s, "/") {
+ return true
+ }
+ allDigits := true
+ for _, c := range s {
+ if c < '0' || c > '9' {
+ allDigits = false
+ break
+ }
+ }
+ return allDigits
+}
+
+// outputMentionsQuery is case-insensitive; splits the query on whitespace
+// and succeeds if any token (with singular/plural tolerance) appears in the
+// output. Mirrors the permissive relevance check used inside generated CLIs.
+func outputMentionsQuery(output, query string) bool {
+ lowered := strings.ToLower(output)
+ for _, tok := range strings.Fields(strings.ToLower(query)) {
+ tok = strings.TrimFunc(tok, func(r rune) bool { return r == '"' || r == '\'' })
+ if len(tok) < 3 {
+ continue
+ }
+ if strings.Contains(lowered, tok) {
+ return true
+ }
+ if strings.HasSuffix(tok, "s") && len(tok) > 3 {
+ if strings.Contains(lowered, tok[:len(tok)-1]) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func trimOutput(s string) string {
+ s = strings.TrimSpace(s)
+ if len(s) > 300 {
+ s = s[:300] + "..."
+ }
+ return s
+}
+
+// InsightCapFromLiveCheck returns the maximum Insight score a CLI should
+// receive given its live-check pass rate. A CLI whose flagships return
+// broken output shouldn't earn a Grade A scorecard.
+//
+// - Unable or zero checked: no cap (nil return)
+// - PassRate >= 0.8: no cap
+// - PassRate >= 0.5: cap at 7
+// - PassRate < 0.5: cap at 4
+func InsightCapFromLiveCheck(r *LiveCheckResult) *int {
+ if r == nil || r.Unable || r.Checked() == 0 {
+ return nil
+ }
+ var cap int
+ switch {
+ case r.PassRate >= 0.8:
+ return nil
+ case r.PassRate >= 0.5:
+ cap = 7
+ default:
+ cap = 4
+ }
+ return &cap
+}
+
+// MarshalJSON emits a rounded pass_rate_pct alongside the raw counters so
+// JSON consumers don't have to deal with floating-point noise. PassRate is
+// hidden via json:"-" on the struct; this method computes the percentage
+// once using an alias to avoid infinite recursion.
+func (r *LiveCheckResult) MarshalJSON() ([]byte, error) {
+ type alias LiveCheckResult
+ return json.Marshal(&struct {
+ *alias
+ Checked int `json:"checked"`
+ PassRatePct int `json:"pass_rate_pct"`
+ }{
+ alias: (*alias)(r),
+ Checked: r.Checked(),
+ PassRatePct: int(r.PassRate*100 + 0.5),
+ })
+}
diff --git a/internal/pipeline/live_check_test.go b/internal/pipeline/live_check_test.go
new file mode 100644
index 00000000..cbb0e9a8
--- /dev/null
+++ b/internal/pipeline/live_check_test.go
@@ -0,0 +1,355 @@
+package pipeline
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// writeStubBinary drops a tiny shell script at cliDir/<name> that echoes a
+// response based on its arguments. Used to simulate the CLI under test.
+// Skips the test on Windows since we shell out via sh -c.
+func writeStubBinary(t *testing.T, cliDir, name, script string) string {
+ t.Helper()
+ if runtime.GOOS == "windows" {
+ t.Skip("shell-script stub not supported on Windows")
+ }
+ path := filepath.Join(cliDir, name)
+ require.NoError(t, os.WriteFile(path, []byte("#!/bin/sh\n"+script+"\n"), 0o755))
+ return path
+}
+
+// writeTestResearchJSON writes a minimal research.json with the given features.
+func writeTestResearchJSON(t *testing.T, cliDir string, features []NovelFeature) {
+ t.Helper()
+ data := map[string]any{
+ "api_name": "live-check-test",
+ "novel_features": features,
+ }
+ body, err := json.Marshal(data)
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(filepath.Join(cliDir, "research.json"), body, 0o644))
+}
+
+// TestLiveCheck_UnableWhenNoResearch verifies the check gracefully reports
+// Unable=true when research.json is missing rather than treating the absent
+// data as failure. A CLI without research.json should not be penalized by
+// the live check.
+func TestLiveCheck_UnableWhenNoResearch(t *testing.T) {
+ dir := t.TempDir()
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "bin", Timeout: time.Second})
+ require.True(t, result.Unable)
+ require.Contains(t, result.Reason, "research.json")
+ require.Zero(t, result.Checked())
+}
+
+// TestLiveCheck_UnableWhenNoExamples verifies the check skips when research
+// exists but no novel feature has an Example command.
+func TestLiveCheck_UnableWhenNoExamples(t *testing.T) {
+ dir := t.TempDir()
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Feature A", Command: "foo", Description: "no example"},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "bin", Timeout: time.Second})
+ require.True(t, result.Unable)
+ require.Contains(t, result.Reason, "Example")
+}
+
+// TestLiveCheck_UnableWhenNoBinary verifies the check reports Unable when the
+// binary doesn't exist — distinguishing "CLI wasn't built" from "CLI flunked".
+func TestLiveCheck_UnableWhenNoBinary(t *testing.T) {
+ dir := t.TempDir()
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "A", Command: "a", Example: "bin a --flag"},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "missing-binary", Timeout: time.Second})
+ require.True(t, result.Unable)
+ require.Contains(t, result.Reason, "binary")
+}
+
+// TestLiveCheck_PassOnHappyPath verifies a feature that exits 0 with output
+// matching the query word passes.
+func TestLiveCheck_PassOnHappyPath(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo "Found 3 brownie recipes matching your query"`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Best ranker", Command: "goat", Example: `stub goat "brownies" --limit 5`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.False(t, result.Unable, "result was Unable: %s", result.Reason)
+ require.Equal(t, 1, result.Checked())
+ require.Equal(t, 1, result.Passed)
+ require.Zero(t, result.Failed)
+ require.Equal(t, 1.0, result.PassRate)
+}
+
+// TestLiveCheck_FailOnIrrelevantOutput verifies the relevance check catches
+// the Recipe GOAT pattern: command runs successfully but returns results that
+// don't match the query (e.g., "brownies" → Texas Chili).
+func TestLiveCheck_FailOnIrrelevantOutput(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo "Found 5 Texas Chili recipes ranked by rating"`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Best ranker", Command: "goat", Example: `stub goat "brownies" --limit 5`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.False(t, result.Unable)
+ require.Equal(t, 1, result.Failed, "expected irrelevant output to fail")
+ require.Equal(t, 0.0, result.PassRate)
+ require.Contains(t, result.Features[0].Reason, "query")
+}
+
+// TestLiveCheck_FailOnExitError verifies a command that exits non-zero is
+// recorded as fail, not skip.
+func TestLiveCheck_FailOnExitError(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo "something went wrong" >&2; exit 5`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Broken", Command: "b", Example: `stub b --flag`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.Equal(t, 1, result.Failed)
+ require.Contains(t, result.Features[0].Reason, "exit 5")
+}
+
+// TestLiveCheck_FailOnEmptyOutput ensures stdout must be non-empty.
+func TestLiveCheck_FailOnEmptyOutput(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `exit 0`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Quiet", Command: "q", Example: `stub q`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.Equal(t, 1, result.Failed)
+ require.Contains(t, result.Features[0].Reason, "empty output")
+}
+
+// TestLiveCheck_PrefersBuiltFeatures verifies the check samples the verified
+// `novel_features_built` list (dogfood-validated) over the aspirational
+// `novel_features` list when both are present.
+func TestLiveCheck_PrefersBuiltFeatures(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `echo "matched built-feature output"`)
+ built := []NovelFeature{
+ {Name: "Built", Command: "b", Example: `stub b "built-feature" --flag`},
+ }
+ data := map[string]any{
+ "api_name": "live-check-test",
+ "novel_features": []NovelFeature{{Name: "Planned", Example: `stub p "planned" --flag`}},
+ "novel_features_built": built,
+ }
+ body, err := json.Marshal(data)
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "research.json"), body, 0o644))
+
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.Equal(t, 1, result.Checked())
+ require.Equal(t, "Built", result.Features[0].Name,
+ "should use novel_features_built when present")
+}
+
+// TestInsightCap verifies the pass-rate → cap mapping, which is the scorecard
+// integration contract.
+func TestInsightCap(t *testing.T) {
+ cases := []struct {
+ name string
+ input *LiveCheckResult
+ wantNil bool
+ wantCap int
+ }{
+ {"nil", nil, true, 0},
+ {"unable", &LiveCheckResult{Unable: true}, true, 0},
+ {"zero checked", &LiveCheckResult{}, true, 0},
+ {"100% pass", &LiveCheckResult{Passed: 5, PassRate: 1.0}, true, 0},
+ {"80% pass", &LiveCheckResult{Passed: 8, Failed: 2, PassRate: 0.8}, true, 0},
+ {"50% pass", &LiveCheckResult{Passed: 5, Failed: 5, PassRate: 0.5}, false, 7},
+ {"30% pass", &LiveCheckResult{Passed: 3, Failed: 7, PassRate: 0.3}, false, 4},
+ {"0% pass", &LiveCheckResult{Failed: 5, PassRate: 0.0}, false, 4},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := InsightCapFromLiveCheck(tc.input)
+ if tc.wantNil {
+ require.Nil(t, got)
+ } else {
+ require.NotNil(t, got)
+ require.Equal(t, tc.wantCap, *got)
+ }
+ })
+ }
+}
+
+// TestShellSplit covers the quoted-query parsing used for Example commands.
+func TestShellSplit(t *testing.T) {
+ cases := []struct {
+ in string
+ want []string
+ }{
+ {`cli goat brownies`, []string{"cli", "goat", "brownies"}},
+ {`cli goat "chicken tikka masala" --limit 5`, []string{"cli", "goat", "chicken tikka masala", "--limit", "5"}},
+ {`cli multiple spaces`, []string{"cli", "multiple", "spaces"}},
+ }
+ for _, tc := range cases {
+ got, err := shellSplit(tc.in)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got, "input=%q", tc.in)
+ }
+ _, err := shellSplit(`cli "unclosed`)
+ require.Error(t, err)
+}
+
+// TestExtractQueryToken covers the query detection used for relevance checks.
+// The extractor is deliberately simple: it returns the last positional
+// argument before the first flag, after excluding URLs and numeric IDs
+// (which won't appear as text in the CLI output). For multi-word command
+// paths like `cookbook list`, the extractor will return the 2nd word and
+// the downstream relevance check will usually succeed vacuously — that's
+// an acceptable cost for a stateless heuristic.
+func TestExtractQueryToken(t *testing.T) {
+ cases := []struct {
+ args []string
+ want string
+ }{
+ {[]string{"goat", "brownies", "--limit", "5"}, "brownies"},
+ {[]string{"sub", "buttermilk"}, "buttermilk"},
+ {[]string{"recipe", "get", "https://example.com/recipe"}, ""},
+ {[]string{"recipe", "open", "42"}, ""},
+ {[]string{"tonight", "--max-time", "30m"}, ""},
+ {[]string{"cookbook"}, ""},
+ }
+ for _, tc := range cases {
+ got := extractQueryToken(tc.args)
+ require.Equal(t, tc.want, got, "args=%v", tc.args)
+ }
+}
+
+// TestOutputMentionsQuery ensures case-insensitive per-token matching.
+func TestOutputMentionsQuery(t *testing.T) {
+ require.True(t, outputMentionsQuery("Found 5 Brownie Recipes", "brownies"))
+ require.True(t, outputMentionsQuery("chicken tikka masala results", "chicken"))
+ require.False(t, outputMentionsQuery("Texas Chili Recipes", "brownies"))
+ // Tokens under 3 chars are ignored (too generic).
+ require.False(t, outputMentionsQuery("irrelevant", "to"))
+}
+
+// TestLiveCheckMarshalJSON verifies the custom marshaller emits pass_rate_pct.
+func TestLiveCheckMarshalJSON(t *testing.T) {
+ r := &LiveCheckResult{Passed: 2, PassRate: 2.0 / 3.0}
+ body, err := json.Marshal(r)
+ require.NoError(t, err)
+ require.Contains(t, string(body), `"pass_rate_pct":67`)
+ require.NotContains(t, string(body), "0.6666")
+}
+
+// smoke test that ties research, a stub binary, and the full RunLiveCheck
+// path together.
+func TestLiveCheck_SmokeTest(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `
+case "$1" in
+ goat) echo "Best brownie recipes: 1. Classic Brownies 2. Fudgy Brownies";;
+ sub) echo "Substitutions for buttermilk: milk + lemon juice";;
+ *) echo "unknown command $1" >&2; exit 2;;
+esac
+`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Ranker", Command: "goat", Example: `stub goat "brownies" --limit 5`},
+ {Name: "Subs", Command: "sub", Example: `stub sub buttermilk`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second})
+ require.Equal(t, 2, result.Checked())
+ require.Equal(t, 2, result.Passed)
+ require.Equal(t, 1.0, result.PassRate)
+ // Ensure pass_rate_pct marshals cleanly.
+ body, err := json.Marshal(result)
+ require.NoError(t, err)
+ require.True(t, strings.Contains(string(body), `"pass_rate_pct":100`))
+}
+
+// TestLiveCheck_ConcurrentExecutionPreservesOrder ensures the worker pool
+// produces Features in the input order (not the order workers finish). A
+// slow-first feature would otherwise land at the end of the results slice.
+func TestLiveCheck_ConcurrentExecutionPreservesOrder(t *testing.T) {
+ dir := t.TempDir()
+ // Each invocation sleeps inversely proportional to the argument so the
+ // first feature is the slowest — if ordering leaked through the pool,
+ // results would come back reversed.
+ writeStubBinary(t, dir, "stub", `
+case "$2" in
+ aaaa) sleep 0.15; echo "AAAA matched aaaa";;
+ bbbb) sleep 0.05; echo "BBBB matched bbbb";;
+ cccc) sleep 0.01; echo "CCCC matched cccc";;
+esac
+`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "First", Command: "c", Example: `stub c aaaa`},
+ {Name: "Second", Command: "c", Example: `stub c bbbb`},
+ {Name: "Third", Command: "c", Example: `stub c cccc`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{
+ CLIDir: dir, BinaryName: "stub", Timeout: 5 * time.Second, Concurrency: 3,
+ })
+ require.Equal(t, 3, result.Checked())
+ require.Equal(t, "First", result.Features[0].Name)
+ require.Equal(t, "Second", result.Features[1].Name)
+ require.Equal(t, "Third", result.Features[2].Name)
+}
+
+// TestLiveCheck_OutputCap guards against OOM from a runaway feature that
+// streams megabytes of output. The cap is MaxOutputBytes (1 MiB); the test
+// writes 2 MiB so the limitedWriter has to truncate without blowing up the
+// process. The Example has only one positional so no relevance check fires
+// against the (mostly 'x') output.
+func TestLiveCheck_OutputCap(t *testing.T) {
+ dir := t.TempDir()
+ writeStubBinary(t, dir, "stub", `head -c 2097152 /dev/zero | tr '\0' 'x'`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "Noisy", Command: "n", Example: `stub n`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, BinaryName: "stub", Timeout: 10 * time.Second})
+ require.Equal(t, 1, result.Passed, "run should complete despite bounded output")
+}
+
+// TestLiveCheck_BinaryAutoDerivation verifies RunLiveCheck finds the binary
+// when BinaryName is empty by trying <base>-pp-cli then <base>.
+func TestLiveCheck_BinaryAutoDerivation(t *testing.T) {
+ dir := t.TempDir()
+ // CLIDir basename is the last path segment. Build a stub named that way
+ // and a stub named `<name>-pp-cli`; the latter should be preferred.
+ base := filepath.Base(dir)
+ writeStubBinary(t, dir, base+"-pp-cli", `echo "matched via -pp-cli"`)
+ writeStubBinary(t, dir, base, `echo "matched via base"`)
+ writeTestResearchJSON(t, dir, []NovelFeature{
+ {Name: "X", Command: "x", Example: `stub x matched`},
+ })
+ result := RunLiveCheck(LiveCheckOptions{CLIDir: dir, Timeout: 5 * time.Second})
+ require.False(t, result.Unable, "should have found a binary: %s", result.Reason)
+ require.Equal(t, 1, result.Passed)
+ require.Contains(t, result.Features[0].Example, "stub x matched")
+}
+
+// TestChecked_DerivedFromCounters ensures the Checked() method is a pure
+// derivation — if it ever drifts from Passed+Failed+Skipped the live-check
+// invariant is broken.
+func TestChecked_DerivedFromCounters(t *testing.T) {
+ cases := []struct {
+ r LiveCheckResult
+ want int
+ }{
+ {LiveCheckResult{}, 0},
+ {LiveCheckResult{Passed: 3}, 3},
+ {LiveCheckResult{Passed: 1, Failed: 2, Skipped: 3}, 6},
+ }
+ for _, tc := range cases {
+ require.Equal(t, tc.want, tc.r.Checked())
+ }
+ // Also: nil receiver must not panic.
+ var nilRes *LiveCheckResult
+ require.Zero(t, nilRes.Checked())
+}
← caa283ed feat(cli): kind: synthetic spec attribute for multi-source C
·
back to Cli Printing Press
·
feat(cli): auth.optional spec field — doctor INFO not FAIL, 831040d5 →