← back to Cli Printing Press
fix(cli): add primary workflow verification to printing press pipeline (#112)
a28d1b4851c0de9d5b07b57edf581e294faaed7e · 2026-04-02 13:23:43 -0700 · Matt Van Horn
* feat(cli): add static wiring checks to dogfood phase
* feat(cli): add workflow verification manifest types and loader
* feat(cli): add workflow verification runner
* feat(cli): integrate workflow verification into shipcheck pipeline
---------
Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Files touched
M internal/cli/root.goA internal/cli/workflow_verify.goM internal/pipeline/dogfood.goM internal/pipeline/dogfood_test.goM internal/pipeline/fullrun.goA internal/pipeline/workflow_manifest.goA internal/pipeline/workflow_manifest_test.goA internal/pipeline/workflow_verify.goA internal/pipeline/workflow_verify_test.goM skills/printing-press/SKILL.md
Diff
commit a28d1b4851c0de9d5b07b57edf581e294faaed7e
Author: Matt Van Horn <mvanhorn@users.noreply.github.com>
Date: Thu Apr 2 13:23:43 2026 -0700
fix(cli): add primary workflow verification to printing press pipeline (#112)
* feat(cli): add static wiring checks to dogfood phase
* feat(cli): add workflow verification manifest types and loader
* feat(cli): add workflow verification runner
* feat(cli): integrate workflow verification into shipcheck pipeline
---------
Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
---
internal/cli/root.go | 1 +
internal/cli/workflow_verify.go | 77 +++++
internal/pipeline/dogfood.go | 392 ++++++++++++++++++++++++++
internal/pipeline/dogfood_test.go | 177 ++++++++++++
internal/pipeline/fullrun.go | 35 +++
internal/pipeline/workflow_manifest.go | 156 +++++++++++
internal/pipeline/workflow_manifest_test.go | 189 +++++++++++++
internal/pipeline/workflow_verify.go | 418 ++++++++++++++++++++++++++++
internal/pipeline/workflow_verify_test.go | 250 +++++++++++++++++
skills/printing-press/SKILL.md | 12 +-
10 files changed, 1703 insertions(+), 4 deletions(-)
diff --git a/internal/cli/root.go b/internal/cli/root.go
index 116b52e3..31e8a9b6 100644
--- a/internal/cli/root.go
+++ b/internal/cli/root.go
@@ -51,6 +51,7 @@ func Execute() error {
rootCmd.AddCommand(newLibraryCmd())
rootCmd.AddCommand(newPublishCmd())
rootCmd.AddCommand(newPolishCmd())
+ rootCmd.AddCommand(newWorkflowVerifyCmd())
return rootCmd.Execute()
}
diff --git a/internal/cli/workflow_verify.go b/internal/cli/workflow_verify.go
new file mode 100644
index 00000000..90c1e18c
--- /dev/null
+++ b/internal/cli/workflow_verify.go
@@ -0,0 +1,77 @@
+package cli
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/mvanhorn/cli-printing-press/internal/pipeline"
+ "github.com/spf13/cobra"
+)
+
+func newWorkflowVerifyCmd() *cobra.Command {
+ var dir string
+ var asJSON bool
+
+ cmd := &cobra.Command{
+ Use: "workflow-verify",
+ Short: "Verify a generated CLI can complete its primary workflow",
+ Long: "Run the workflow verification manifest against a built CLI binary, testing that multi-step flows (e.g., ordering a pizza, creating a project) actually work end-to-end.",
+ Example: ` # Verify a generated CLI's primary workflow
+ printing-press workflow-verify --dir ./generated/dominos-pp-cli
+
+ # Output as JSON for programmatic use
+ printing-press workflow-verify --dir ./generated/dominos-pp-cli --json`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ report, err := pipeline.RunWorkflowVerification(dir)
+ if err != nil {
+ return &ExitError{Code: ExitGenerationError, Err: fmt.Errorf("workflow verification: %w", err)}
+ }
+
+ if asJSON {
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetIndent("", " ")
+ return enc.Encode(report)
+ }
+
+ printWorkflowVerifyReport(report)
+ return nil
+ },
+ }
+
+ cmd.Flags().StringVar(&dir, "dir", "", "Path to the generated CLI directory (required)")
+ cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON")
+ _ = cmd.MarkFlagRequired("dir")
+ return cmd
+}
+
+func printWorkflowVerifyReport(report *pipeline.WorkflowVerifyReport) {
+ name := filepath.Base(report.Dir)
+
+ fmt.Printf("Workflow Verification: %s\n", name)
+ fmt.Println("================================")
+ fmt.Println()
+
+ for _, w := range report.Workflows {
+ primary := ""
+ if w.Primary {
+ primary = " (primary)"
+ }
+ fmt.Printf("Workflow: %s%s\n", w.Name, primary)
+ fmt.Printf(" Verdict: %s\n", w.Verdict)
+
+ for i, step := range w.Steps {
+ fmt.Printf(" Step %d: %s -> %s\n", i+1, step.Command, step.Status)
+ if step.Error != "" {
+ fmt.Printf(" Error: %s\n", step.Error)
+ }
+ }
+ fmt.Println()
+ }
+
+ fmt.Printf("Overall Verdict: %s\n", report.Verdict)
+ for _, issue := range report.Issues {
+ fmt.Printf(" - %s\n", issue)
+ }
+}
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
index 051b1a6f..ec729bf7 100644
--- a/internal/pipeline/dogfood.go
+++ b/internal/pipeline/dogfood.go
@@ -15,6 +15,7 @@ import (
"github.com/mvanhorn/cli-printing-press/internal/naming"
openapiparser "github.com/mvanhorn/cli-printing-press/internal/openapi"
apispec "github.com/mvanhorn/cli-printing-press/internal/spec"
+ "gopkg.in/yaml.v3"
)
type DogfoodReport struct {
@@ -27,6 +28,7 @@ type DogfoodReport struct {
DeadFuncs DeadCodeResult `json:"dead_functions"`
PipelineCheck PipelineResult `json:"pipeline_check"`
ExampleCheck ExampleCheckResult `json:"example_check"`
+ WiringCheck WiringCheckResult `json:"wiring_check"`
Issues []string `json:"issues"`
}
@@ -67,6 +69,33 @@ type ExampleCheckResult struct {
Detail string `json:"detail"`
}
+type WiringCheckResult struct {
+ CommandTree CommandTreeResult `json:"command_tree"`
+ ConfigConsist ConfigConsistResult `json:"config_consistency"`
+ WorkflowComplete WorkflowCompleteResult `json:"workflow_completeness"`
+}
+
+type CommandTreeResult struct {
+ Defined int `json:"defined"`
+ Registered int `json:"registered"`
+ Unregistered []string `json:"unregistered,omitempty"`
+}
+
+type ConfigConsistResult struct {
+ WriteFields []string `json:"write_fields,omitempty"`
+ ReadFields []string `json:"read_fields,omitempty"`
+ Mismatched []string `json:"mismatched,omitempty"`
+ Consistent bool `json:"consistent"`
+}
+
+type WorkflowCompleteResult struct {
+ Skipped bool `json:"skipped,omitempty"`
+ TotalSteps int `json:"total_steps"`
+ MappedSteps int `json:"mapped_steps"`
+ UnmappedSteps []string `json:"unmapped_steps,omitempty"`
+ Detail string `json:"detail"`
+}
+
type openAPISpec struct {
Paths []string
Auth apispec.AuthConfig
@@ -100,6 +129,7 @@ func RunDogfood(dir, specPath string) (*DogfoodReport, error) {
report.DeadFuncs = checkDeadFunctions(dir)
report.PipelineCheck = checkPipelineIntegrity(dir)
report.ExampleCheck = checkExamples(dir)
+ report.WiringCheck = checkWiring(dir)
report.Issues = collectDogfoodIssues(report, spec != nil)
report.Verdict = deriveDogfoodVerdict(report, spec != nil)
@@ -549,6 +579,15 @@ func deriveDogfoodVerdict(report *DogfoodReport, hasSpec bool) string {
if report.ExampleCheck.Skipped {
return "WARN"
}
+ if len(report.WiringCheck.CommandTree.Unregistered) > 0 {
+ return "FAIL"
+ }
+ if !report.WiringCheck.ConfigConsist.Consistent && len(report.WiringCheck.ConfigConsist.Mismatched) > 0 {
+ return "FAIL"
+ }
+ if len(report.WiringCheck.WorkflowComplete.UnmappedSteps) > 0 {
+ return "WARN"
+ }
return "PASS"
}
@@ -580,6 +619,21 @@ func collectDogfoodIssues(report *DogfoodReport, hasSpec bool) []string {
if report.ExampleCheck.Skipped {
issues = append(issues, fmt.Sprintf("example check skipped: %s", report.ExampleCheck.Detail))
}
+ if len(report.WiringCheck.CommandTree.Unregistered) > 0 {
+ issues = append(issues, fmt.Sprintf("%d unregistered commands: %s",
+ len(report.WiringCheck.CommandTree.Unregistered),
+ strings.Join(report.WiringCheck.CommandTree.Unregistered, ", ")))
+ }
+ if !report.WiringCheck.ConfigConsist.Consistent && len(report.WiringCheck.ConfigConsist.Mismatched) > 0 {
+ issues = append(issues, fmt.Sprintf("config inconsistency: write fields %v vs read fields %v",
+ report.WiringCheck.ConfigConsist.WriteFields,
+ report.WiringCheck.ConfigConsist.ReadFields))
+ }
+ if len(report.WiringCheck.WorkflowComplete.UnmappedSteps) > 0 {
+ issues = append(issues, fmt.Sprintf("%d unmapped workflow steps: %s",
+ len(report.WiringCheck.WorkflowComplete.UnmappedSteps),
+ strings.Join(report.WiringCheck.WorkflowComplete.UnmappedSteps, ", ")))
+ }
return issues
}
@@ -890,3 +944,341 @@ func containsAny(sources []string, needle string) bool {
}
return false
}
+
+// checkWiring orchestrates all three wiring sub-checks.
+func checkWiring(dir string) WiringCheckResult {
+ return WiringCheckResult{
+ CommandTree: checkCommandTree(dir),
+ ConfigConsist: checkConfigConsistency(dir),
+ WorkflowComplete: checkWorkflowCompleteness(dir),
+ }
+}
+
+// checkCommandTree scans internal/cli/*.go for func new*Cmd() patterns,
+// then runs the built binary's --help to see which commands are registered.
+// Any newXxxCmd() whose command name doesn't appear in help is flagged.
+func checkCommandTree(dir string) CommandTreeResult {
+ result := CommandTreeResult{}
+
+ cliDir := filepath.Join(dir, "internal", "cli")
+ files := listGoFiles(cliDir)
+
+ // Scan for func new*Cmd() patterns
+ cmdFuncRe := regexp.MustCompile(`(?m)^func\s+new(\w+)Cmd\s*\(`)
+ definedCmds := make(map[string]struct{})
+ for _, file := range files {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ continue
+ }
+ matches := cmdFuncRe.FindAllStringSubmatch(string(data), -1)
+ for _, match := range matches {
+ // Convert CamelCase function name part to lowercase command name.
+ // e.g., newAuthLoginCmd -> "authlogin" but we also store original.
+ // The convention is that the command name is the lowercase version
+ // of the captured group, e.g., AuthLogin -> "auth login" or "auth-login".
+ // We'll normalize to lowercase for matching against help output.
+ cmdName := strings.ToLower(match[1])
+ definedCmds[cmdName] = struct{}{}
+ }
+ }
+
+ result.Defined = len(definedCmds)
+ if result.Defined == 0 {
+ return result
+ }
+
+ // Try to build the binary and get help output
+ cliName := findCLIName(dir)
+ if cliName == "" {
+ // Can't verify without a binary, treat all as registered
+ result.Registered = result.Defined
+ return result
+ }
+
+ binaryPath, err := buildDogfoodBinary(dir, cliName)
+ if err != nil {
+ result.Registered = result.Defined
+ return result
+ }
+ defer func() { _ = os.Remove(binaryPath) }()
+
+ helpOut, err := runDogfoodCmd(binaryPath, 15*time.Second, "--help")
+ if err != nil {
+ result.Registered = result.Defined
+ return result
+ }
+
+ // Also gather subcommand help by extracting top-level commands
+ helpLower := strings.ToLower(helpOut)
+ topCmds := extractCommandNames(helpOut)
+ for _, topCmd := range topCmds {
+ subOut, err := runDogfoodCmd(binaryPath, 15*time.Second, topCmd, "--help")
+ if err == nil {
+ helpLower += "\n" + strings.ToLower(subOut)
+ }
+ }
+
+ for cmdName := range definedCmds {
+ if strings.Contains(helpLower, cmdName) {
+ result.Registered++
+ } else {
+ result.Unregistered = append(result.Unregistered, cmdName)
+ }
+ }
+ sort.Strings(result.Unregistered)
+ return result
+}
+
+// extractCommandNames extracts command names from cobra --help output.
+// It looks for the "Available Commands:" section.
+func extractCommandNames(helpOutput string) []string {
+ lines := strings.Split(helpOutput, "\n")
+ var inCommands bool
+ var cmds []string
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+ if trimmed == "Available Commands:" {
+ inCommands = true
+ continue
+ }
+ if inCommands {
+ if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
+ break
+ }
+ parts := strings.Fields(trimmed)
+ if len(parts) > 0 {
+ cmds = append(cmds, parts[0])
+ }
+ }
+ }
+ return cmds
+}
+
+// checkConfigConsistency scans CLI source for token/credential write and read
+// sites, then verifies they reference the same config field names.
+func checkConfigConsistency(dir string) ConfigConsistResult {
+ result := ConfigConsistResult{Consistent: true}
+
+ // Collect all Go source files recursively
+ var sources []string
+ _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return nil
+ }
+ if !info.IsDir() && strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") {
+ sources = append(sources, path)
+ }
+ return nil
+ })
+
+ writePatterns := []*regexp.Regexp{
+ regexp.MustCompile(`(?i)SaveTokens?\s*\(`),
+ regexp.MustCompile(`(?i)SetTokens?\s*\(`),
+ regexp.MustCompile(`(?i)WriteTokens?\s*\(`),
+ regexp.MustCompile(`(?i)config\.Set\s*\(\s*"([^"]+)"`),
+ regexp.MustCompile(`(?i)viper\.Set\s*\(\s*"([^"]+)"`),
+ }
+ readPatterns := []*regexp.Regexp{
+ regexp.MustCompile(`(?i)AuthHeader\s*\(`),
+ regexp.MustCompile(`(?i)GetTokens?\s*\(`),
+ regexp.MustCompile(`(?i)ReadTokens?\s*\(`),
+ regexp.MustCompile(`(?i)config\.Get\s*\(\s*"([^"]+)"`),
+ regexp.MustCompile(`(?i)viper\.Get\s*\(\s*"([^"]+)"`),
+ }
+
+ // Also look for string literals that name token fields
+ fieldExtractRe := regexp.MustCompile(`"([^"]*(?i:token|credential|secret|key|auth)[^"]*)"`)
+
+ writeFields := make(map[string]struct{})
+ readFields := make(map[string]struct{})
+
+ for _, srcPath := range sources {
+ data, err := os.ReadFile(srcPath)
+ if err != nil {
+ continue
+ }
+ content := string(data)
+
+ for _, pat := range writePatterns {
+ if pat.MatchString(content) {
+ // Extract field names from the same lines
+ matches := pat.FindAllStringSubmatch(content, -1)
+ for _, m := range matches {
+ if len(m) > 1 && m[1] != "" {
+ writeFields[m[1]] = struct{}{}
+ }
+ }
+ // Also extract nearby token-related string literals
+ for _, line := range strings.Split(content, "\n") {
+ if pat.MatchString(line) {
+ fieldMatches := fieldExtractRe.FindAllStringSubmatch(line, -1)
+ for _, fm := range fieldMatches {
+ writeFields[fm[1]] = struct{}{}
+ }
+ }
+ }
+ }
+ }
+
+ for _, pat := range readPatterns {
+ if pat.MatchString(content) {
+ matches := pat.FindAllStringSubmatch(content, -1)
+ for _, m := range matches {
+ if len(m) > 1 && m[1] != "" {
+ readFields[m[1]] = struct{}{}
+ }
+ }
+ for _, line := range strings.Split(content, "\n") {
+ if pat.MatchString(line) {
+ fieldMatches := fieldExtractRe.FindAllStringSubmatch(line, -1)
+ for _, fm := range fieldMatches {
+ readFields[fm[1]] = struct{}{}
+ }
+ }
+ }
+ }
+ }
+ }
+
+ result.WriteFields = sortedKeys(writeFields)
+ result.ReadFields = sortedKeys(readFields)
+
+ // If both sides have fields, check overlap
+ if len(writeFields) > 0 && len(readFields) > 0 {
+ overlap := false
+ for wf := range writeFields {
+ if _, ok := readFields[wf]; ok {
+ overlap = true
+ break
+ }
+ }
+ if !overlap {
+ result.Consistent = false
+ // Mismatched = write fields not found in read fields
+ for _, wf := range result.WriteFields {
+ if _, ok := readFields[wf]; !ok {
+ result.Mismatched = append(result.Mismatched, wf)
+ }
+ }
+ for _, rf := range result.ReadFields {
+ if _, ok := writeFields[rf]; !ok {
+ result.Mismatched = append(result.Mismatched, rf)
+ }
+ }
+ result.Mismatched = uniqueSorted(result.Mismatched)
+ }
+ }
+
+ return result
+}
+
+// workflowManifest represents the structure of workflow_verify.yaml.
+type workflowManifest struct {
+ Workflows []workflowDef `yaml:"workflows"`
+}
+
+type workflowDef struct {
+ Name string `yaml:"name"`
+ Steps []workflowStep `yaml:"steps"`
+}
+
+type workflowStep struct {
+ Command string `yaml:"command"`
+ Name string `yaml:"name"`
+}
+
+// checkWorkflowCompleteness verifies that every step in a workflow_verify.yaml
+// manifest has a corresponding registered CLI command.
+func checkWorkflowCompleteness(dir string) WorkflowCompleteResult {
+ manifestPath := filepath.Join(dir, "workflow_verify.yaml")
+ data, err := os.ReadFile(manifestPath)
+ if err != nil {
+ return WorkflowCompleteResult{
+ Skipped: true,
+ Detail: "no workflow_verify.yaml found",
+ }
+ }
+
+ var manifest workflowManifest
+ if err := yaml.Unmarshal(data, &manifest); err != nil {
+ return WorkflowCompleteResult{
+ Skipped: true,
+ Detail: fmt.Sprintf("failed to parse workflow_verify.yaml: %v", err),
+ }
+ }
+
+ // Collect all step commands
+ var stepCommands []string
+ for _, wf := range manifest.Workflows {
+ for _, step := range wf.Steps {
+ if step.Command != "" {
+ stepCommands = append(stepCommands, step.Command)
+ }
+ }
+ }
+
+ result := WorkflowCompleteResult{
+ TotalSteps: len(stepCommands),
+ }
+
+ if len(stepCommands) == 0 {
+ result.Detail = "manifest has no steps"
+ return result
+ }
+
+ // Get help output to check command existence
+ cliName := findCLIName(dir)
+ if cliName == "" {
+ result.Detail = "no CLI binary to verify against"
+ result.MappedSteps = result.TotalSteps
+ return result
+ }
+
+ binaryPath, err := buildDogfoodBinary(dir, cliName)
+ if err != nil {
+ result.Detail = fmt.Sprintf("could not build CLI binary: %v", err)
+ result.MappedSteps = result.TotalSteps
+ return result
+ }
+ defer func() { _ = os.Remove(binaryPath) }()
+
+ helpOut, err := runDogfoodCmd(binaryPath, 15*time.Second, "--help")
+ if err != nil {
+ result.Detail = fmt.Sprintf("failed to run --help: %v", err)
+ result.MappedSteps = result.TotalSteps
+ return result
+ }
+
+ // Gather subcommand help too
+ helpLower := strings.ToLower(helpOut)
+ topCmds := extractCommandNames(helpOut)
+ for _, topCmd := range topCmds {
+ subOut, err := runDogfoodCmd(binaryPath, 15*time.Second, topCmd, "--help")
+ if err == nil {
+ helpLower += "\n" + strings.ToLower(subOut)
+ }
+ }
+
+ for _, cmd := range stepCommands {
+ // Check if all parts of the command appear in help
+ cmdLower := strings.ToLower(cmd)
+ parts := strings.Fields(cmdLower)
+ found := true
+ for _, part := range parts {
+ if !strings.Contains(helpLower, part) {
+ found = false
+ break
+ }
+ }
+ if found {
+ result.MappedSteps++
+ } else {
+ result.UnmappedSteps = append(result.UnmappedSteps, cmd)
+ }
+ }
+
+ result.UnmappedSteps = uniqueSorted(result.UnmappedSteps)
+ result.Detail = fmt.Sprintf("%d/%d workflow steps mapped to commands", result.MappedSteps, result.TotalSteps)
+ return result
+}
diff --git a/internal/pipeline/dogfood_test.go b/internal/pipeline/dogfood_test.go
index 0aa6423d..f6bb15c3 100644
--- a/internal/pipeline/dogfood_test.go
+++ b/internal/pipeline/dogfood_test.go
@@ -325,6 +325,183 @@ func TestExtractFlagNames(t *testing.T) {
}
}
+func TestCheckCommandTree(t *testing.T) {
+ dir := t.TempDir()
+ cliDir := filepath.Join(dir, "internal", "cli")
+ require.NoError(t, os.MkdirAll(cliDir, 0o755))
+
+ // Define two commands via newXxxCmd() functions
+ writeTestFile(t, filepath.Join(cliDir, "foo.go"), `package cli
+func newFooCmd() {}
+`)
+ writeTestFile(t, filepath.Join(cliDir, "bar.go"), `package cli
+func newBarCmd() {}
+`)
+
+ // Without a buildable binary, checkCommandTree can't verify registration,
+ // so it treats all as registered. We test the scanning logic directly.
+ result := checkCommandTree(dir)
+ assert.Equal(t, 2, result.Defined)
+ // No cmd/ directory means no binary, so all treated as registered
+ assert.Equal(t, 2, result.Registered)
+ assert.Empty(t, result.Unregistered)
+}
+
+func TestCheckConfigConsistency(t *testing.T) {
+ dir := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "client"), 0o755))
+
+ // Write site uses "AccessToken"
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "auth.go"), `package cli
+func saveAuth() {
+ config.Set("AccessToken", token)
+}
+`)
+ // Read site uses "DominosToken" - a mismatch
+ writeTestFile(t, filepath.Join(dir, "internal", "client", "client.go"), `package client
+func getAuth() string {
+ return config.Get("DominosToken")
+}
+`)
+
+ result := checkConfigConsistency(dir)
+ assert.False(t, result.Consistent)
+ assert.Contains(t, result.WriteFields, "AccessToken")
+ assert.Contains(t, result.ReadFields, "DominosToken")
+ assert.NotEmpty(t, result.Mismatched)
+}
+
+func TestCheckConfigConsistency_Consistent(t *testing.T) {
+ dir := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+
+ // Both write and read use the same field
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "auth.go"), `package cli
+func saveAuth() {
+ config.Set("AccessToken", token)
+}
+func getAuth() string {
+ return config.Get("AccessToken")
+}
+`)
+
+ result := checkConfigConsistency(dir)
+ assert.True(t, result.Consistent)
+}
+
+func TestCheckWorkflowCompleteness_NoManifest(t *testing.T) {
+ dir := t.TempDir()
+ result := checkWorkflowCompleteness(dir)
+ assert.True(t, result.Skipped)
+ assert.Contains(t, result.Detail, "no workflow_verify.yaml found")
+}
+
+func TestCheckWorkflowCompleteness_HappyPath(t *testing.T) {
+ dir := t.TempDir()
+
+ // Create a manifest with commands that "exist"
+ // Since there's no cmd/ dir, the check will treat all as mapped (can't build binary)
+ writeTestFile(t, filepath.Join(dir, "workflow_verify.yaml"), `workflows:
+ - name: order flow
+ steps:
+ - command: auth login
+ name: login
+ - command: menu list
+ name: browse menu
+`)
+
+ result := checkWorkflowCompleteness(dir)
+ assert.False(t, result.Skipped)
+ assert.Equal(t, 2, result.TotalSteps)
+ // No cmd/ dir means no binary, so all steps treated as mapped
+ assert.Equal(t, 2, result.MappedSteps)
+ assert.Empty(t, result.UnmappedSteps)
+}
+
+func TestCheckWorkflowCompleteness_MissingCommand(t *testing.T) {
+ // This test verifies parsing works correctly for a manifest with steps.
+ // Without a buildable binary, the check can't actually verify commands,
+ // so we test that the YAML parsing and step counting work.
+ dir := t.TempDir()
+
+ writeTestFile(t, filepath.Join(dir, "workflow_verify.yaml"), `workflows:
+ - name: order flow
+ steps:
+ - command: cart checkout
+ name: checkout
+ - command: auth login
+ name: login
+`)
+
+ result := checkWorkflowCompleteness(dir)
+ assert.False(t, result.Skipped)
+ assert.Equal(t, 2, result.TotalSteps)
+}
+
+func TestWiringCheckIntegration(t *testing.T) {
+ dir := t.TempDir()
+
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "client"), 0o755))
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "store"), 0o755))
+
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "root.go"), `package cli
+type rootFlags struct{}
+func initFlags(flags *rootFlags) { _ = flags }
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "client", "client.go"), `package client
+func authHeader(token string) string {
+ return "Bearer " + token
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "store", "store.go"), "package store\n")
+
+ report, err := RunDogfood(dir, "")
+ require.NoError(t, err)
+
+ // WiringCheck should be populated in the report
+ assert.True(t, report.WiringCheck.ConfigConsist.Consistent)
+ assert.True(t, report.WiringCheck.WorkflowComplete.Skipped)
+ assert.Equal(t, 0, report.WiringCheck.CommandTree.Defined)
+}
+
+func TestDeriveDogfoodVerdict_WiringChecks(t *testing.T) {
+ // Test that unregistered commands cause FAIL
+ report := &DogfoodReport{
+ PathCheck: PathCheckResult{Tested: 10, Valid: 10, Pct: 100},
+ AuthCheck: AuthCheckResult{Match: true},
+ DeadFlags: DeadCodeResult{Dead: 0},
+ DeadFuncs: DeadCodeResult{Dead: 0},
+ PipelineCheck: PipelineResult{SyncCallsDomain: true},
+ WiringCheck: WiringCheckResult{
+ CommandTree: CommandTreeResult{Defined: 2, Registered: 1, Unregistered: []string{"bar"}},
+ ConfigConsist: ConfigConsistResult{Consistent: true},
+ WorkflowComplete: WorkflowCompleteResult{Skipped: true},
+ },
+ }
+ assert.Equal(t, "FAIL", deriveDogfoodVerdict(report, true))
+
+ // Test that config inconsistency causes FAIL
+ report.WiringCheck.CommandTree.Unregistered = nil
+ report.WiringCheck.ConfigConsist.Consistent = false
+ report.WiringCheck.ConfigConsist.Mismatched = []string{"AccessToken", "DominosToken"}
+ assert.Equal(t, "FAIL", deriveDogfoodVerdict(report, true))
+
+ // Test that unmapped workflow steps cause WARN
+ report.WiringCheck.ConfigConsist.Consistent = true
+ report.WiringCheck.WorkflowComplete = WorkflowCompleteResult{
+ TotalSteps: 2,
+ MappedSteps: 1,
+ UnmappedSteps: []string{"cart checkout"},
+ }
+ assert.Equal(t, "WARN", deriveDogfoodVerdict(report, true))
+
+ // Test that clean wiring passes
+ report.WiringCheck.WorkflowComplete = WorkflowCompleteResult{Skipped: true}
+ assert.Equal(t, "PASS", deriveDogfoodVerdict(report, true))
+}
+
func writeTestFile(t *testing.T, path string, content string) {
t.Helper()
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
diff --git a/internal/pipeline/fullrun.go b/internal/pipeline/fullrun.go
index a30dfd33..16358a2c 100644
--- a/internal/pipeline/fullrun.go
+++ b/internal/pipeline/fullrun.go
@@ -49,6 +49,10 @@ type FullRunResult struct {
VerificationError string
Remediation *RemediationResult
+ // Step 5.6: Workflow Verification
+ WorkflowVerify *WorkflowVerifyReport
+ WorkflowVerifyError string
+
// Step 5: Scorecard
Scorecard *Scorecard
ScorecardError string
@@ -222,6 +226,15 @@ func MakeBestCLI(apiName, level, specFlag, specURL, outputDir, pressBinary strin
}
}
+ // Step 5.6: Workflow Verification
+ wfReport, wfErr := RunWorkflowVerification(workingDir)
+ if wfErr != nil {
+ result.WorkflowVerifyError = wfErr.Error()
+ result.Errors = append(result.Errors, fmt.Sprintf("workflow verification: %v", wfErr))
+ } else {
+ result.WorkflowVerify = wfReport
+ }
+
// Step 6: Scorecard
scorecard, scErr := RunScorecard(workingDir, proofsDir, qualitySpecPath, nil)
if scErr != nil {
@@ -498,6 +511,14 @@ func PrintComparisonTable(results []*FullRunResult) string {
return summary
})
+ // Workflow Verification
+ writeRow(&b, "Workflow Verify", results, func(r *FullRunResult) string {
+ if r.WorkflowVerify == nil {
+ return "n/a"
+ }
+ return string(r.WorkflowVerify.Verdict)
+ })
+
// LLM Polish
writeRow(&b, "LLM Polish", results, func(r *FullRunResult) string {
if r.PolishResult == nil {
@@ -688,6 +709,20 @@ func GenerateLearningsPlan(results []*FullRunResult, outputPath string) error {
}
b.WriteString("\n")
+ // Workflow Verification summary
+ b.WriteString("## Workflow Verification Summary\n\n")
+ for _, r := range results {
+ if r.WorkflowVerify != nil {
+ fmt.Fprintf(&b, "- **%s** - %s\n", r.APIName, r.WorkflowVerify.Verdict)
+ for _, issue := range r.WorkflowVerify.Issues {
+ fmt.Fprintf(&b, " - %s\n", issue)
+ }
+ } else {
+ fmt.Fprintf(&b, "- **%s** - not run (%s)\n", r.APIName, r.WorkflowVerifyError)
+ }
+ }
+ b.WriteString("\n")
+
// Next steps
b.WriteString("## Next Steps\n\n")
b.WriteString("1. Fix the highest-priority template gaps listed above\n")
diff --git a/internal/pipeline/workflow_manifest.go b/internal/pipeline/workflow_manifest.go
new file mode 100644
index 00000000..e3861c3b
--- /dev/null
+++ b/internal/pipeline/workflow_manifest.go
@@ -0,0 +1,156 @@
+package pipeline
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "gopkg.in/yaml.v3"
+)
+
+// WorkflowManifest is the top-level structure for workflow_verify.yaml
+type WorkflowManifest struct {
+ Workflows []Workflow `yaml:"workflows" json:"workflows"`
+}
+
+// Workflow represents a single testable workflow
+type Workflow struct {
+ Name string `yaml:"name" json:"name"`
+ Primary bool `yaml:"primary" json:"primary"`
+ Steps []WorkflowStep `yaml:"steps" json:"steps"`
+}
+
+// WorkflowStep represents one step in a workflow
+type WorkflowStep struct {
+ Command string `yaml:"command" json:"command"`
+ Args map[string]string `yaml:"args,omitempty" json:"args,omitempty"`
+ ArgsStdin bool `yaml:"args_stdin,omitempty" json:"args_stdin,omitempty"`
+ Extract map[string]string `yaml:"extract,omitempty" json:"extract,omitempty"`
+ Mode StepMode `yaml:"mode" json:"mode"`
+ ExpectFields []string `yaml:"expect_fields,omitempty" json:"expect_fields,omitempty"`
+ AuthRequired bool `yaml:"auth_required,omitempty" json:"auth_required,omitempty"`
+}
+
+// StepMode controls how a step is executed during verification
+type StepMode string
+
+const (
+ StepModeLive StepMode = "live"
+ StepModeMock StepMode = "mock"
+ StepModeLocal StepMode = "local"
+)
+
+// StepResult holds the outcome of executing a single workflow step
+type StepResult struct {
+ Command string `json:"command"`
+ Status StepStatus `json:"status"`
+ Output string `json:"output,omitempty"`
+ Error string `json:"error,omitempty"`
+ Extracted map[string]string `json:"extracted,omitempty"`
+}
+
+// StepStatus classifies the result of a step execution
+type StepStatus string
+
+const (
+ StepStatusPass StepStatus = "pass"
+ StepStatusFailCLIBug StepStatus = "fail-cli-bug"
+ StepStatusBlockedAuth StepStatus = "blocked-auth"
+ StepStatusBlockedTransient StepStatus = "blocked-transient"
+ StepStatusSkippedNoInput StepStatus = "skipped-no-input"
+ StepStatusSkippedAuth StepStatus = "skipped-auth-required"
+)
+
+// WorkflowResult holds the overall result of running a workflow
+type WorkflowResult struct {
+ Name string `json:"name"`
+ Primary bool `json:"primary"`
+ Steps []StepResult `json:"steps"`
+ Verdict WorkflowVerdict `json:"verdict"`
+}
+
+// WorkflowVerdict is the overall outcome of a workflow verification
+type WorkflowVerdict string
+
+const (
+ WorkflowVerdictPass WorkflowVerdict = "workflow-pass"
+ WorkflowVerdictFail WorkflowVerdict = "workflow-fail"
+ WorkflowVerdictUnverified WorkflowVerdict = "unverified-needs-auth"
+)
+
+// WorkflowVerifyReport holds the complete verification results
+type WorkflowVerifyReport struct {
+ Dir string `json:"dir"`
+ Workflows []WorkflowResult `json:"workflows"`
+ Verdict WorkflowVerdict `json:"verdict"`
+ Issues []string `json:"issues,omitempty"`
+}
+
+// LoadWorkflowManifest reads and parses a workflow_verify.yaml from the given directory.
+// Returns nil, nil if the file does not exist (not an error).
+func LoadWorkflowManifest(dir string) (*WorkflowManifest, error) {
+ path := filepath.Join(dir, "workflow_verify.yaml")
+ data, err := os.ReadFile(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("reading workflow manifest: %w", err)
+ }
+
+ var manifest WorkflowManifest
+ if err := yaml.Unmarshal(data, &manifest); err != nil {
+ return nil, fmt.Errorf("parsing workflow manifest: %w", err)
+ }
+
+ return &manifest, nil
+}
+
+// PrimaryWorkflow returns the first workflow marked as primary, or nil if none.
+func (m *WorkflowManifest) PrimaryWorkflow() *Workflow {
+ if m == nil {
+ return nil
+ }
+ for i := range m.Workflows {
+ if m.Workflows[i].Primary {
+ return &m.Workflows[i]
+ }
+ }
+ return nil
+}
+
+// CommandNames returns all unique command names referenced across all workflow steps.
+func (m *WorkflowManifest) CommandNames() []string {
+ if m == nil {
+ return nil
+ }
+ seen := make(map[string]struct{})
+ var names []string
+ for _, w := range m.Workflows {
+ for _, s := range w.Steps {
+ cmd := s.BaseCommand()
+ if _, ok := seen[cmd]; !ok {
+ seen[cmd] = struct{}{}
+ names = append(names, cmd)
+ }
+ }
+ }
+ sort.Strings(names)
+ return names
+}
+
+// BaseCommand returns the command name without variable substitutions.
+// e.g., "cart add ${item_code}" -> "cart add"
+func (s *WorkflowStep) BaseCommand() string {
+ parts := strings.Fields(s.Command)
+ var clean []string
+ for _, p := range parts {
+ if strings.HasPrefix(p, "${") {
+ break
+ }
+ clean = append(clean, p)
+ }
+ return strings.Join(clean, " ")
+}
diff --git a/internal/pipeline/workflow_manifest_test.go b/internal/pipeline/workflow_manifest_test.go
new file mode 100644
index 00000000..8e8ed736
--- /dev/null
+++ b/internal/pipeline/workflow_manifest_test.go
@@ -0,0 +1,189 @@
+package pipeline
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const testManifestYAML = `workflows:
+ - name: "Order a pizza for delivery"
+ primary: true
+ steps:
+ - command: "stores find_stores"
+ args:
+ s: "1600 Pennsylvania Ave"
+ c: "Washington, DC 20500"
+ extract:
+ store_id: "$.Stores[0].StoreID"
+ mode: live
+ - command: "menu get_menu"
+ args:
+ store: "${store_id}"
+ extract:
+ item_code: "$.MenuCategories[0].Products[0].Code"
+ mode: live
+ - command: "cart new"
+ args:
+ store: "${store_id}"
+ service: "delivery"
+ mode: local
+ - command: "cart add ${item_code}"
+ mode: local
+ - command: "orders price_order"
+ args_stdin: true
+ mode: mock
+ expect_fields:
+ - "Order.Amounts.Customer"
+ - "Order.Products"
+`
+
+func TestLoadWorkflowManifest_Valid(t *testing.T) {
+ dir := t.TempDir()
+ writeTestFile(t, filepath.Join(dir, "workflow_verify.yaml"), testManifestYAML)
+
+ m, err := LoadWorkflowManifest(dir)
+ require.NoError(t, err)
+ require.NotNil(t, m)
+
+ require.Len(t, m.Workflows, 1)
+ wf := m.Workflows[0]
+ assert.Equal(t, "Order a pizza for delivery", wf.Name)
+ assert.True(t, wf.Primary)
+ require.Len(t, wf.Steps, 5)
+
+ // Step 0: stores find_stores
+ s0 := wf.Steps[0]
+ assert.Equal(t, "stores find_stores", s0.Command)
+ assert.Equal(t, StepModeLive, s0.Mode)
+ assert.Equal(t, "1600 Pennsylvania Ave", s0.Args["s"])
+ assert.Equal(t, "Washington, DC 20500", s0.Args["c"])
+ assert.Equal(t, "$.Stores[0].StoreID", s0.Extract["store_id"])
+ assert.False(t, s0.ArgsStdin)
+
+ // Step 1: menu get_menu
+ s1 := wf.Steps[1]
+ assert.Equal(t, "menu get_menu", s1.Command)
+ assert.Equal(t, StepModeLive, s1.Mode)
+ assert.Equal(t, "${store_id}", s1.Args["store"])
+ assert.Equal(t, "$.MenuCategories[0].Products[0].Code", s1.Extract["item_code"])
+
+ // Step 2: cart new
+ s2 := wf.Steps[2]
+ assert.Equal(t, "cart new", s2.Command)
+ assert.Equal(t, StepModeLocal, s2.Mode)
+ assert.Equal(t, "${store_id}", s2.Args["store"])
+ assert.Equal(t, "delivery", s2.Args["service"])
+
+ // Step 3: cart add ${item_code}
+ s3 := wf.Steps[3]
+ assert.Equal(t, "cart add ${item_code}", s3.Command)
+ assert.Equal(t, StepModeLocal, s3.Mode)
+
+ // Step 4: orders price_order
+ s4 := wf.Steps[4]
+ assert.Equal(t, "orders price_order", s4.Command)
+ assert.Equal(t, StepModeMock, s4.Mode)
+ assert.True(t, s4.ArgsStdin)
+ assert.Equal(t, []string{"Order.Amounts.Customer", "Order.Products"}, s4.ExpectFields)
+}
+
+func TestLoadWorkflowManifest_NotFound(t *testing.T) {
+ dir := t.TempDir()
+
+ m, err := LoadWorkflowManifest(dir)
+ assert.NoError(t, err)
+ assert.Nil(t, m)
+}
+
+func TestLoadWorkflowManifest_InvalidYAML(t *testing.T) {
+ dir := t.TempDir()
+ writeTestFile(t, filepath.Join(dir, "workflow_verify.yaml"), "{{not valid yaml]]]")
+
+ m, err := LoadWorkflowManifest(dir)
+ assert.Error(t, err)
+ assert.Nil(t, m)
+ assert.Contains(t, err.Error(), "parsing workflow manifest")
+}
+
+func TestPrimaryWorkflow(t *testing.T) {
+ m := &WorkflowManifest{
+ Workflows: []Workflow{
+ {Name: "secondary", Primary: false},
+ {Name: "primary-one", Primary: true},
+ {Name: "another", Primary: false},
+ },
+ }
+
+ pw := m.PrimaryWorkflow()
+ require.NotNil(t, pw)
+ assert.Equal(t, "primary-one", pw.Name)
+ assert.True(t, pw.Primary)
+}
+
+func TestPrimaryWorkflow_None(t *testing.T) {
+ m := &WorkflowManifest{
+ Workflows: []Workflow{
+ {Name: "a", Primary: false},
+ {Name: "b", Primary: false},
+ },
+ }
+
+ assert.Nil(t, m.PrimaryWorkflow())
+
+ // Also test nil receiver
+ var nilManifest *WorkflowManifest
+ assert.Nil(t, nilManifest.PrimaryWorkflow())
+}
+
+func TestCommandNames(t *testing.T) {
+ m := &WorkflowManifest{
+ Workflows: []Workflow{
+ {
+ Name: "workflow-a",
+ Steps: []WorkflowStep{
+ {Command: "stores find_stores"},
+ {Command: "cart add ${item_code}"},
+ {Command: "menu get_menu"},
+ },
+ },
+ {
+ Name: "workflow-b",
+ Steps: []WorkflowStep{
+ {Command: "stores find_stores"}, // duplicate
+ {Command: "orders price_order"},
+ {Command: "cart add ${other}"}, // same base as "cart add" above
+ },
+ },
+ },
+ }
+
+ names := m.CommandNames()
+ assert.Equal(t, []string{"cart add", "menu get_menu", "orders price_order", "stores find_stores"}, names)
+
+ // Nil receiver
+ var nilManifest *WorkflowManifest
+ assert.Nil(t, nilManifest.CommandNames())
+}
+
+func TestBaseCommand(t *testing.T) {
+ tests := []struct {
+ command string
+ expected string
+ }{
+ {"stores find_stores", "stores find_stores"},
+ {"cart add ${item_code}", "cart add"},
+ {"${var}", ""},
+ {"menu get_menu ${store_id} ${extra}", "menu get_menu"},
+ {"simple", "simple"},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.command, func(t *testing.T) {
+ s := &WorkflowStep{Command: tc.command}
+ assert.Equal(t, tc.expected, s.BaseCommand())
+ })
+ }
+}
diff --git a/internal/pipeline/workflow_verify.go b/internal/pipeline/workflow_verify.go
new file mode 100644
index 00000000..60fb8e6f
--- /dev/null
+++ b/internal/pipeline/workflow_verify.go
@@ -0,0 +1,418 @@
+package pipeline
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+)
+
+// RunWorkflowVerification builds the CLI and runs all workflows from the manifest.
+func RunWorkflowVerification(dir string) (*WorkflowVerifyReport, error) {
+ manifest, err := LoadWorkflowManifest(dir)
+ if err != nil {
+ return nil, fmt.Errorf("loading workflow manifest: %w", err)
+ }
+
+ if manifest == nil {
+ report := &WorkflowVerifyReport{
+ Dir: dir,
+ Verdict: WorkflowVerdictPass,
+ Issues: []string{"no workflow manifest found, skipping"},
+ }
+ if err := writeWorkflowVerifyReport(dir, report); err != nil {
+ return nil, err
+ }
+ return report, nil
+ }
+
+ // Build the CLI binary.
+ cliName := findCLIName(dir)
+ if cliName == "" {
+ report := &WorkflowVerifyReport{
+ Dir: dir,
+ Verdict: WorkflowVerdictFail,
+ Issues: []string{"no CLI command directory found, cannot build binary"},
+ }
+ if err := writeWorkflowVerifyReport(dir, report); err != nil {
+ return nil, err
+ }
+ return report, nil
+ }
+
+ binary, err := buildDogfoodBinary(dir, cliName)
+ if err != nil {
+ return nil, fmt.Errorf("building CLI binary: %w", err)
+ }
+ defer os.Remove(binary)
+
+ report := &WorkflowVerifyReport{
+ Dir: dir,
+ }
+
+ for _, wf := range manifest.Workflows {
+ result := runWorkflow(binary, wf, dir)
+ report.Workflows = append(report.Workflows, result)
+ }
+
+ report.Verdict = deriveOverallVerdict(manifest, report.Workflows)
+
+ if err := writeWorkflowVerifyReport(dir, report); err != nil {
+ return nil, err
+ }
+ return report, nil
+}
+
+// runWorkflow executes all steps in a workflow sequentially.
+func runWorkflow(binary string, wf Workflow, dir string) WorkflowResult {
+ result := WorkflowResult{
+ Name: wf.Name,
+ Primary: wf.Primary,
+ }
+
+ vars := make(map[string]string)
+ authBlocked := false
+
+ for _, step := range wf.Steps {
+ var sr StepResult
+ sr.Command = step.Command
+
+ // Check if a prior step's failure should skip this one.
+ if authBlocked {
+ sr.Status = StepStatusSkippedAuth
+ result.Steps = append(result.Steps, sr)
+ continue
+ }
+
+ // Check if we need variables from a prior step that aren't available.
+ cmdExpanded := substituteVars(step.Command, vars)
+ if strings.Contains(cmdExpanded, "${") {
+ sr.Status = StepStatusSkippedNoInput
+ sr.Error = "missing variable from prior step"
+ result.Steps = append(result.Steps, sr)
+ continue
+ }
+
+ sr = executeStep(binary, step, cmdExpanded, dir)
+
+ // Extract values on success.
+ if sr.Status == StepStatusPass && len(step.Extract) > 0 {
+ sr.Extracted = make(map[string]string)
+ for varName, jsonPath := range step.Extract {
+ val, err := extractJSONField([]byte(sr.Output), jsonPath)
+ if err == nil {
+ vars[varName] = val
+ sr.Extracted[varName] = val
+ }
+ }
+ }
+
+ if sr.Status == StepStatusBlockedAuth {
+ authBlocked = true
+ }
+
+ result.Steps = append(result.Steps, sr)
+ }
+
+ result.Verdict = deriveWorkflowVerdict(result.Steps)
+ return result
+}
+
+// executeStep runs a single workflow step and classifies the result.
+func executeStep(binary string, step WorkflowStep, cmdExpanded string, dir string) StepResult {
+ sr := StepResult{
+ Command: step.Command,
+ }
+
+ args := strings.Fields(cmdExpanded)
+ args = append(args, "--json")
+
+ maxAttempts := 1
+ if step.Mode == StepModeLive {
+ maxAttempts = 3
+ }
+
+ var output string
+ var cmdErr error
+
+ for attempt := 0; attempt < maxAttempts; attempt++ {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ cmd := exec.CommandContext(ctx, binary, args...)
+ cmd.Dir = dir
+ out, err := cmd.CombinedOutput()
+ cancel()
+
+ output = string(out)
+ cmdErr = err
+
+ if err != nil && ctx.Err() == context.DeadlineExceeded {
+ if attempt < maxAttempts-1 {
+ continue
+ }
+ sr.Status = StepStatusBlockedTransient
+ sr.Error = "timed out after 30s"
+ sr.Output = output
+ return sr
+ }
+
+ if err == nil {
+ break
+ }
+
+ // On transient errors for live mode, retry.
+ if step.Mode == StepModeLive && isTransientError(output) && attempt < maxAttempts-1 {
+ continue
+ }
+ break
+ }
+
+ sr.Output = output
+
+ if cmdErr != nil {
+ return classifyError(sr, output)
+ }
+
+ // Validate JSON output.
+ var parsed interface{}
+ if err := json.Unmarshal([]byte(output), &parsed); err != nil {
+ sr.Status = StepStatusFailCLIBug
+ sr.Error = "output is not valid JSON"
+ return sr
+ }
+
+ // Check expect_fields if specified.
+ if len(step.ExpectFields) > 0 {
+ if obj, ok := parsed.(map[string]interface{}); ok {
+ for _, field := range step.ExpectFields {
+ if _, exists := obj[field]; !exists {
+ sr.Status = StepStatusFailCLIBug
+ sr.Error = fmt.Sprintf("expected field %q not found in output", field)
+ return sr
+ }
+ }
+ } else {
+ sr.Status = StepStatusFailCLIBug
+ sr.Error = "output is not a JSON object, cannot check expect_fields"
+ return sr
+ }
+ }
+
+ sr.Status = StepStatusPass
+ return sr
+}
+
+// classifyError determines the step status from a command failure.
+func classifyError(sr StepResult, output string) StepResult {
+ lower := strings.ToLower(output)
+
+ if strings.Contains(lower, "unknown command") || strings.Contains(lower, "unknown flag") {
+ sr.Status = StepStatusFailCLIBug
+ sr.Error = "unknown command or flag"
+ return sr
+ }
+
+ if strings.Contains(lower, "unauthorized") || strings.Contains(lower, "forbidden") ||
+ strings.Contains(lower, "authentication required") || strings.Contains(lower, "403") {
+ sr.Status = StepStatusBlockedAuth
+ sr.Error = "authentication required"
+ return sr
+ }
+
+ if isTransientError(output) {
+ sr.Status = StepStatusBlockedTransient
+ sr.Error = "transient error"
+ return sr
+ }
+
+ sr.Status = StepStatusFailCLIBug
+ sr.Error = "command failed with non-zero exit"
+ return sr
+}
+
+// isTransientError checks if error output suggests a transient failure.
+func isTransientError(output string) bool {
+ lower := strings.ToLower(output)
+ return strings.Contains(lower, "500") ||
+ strings.Contains(lower, "502") ||
+ strings.Contains(lower, "503") ||
+ strings.Contains(lower, "504") ||
+ strings.Contains(lower, "connection refused") ||
+ strings.Contains(lower, "timeout")
+}
+
+// substituteVars replaces ${var_name} placeholders with values from vars.
+func substituteVars(s string, vars map[string]string) string {
+ for k, v := range vars {
+ s = strings.ReplaceAll(s, "${"+k+"}", v)
+ }
+ return s
+}
+
+// extractJSONField extracts a value from JSON data using a simplified jq-style path.
+// Supports paths like $.name, $.data.id, $.items[0].code.
+func extractJSONField(jsonData []byte, path string) (string, error) {
+ // Strip $. prefix
+ path = strings.TrimPrefix(path, "$.")
+ if path == "" {
+ return "", fmt.Errorf("empty path")
+ }
+
+ var raw interface{}
+ if err := json.Unmarshal(jsonData, &raw); err != nil {
+ return "", fmt.Errorf("parsing JSON: %w", err)
+ }
+
+ // Split by . but handle array notation
+ segments := splitJSONPath(path)
+
+ current := raw
+ for _, seg := range segments {
+ name, idx, hasIdx := parseSegment(seg)
+
+ // Navigate into map
+ if name != "" {
+ obj, ok := current.(map[string]interface{})
+ if !ok {
+ return "", fmt.Errorf("expected object at %q, got %T", seg, current)
+ }
+ val, exists := obj[name]
+ if !exists {
+ return "", fmt.Errorf("field %q not found", name)
+ }
+ current = val
+ }
+
+ // Navigate into array if index present
+ if hasIdx {
+ arr, ok := current.([]interface{})
+ if !ok {
+ return "", fmt.Errorf("expected array at %q, got %T", seg, current)
+ }
+ if idx < 0 || idx >= len(arr) {
+ return "", fmt.Errorf("index %d out of range for array of length %d", idx, len(arr))
+ }
+ current = arr[idx]
+ }
+ }
+
+ // Convert to string
+ switch v := current.(type) {
+ case string:
+ return v, nil
+ case float64:
+ if v == float64(int64(v)) {
+ return fmt.Sprintf("%d", int64(v)), nil
+ }
+ return fmt.Sprintf("%g", v), nil
+ case bool:
+ return fmt.Sprintf("%t", v), nil
+ case nil:
+ return "", fmt.Errorf("value is null")
+ default:
+ b, err := json.Marshal(v)
+ if err != nil {
+ return "", fmt.Errorf("marshaling value: %w", err)
+ }
+ return string(b), nil
+ }
+}
+
+// arrayIndexRe matches segments like "items[0]".
+var arrayIndexRe = regexp.MustCompile(`^([a-zA-Z_][a-zA-Z0-9_]*)\[(\d+)\]$`)
+
+// splitJSONPath splits a dot-notation path into segments.
+func splitJSONPath(path string) []string {
+ return strings.Split(path, ".")
+}
+
+// parseSegment extracts a field name and optional array index from a path segment.
+func parseSegment(seg string) (name string, idx int, hasIdx bool) {
+ m := arrayIndexRe.FindStringSubmatch(seg)
+ if m != nil {
+ idx := 0
+ fmt.Sscanf(m[2], "%d", &idx)
+ return m[1], idx, true
+ }
+ return seg, 0, false
+}
+
+// deriveWorkflowVerdict determines the verdict for a single workflow from its step results.
+func deriveWorkflowVerdict(steps []StepResult) WorkflowVerdict {
+ hasPass := false
+ for _, s := range steps {
+ switch s.Status {
+ case StepStatusFailCLIBug:
+ return WorkflowVerdictFail
+ case StepStatusPass:
+ hasPass = true
+ }
+ }
+
+ // Check if auth blocked before any substantive pass.
+ if !hasPass {
+ for _, s := range steps {
+ if s.Status == StepStatusBlockedAuth || s.Status == StepStatusSkippedAuth {
+ return WorkflowVerdictUnverified
+ }
+ }
+ }
+
+ return WorkflowVerdictPass
+}
+
+// deriveOverallVerdict determines the report verdict from workflow results.
+func deriveOverallVerdict(manifest *WorkflowManifest, results []WorkflowResult) WorkflowVerdict {
+ // Use primary workflow if available.
+ primary := manifest.PrimaryWorkflow()
+ if primary != nil {
+ for _, r := range results {
+ if r.Name == primary.Name {
+ return r.Verdict
+ }
+ }
+ }
+
+ // Fall back to worst-case across all workflows.
+ worst := WorkflowVerdictPass
+ for _, r := range results {
+ if r.Verdict == WorkflowVerdictFail {
+ return WorkflowVerdictFail
+ }
+ if r.Verdict == WorkflowVerdictUnverified && worst == WorkflowVerdictPass {
+ worst = WorkflowVerdictUnverified
+ }
+ }
+ return worst
+}
+
+// writeWorkflowVerifyReport writes the report as JSON to the given directory.
+func writeWorkflowVerifyReport(dir string, report *WorkflowVerifyReport) error {
+ data, err := json.MarshalIndent(report, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshaling workflow verify report: %w", err)
+ }
+ path := filepath.Join(dir, "workflow-verify-report.json")
+ if err := os.WriteFile(path, data, 0o644); err != nil {
+ return fmt.Errorf("writing workflow verify report: %w", err)
+ }
+ return nil
+}
+
+// LoadWorkflowVerifyReport reads a previously written workflow verify report.
+func LoadWorkflowVerifyReport(dir string) (*WorkflowVerifyReport, error) {
+ path := filepath.Join(dir, "workflow-verify-report.json")
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ var report WorkflowVerifyReport
+ if err := json.Unmarshal(data, &report); err != nil {
+ return nil, err
+ }
+ return &report, nil
+}
diff --git a/internal/pipeline/workflow_verify_test.go b/internal/pipeline/workflow_verify_test.go
new file mode 100644
index 00000000..6cdc8b78
--- /dev/null
+++ b/internal/pipeline/workflow_verify_test.go
@@ -0,0 +1,250 @@
+package pipeline
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRunWorkflowVerification_NoManifest(t *testing.T) {
+ dir := t.TempDir()
+
+ report, err := RunWorkflowVerification(dir)
+ require.NoError(t, err)
+ require.NotNil(t, report)
+
+ assert.Equal(t, WorkflowVerdictPass, report.Verdict)
+ require.Len(t, report.Issues, 1)
+ assert.Contains(t, report.Issues[0], "no workflow manifest found")
+
+ // Verify report was written to disk.
+ loaded, err := LoadWorkflowVerifyReport(dir)
+ require.NoError(t, err)
+ assert.Equal(t, report.Verdict, loaded.Verdict)
+}
+
+func TestRunWorkflowVerification_NoCliName(t *testing.T) {
+ dir := t.TempDir()
+
+ // Write a manifest but no cmd/ directory.
+ writeTestFile(t, filepath.Join(dir, "workflow_verify.yaml"), `workflows:
+ - name: test flow
+ steps:
+ - command: hello
+ mode: local
+`)
+
+ report, err := RunWorkflowVerification(dir)
+ require.NoError(t, err)
+ require.NotNil(t, report)
+
+ assert.Equal(t, WorkflowVerdictFail, report.Verdict)
+ require.NotEmpty(t, report.Issues)
+ assert.Contains(t, report.Issues[0], "no CLI command directory found")
+}
+
+func TestSubstituteVars(t *testing.T) {
+ tests := []struct {
+ name string
+ s string
+ vars map[string]string
+ want string
+ }{
+ {
+ name: "single variable",
+ s: "stores find ${store_id}",
+ vars: map[string]string{"store_id": "123"},
+ want: "stores find 123",
+ },
+ {
+ name: "no variables",
+ s: "no vars here",
+ vars: map[string]string{},
+ want: "no vars here",
+ },
+ {
+ name: "multiple variables",
+ s: "${a} and ${b}",
+ vars: map[string]string{"a": "1", "b": "2"},
+ want: "1 and 2",
+ },
+ {
+ name: "nil vars map",
+ s: "hello ${world}",
+ vars: nil,
+ want: "hello ${world}",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, substituteVars(tt.s, tt.vars))
+ })
+ }
+}
+
+func TestExtractJSONField(t *testing.T) {
+ tests := []struct {
+ name string
+ json string
+ path string
+ want string
+ wantErr bool
+ }{
+ {
+ name: "simple field",
+ json: `{"name": "John"}`,
+ path: "$.name",
+ want: "John",
+ },
+ {
+ name: "nested field",
+ json: `{"data": {"id": "123"}}`,
+ path: "$.data.id",
+ want: "123",
+ },
+ {
+ name: "array index",
+ json: `{"items": [{"code": "abc"}]}`,
+ path: "$.items[0].code",
+ want: "abc",
+ },
+ {
+ name: "numeric value",
+ json: `{"count": 42}`,
+ path: "$.count",
+ want: "42",
+ },
+ {
+ name: "invalid path - missing field",
+ json: `{"name": "John"}`,
+ path: "$.missing",
+ wantErr: true,
+ },
+ {
+ name: "invalid path - not an object",
+ json: `{"name": "John"}`,
+ path: "$.name.sub",
+ wantErr: true,
+ },
+ {
+ name: "invalid JSON",
+ json: `not json`,
+ path: "$.name",
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := extractJSONField([]byte(tt.json), tt.path)
+ if tt.wantErr {
+ assert.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ }
+ })
+ }
+}
+
+func TestDeriveWorkflowVerdict(t *testing.T) {
+ tests := []struct {
+ name string
+ steps []StepResult
+ want WorkflowVerdict
+ }{
+ {
+ name: "all pass",
+ steps: []StepResult{
+ {Status: StepStatusPass},
+ {Status: StepStatusPass},
+ },
+ want: WorkflowVerdictPass,
+ },
+ {
+ name: "one fail-cli-bug",
+ steps: []StepResult{
+ {Status: StepStatusPass},
+ {Status: StepStatusFailCLIBug},
+ },
+ want: WorkflowVerdictFail,
+ },
+ {
+ name: "first step blocked-auth no pass",
+ steps: []StepResult{
+ {Status: StepStatusBlockedAuth},
+ {Status: StepStatusSkippedAuth},
+ },
+ want: WorkflowVerdictUnverified,
+ },
+ {
+ name: "auth blocked after a pass",
+ steps: []StepResult{
+ {Status: StepStatusPass},
+ {Status: StepStatusBlockedAuth},
+ },
+ want: WorkflowVerdictPass,
+ },
+ {
+ name: "empty steps",
+ steps: []StepResult{},
+ want: WorkflowVerdictPass,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, deriveWorkflowVerdict(tt.steps))
+ })
+ }
+}
+
+func TestLoadWorkflowVerifyReport(t *testing.T) {
+ dir := t.TempDir()
+
+ report := &WorkflowVerifyReport{
+ Dir: dir,
+ Verdict: WorkflowVerdictPass,
+ Workflows: []WorkflowResult{
+ {
+ Name: "main flow",
+ Primary: true,
+ Verdict: WorkflowVerdictPass,
+ Steps: []StepResult{
+ {
+ Command: "users list",
+ Status: StepStatusPass,
+ Output: `{"users": []}`,
+ },
+ },
+ },
+ },
+ Issues: []string{"test issue"},
+ }
+
+ data, err := json.MarshalIndent(report, "", " ")
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "workflow-verify-report.json"), data, 0o644))
+
+ loaded, err := LoadWorkflowVerifyReport(dir)
+ require.NoError(t, err)
+ assert.Equal(t, report.Dir, loaded.Dir)
+ assert.Equal(t, report.Verdict, loaded.Verdict)
+ require.Len(t, loaded.Workflows, 1)
+ assert.Equal(t, "main flow", loaded.Workflows[0].Name)
+ assert.True(t, loaded.Workflows[0].Primary)
+ require.Len(t, loaded.Workflows[0].Steps, 1)
+ assert.Equal(t, StepStatusPass, loaded.Workflows[0].Steps[0].Status)
+ assert.Equal(t, []string{"test issue"}, loaded.Issues)
+}
+
+func TestLoadWorkflowVerifyReport_NotFound(t *testing.T) {
+ dir := t.TempDir()
+ _, err := LoadWorkflowVerifyReport(dir)
+ assert.Error(t, err)
+}
diff --git a/skills/printing-press/SKILL.md b/skills/printing-press/SKILL.md
index d26a61fa..a96f4e93 100644
--- a/skills/printing-press/SKILL.md
+++ b/skills/printing-press/SKILL.md
@@ -1103,14 +1103,16 @@ Include:
Run one combined verification block.
```bash
-printing-press dogfood --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec>
-printing-press verify --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec> --fix
-printing-press scorecard --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec>
+printing-press dogfood --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec>
+printing-press verify --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec> --fix
+printing-press workflow-verify --dir "$PRESS_LIBRARY/<api>-pp-cli"
+printing-press scorecard --dir "$PRESS_LIBRARY/<api>-pp-cli" --spec <same-spec>
```
Interpretation:
-- `dogfood` catches dead flags, dead helpers, invalid paths, example drift, and broken data wiring
+- `dogfood` catches dead flags, dead helpers, invalid paths, example drift, broken data wiring, and command tree/config field wiring bugs
- `verify` catches runtime breakage and runs the auto-fix loop for common failures
+- `workflow-verify` tests the primary workflow end-to-end using the verification manifest (workflow_verify.yaml). Three verdicts: workflow-pass, workflow-fail, unverified-needs-auth
- `scorecard` is the structural quality snapshot, not the source of truth by itself
Fix order:
@@ -1130,6 +1132,8 @@ When `CODEX_MODE` is false, fix bugs directly.
Ship threshold:
- `verify` verdict is `PASS` or high `WARN` with 0 critical failures
- `dogfood` no longer fails because of spec parsing, binary path, or skipped examples
+- `dogfood` wiring checks pass (no unregistered commands, no config field mismatches)
+- `workflow-verify` verdict is `workflow-pass` or `unverified-needs-auth` (not `workflow-fail`)
- `scorecard` is at least 65, or meaningfully improved and no core behavior is broken
Maximum 2 shipcheck loops by default.
← 0993d60c fix(cli): add --dest flag to publish package for direct repo
·
back to Cli Printing Press
·
feat(cli): runstate isolation and lock lifecycle for paralle 10150ad3 →