← back to Cli Printing Press
feat(cli): kind: synthetic spec attribute for multi-source CLIs — closes #203 (#209)
caa283ed147a40946751f81a9889ad9d32a1bf9b · 2026-04-13 13:56:16 -0700 · Trevin Chow
Files touched
M AGENTS.mdM internal/pipeline/dogfood.goM internal/pipeline/scorecard.goM internal/pipeline/spec_detect.goA internal/pipeline/synthetic_spec_test.goM internal/spec/spec.go
Diff
commit caa283ed147a40946751f81a9889ad9d32a1bf9b
Author: Trevin Chow <trevin@trevinchow.com>
Date: Mon Apr 13 13:56:16 2026 -0700
feat(cli): kind: synthetic spec attribute for multi-source CLIs — closes #203 (#209)
---
AGENTS.md | 2 +-
internal/pipeline/dogfood.go | 19 +++-
internal/pipeline/scorecard.go | 18 +++-
internal/pipeline/spec_detect.go | 2 +
internal/pipeline/synthetic_spec_test.go | 167 +++++++++++++++++++++++++++++++
internal/spec/spec.go | 17 ++++
6 files changed, 220 insertions(+), 5 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index c1a4e304..4ae341d2 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -56,7 +56,7 @@ Key terms used throughout this repo. Several have overloaded meanings — the gl
| **the printing press** / **the machine** | This repo's generator system — the Go binary, templates, skills, and catalog that together produce CLIs. |
| **printed CLI** / **`<api>-pp-cli`** | A CLI produced by the printing press (e.g., `notion-pp-cli`). The `-pp-` infix avoids collisions with official vendor CLIs. When someone says "the CLI" without qualification, they almost always mean a printed CLI. Use "printed CLI" in your responses to keep it clear. |
| **the printing-press binary** | The Go binary built from `cmd/printing-press/`. Commands: `generate`, `verify`, `emboss`, `scorecard`, `publish`, etc. Always say "printing-press binary" or "generator binary" — never just "the CLI" — when referring to this. |
-| **spec** | The API contract that drives generation — OpenAPI 3.0+ YAML/JSON, GraphQL SDL, or internal YAML format. Can come from catalog, URL, local file, or sniff discovery. |
+| **spec** | The API contract that drives generation — OpenAPI 3.0+ YAML/JSON, GraphQL SDL, or internal YAML format. Can come from catalog, URL, local file, or sniff discovery. Internal YAML specs may set `kind: synthetic` to declare a multi-source CLI where hand-built commands intentionally go beyond the spec; dogfood marks path-validity as skipped and scorecard excludes it from the tier-2 denominator. |
| **API slug** | Normalized API name derived from the spec title via `cleanSpecName()`. Directory key in manuscripts (`manuscripts/<api-slug>/`). The CLI name is `<api-slug>-pp-cli`. Distinct from the CLI name — don't use them interchangeably. |
| **brief** | The output of the machine's research phase (Phase 1) — a condensed doc covering API identity, competitors, data layer, and product thesis. Stored in `manuscripts/<api>/<run>/research/`. Drives all downstream decisions. |
| **crowd sniff** / **sniff** | Discovery technique that scrapes npm, PyPI, and GitHub for unofficial API clients to learn undocumented endpoints, auth patterns, and rate limits. Produces a `discovery/` manuscript with sniff-report.md, HAR captures, and unique-paths.txt. Used when no official spec exists or to supplement one. |
diff --git a/internal/pipeline/dogfood.go b/internal/pipeline/dogfood.go
index 6cbf43c9..bf54849a 100644
--- a/internal/pipeline/dogfood.go
+++ b/internal/pipeline/dogfood.go
@@ -47,6 +47,8 @@ type PathCheckResult struct {
Valid int `json:"valid"`
Invalid []string `json:"invalid,omitempty"`
Pct int `json:"valid_pct"`
+ Skipped bool `json:"skipped,omitempty"`
+ Detail string `json:"detail,omitempty"`
}
type AuthCheckResult struct {
@@ -109,6 +111,11 @@ type WorkflowCompleteResult struct {
type openAPISpec struct {
Paths []string
Auth apispec.AuthConfig
+ Kind string // see apispec.KindREST / apispec.KindSynthetic
+}
+
+func (s *openAPISpec) IsSynthetic() bool {
+ return s != nil && s.Kind == apispec.KindSynthetic
}
func RunDogfood(dir, specPath string, opts ...DogfoodOption) (*DogfoodReport, error) {
@@ -131,7 +138,17 @@ func RunDogfood(dir, specPath string, opts ...DogfoodOption) (*DogfoodReport, er
}
spec = loaded
- report.PathCheck = checkPaths(dir, spec.Paths)
+ if spec.IsSynthetic() {
+ // Synthetic CLIs intentionally go beyond the spec; strict
+ // path-validity would flag every hand-built command. Record as
+ // skipped (not as a misleading 100% pass).
+ report.PathCheck = PathCheckResult{
+ Skipped: true,
+ Detail: "synthetic spec: path validity not applicable",
+ }
+ } else {
+ report.PathCheck = checkPaths(dir, spec.Paths)
+ }
report.AuthCheck = checkAuth(dir, spec.Auth)
} else {
report.AuthCheck = AuthCheckResult{
diff --git a/internal/pipeline/scorecard.go b/internal/pipeline/scorecard.go
index 9ec15ce0..22b4c81f 100644
--- a/internal/pipeline/scorecard.go
+++ b/internal/pipeline/scorecard.go
@@ -102,10 +102,17 @@ func RunScorecard(outputDir, pipelineDir, specPath string, verifyReport *VerifyR
return nil, err
}
- pathValidity := evaluatePathValidity(outputDir, spec)
- sc.Steinberger.PathValidity = pathValidity.score
- if !pathValidity.scored {
+ if spec.IsSynthetic() {
+ // Hand-built commands intentionally go beyond the spec; path-validity
+ // is not applicable. Mark unscored so the tier-2 denominator excludes
+ // it rather than awarding a 10-point cushion the CLI didn't earn.
sc.UnscoredDimensions = append(sc.UnscoredDimensions, "path_validity")
+ } else {
+ pathValidity := evaluatePathValidity(outputDir, spec)
+ sc.Steinberger.PathValidity = pathValidity.score
+ if !pathValidity.scored {
+ sc.UnscoredDimensions = append(sc.UnscoredDimensions, "path_validity")
+ }
}
authProtocol := evaluateAuthProtocol(outputDir, spec)
@@ -1074,6 +1081,11 @@ type openAPISpecInfo struct {
Paths []string
SecuritySchemes map[string]openAPISecurityScheme
SecurityRequirements []securityRequirementSet
+ Kind string // see apispec.KindREST / apispec.KindSynthetic
+}
+
+func (s *openAPISpecInfo) IsSynthetic() bool {
+ return s != nil && s.Kind == apispec.KindSynthetic
}
func loadOpenAPISpec(specPath string) (*openAPISpecInfo, error) {
diff --git a/internal/pipeline/spec_detect.go b/internal/pipeline/spec_detect.go
index 7a5d9413..83a233fe 100644
--- a/internal/pipeline/spec_detect.go
+++ b/internal/pipeline/spec_detect.go
@@ -42,6 +42,7 @@ func internalSpecToDogfoodSpec(s *apispec.APISpec) *openAPISpec {
return &openAPISpec{
Paths: collectInternalSpecPaths(s),
Auth: s.Auth,
+ Kind: s.Kind,
}
}
@@ -51,6 +52,7 @@ func internalSpecToOpenAPISpecInfo(s *apispec.APISpec) *openAPISpecInfo {
info := &openAPISpecInfo{
Paths: collectInternalSpecPaths(s),
SecuritySchemes: make(map[string]openAPISecurityScheme),
+ Kind: s.Kind,
}
// Map auth config to a synthetic security scheme so scorecard auth
diff --git a/internal/pipeline/synthetic_spec_test.go b/internal/pipeline/synthetic_spec_test.go
new file mode 100644
index 00000000..bb91b04a
--- /dev/null
+++ b/internal/pipeline/synthetic_spec_test.go
@@ -0,0 +1,167 @@
+package pipeline
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestSyntheticSpec_DogfoodSkipsPathCheck verifies that specs declared with
+// `kind: synthetic` bypass the strict path-validity check in dogfood. This is
+// the fix for combo/cross-site CLIs (e.g., Recipe GOAT) where the spec
+// describes a primary API but the CLI intentionally adds hand-built commands
+// that would otherwise trigger "not in spec" false failures.
+// Regression guard for #203.
+func TestSyntheticSpec_DogfoodSkipsPathCheck(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+
+ // A CLI command whose path intentionally does NOT appear in the spec.
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "recipes_search.go"), `package cli
+func recipesSearch() {
+ path := "/unrelated/endpoint"
+ _ = path
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "root.go"), `package cli
+func main() {}
+`)
+
+ // Synthetic spec: primary endpoint is /widgets but the CLI surface is broader.
+ specPath := filepath.Join(dir, "spec.yaml")
+ require.NoError(t, os.WriteFile(specPath, []byte(`name: recipe-goat
+version: "0.1.0"
+kind: synthetic
+base_url: "https://api.example.com"
+auth:
+ type: api_key
+ header: Authorization
+ format: "Bearer {token}"
+ env_vars: [RECIPE_GOAT_TOKEN]
+config:
+ format: toml
+ path: "~/.config/recipe-goat-pp-cli/config.toml"
+resources:
+ widgets:
+ description: "Widgets"
+ endpoints:
+ list:
+ method: GET
+ path: "/widgets"
+ description: "List widgets"
+`), 0o644))
+
+ report, err := RunDogfood(dir, specPath)
+ require.NoError(t, err)
+
+ require.True(t, report.PathCheck.Skipped,
+ "synthetic spec should mark path check as skipped")
+ require.Contains(t, report.PathCheck.Detail, "synthetic",
+ "skip reason should name the cause")
+ require.Zero(t, report.PathCheck.Tested,
+ "skipped check should not report Tested > 0")
+ require.Empty(t, report.PathCheck.Invalid)
+}
+
+// TestRESTSpec_DogfoodStillChecksPaths ensures the default (non-synthetic)
+// behavior is unchanged: specs without `kind: synthetic` still get the strict
+// path-validity check. Guards against over-broad application of the skip.
+func TestRESTSpec_DogfoodStillChecksPaths(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "widgets_list.go"), `package cli
+func widgetsList() {
+ path := "/widgets"
+ _ = path
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "root.go"), `package cli
+func main() {}
+`)
+
+ specPath := filepath.Join(dir, "spec.yaml")
+ // No kind field — defaults to rest behavior.
+ require.NoError(t, os.WriteFile(specPath, []byte(`name: regular-api
+version: "0.1.0"
+base_url: "https://api.example.com"
+auth:
+ type: api_key
+ header: Authorization
+ format: "Bearer {token}"
+ env_vars: [REGULAR_API_TOKEN]
+config:
+ format: toml
+ path: "~/.config/regular-api-pp-cli/config.toml"
+resources:
+ widgets:
+ description: "Widgets"
+ endpoints:
+ list:
+ method: GET
+ path: "/widgets"
+ description: "List widgets"
+`), 0o644))
+
+ report, err := RunDogfood(dir, specPath)
+ require.NoError(t, err)
+
+ // Default rest spec should run the path check and find the /widgets path.
+ require.Greater(t, report.PathCheck.Tested, 0,
+ "rest spec should run the path check (Tested > 0)")
+}
+
+// TestSyntheticSpec_ScorecardMarksPathValidityUnscored verifies scorecard
+// excludes PathValidity from the tier-2 denominator for synthetic specs
+// rather than awarding a free 10-point cushion. Regression guard for #203.
+func TestSyntheticSpec_ScorecardMarksPathValidityUnscored(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal", "cli"), 0o755))
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "recipes_search.go"), `package cli
+func recipesSearch() {
+ path := "/unrelated/endpoint"
+ _ = path
+}
+`)
+ writeTestFile(t, filepath.Join(dir, "internal", "cli", "root.go"), `package cli
+func main() {}
+`)
+
+ specPath := filepath.Join(dir, "spec.yaml")
+ require.NoError(t, os.WriteFile(specPath, []byte(`name: recipe-goat
+version: "0.1.0"
+kind: synthetic
+base_url: "https://api.example.com"
+auth:
+ type: api_key
+ header: Authorization
+ format: "Bearer {token}"
+ env_vars: [RECIPE_GOAT_TOKEN]
+config:
+ format: toml
+ path: "~/.config/recipe-goat-pp-cli/config.toml"
+resources:
+ widgets:
+ description: "Widgets"
+ endpoints:
+ list:
+ method: GET
+ path: "/widgets"
+ description: "List widgets"
+`), 0o644))
+
+ sc, err := RunScorecard(dir, t.TempDir(), specPath, nil)
+ require.NoError(t, err)
+ require.True(t, sc.IsDimensionUnscored("path_validity"),
+ "synthetic spec should mark path_validity as unscored so the tier-2 denominator excludes it")
+ require.Zero(t, sc.Steinberger.PathValidity,
+ "unscored dimension should not carry a score")
+}
diff --git a/internal/spec/spec.go b/internal/spec/spec.go
index d389dd9e..ce2e7007 100644
--- a/internal/spec/spec.go
+++ b/internal/spec/spec.go
@@ -9,6 +9,15 @@ import (
"gopkg.in/yaml.v3"
)
+// Valid values for APISpec.Kind. A bare string with no const was the
+// established convention for sibling fields (SpecSource, ClientPattern), but
+// Kind is compared in production code at multiple sites, so the constant
+// prevents typos from silently falling through to the default-rest path.
+const (
+ KindREST = "rest" // default; strict path-validity against the spec
+ KindSynthetic = "synthetic" // multi-source / combo CLI; dogfood + scorecard relax path-validity
+)
+
type APISpec struct {
Name string `yaml:"name" json:"name"`
Description string `yaml:"description" json:"description"`
@@ -16,6 +25,7 @@ type APISpec struct {
BaseURL string `yaml:"base_url" json:"base_url"`
BasePath string `yaml:"base_path,omitempty" json:"base_path,omitempty"`
Owner string `yaml:"owner,omitempty" json:"owner,omitempty"` // GitHub owner for import paths and Homebrew tap
+ Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` // "rest" (default) or "synthetic" — synthetic CLIs aggregate multiple sources beyond the spec; dogfood's path-validity check is relaxed accordingly
SpecSource string `yaml:"spec_source,omitempty" json:"spec_source,omitempty"` // official, community, sniffed, docs — affects generated client defaults
ClientPattern string `yaml:"client_pattern,omitempty" json:"client_pattern,omitempty"` // rest (default), proxy-envelope — affects generated HTTP client
ProxyRoutes map[string]string `yaml:"proxy_routes,omitempty" json:"proxy_routes,omitempty"` // path prefix → service name for proxy-envelope routing
@@ -28,6 +38,13 @@ type APISpec struct {
Types map[string]TypeDef `yaml:"types" json:"types"`
}
+// IsSynthetic reports whether this spec declares a multi-source / combo CLI
+// where hand-built commands intentionally go beyond the spec. Dogfood skips
+// strict path-validity and scorecard marks path_validity as unscored.
+func (s *APISpec) IsSynthetic() bool {
+ return s != nil && s.Kind == KindSynthetic
+}
+
// RequiredHeader represents a non-auth header that the API requires on most
// requests (e.g., cal-api-version, Stripe-Version, anthropic-version).
// Detected automatically from OpenAPI specs when a required header parameter
← 26bc9057 feat(cli): enum validation for params declared with enum con
·
back to Cli Printing Press
·
feat(cli): scorecard --live-check samples novel-feature exam df242da4 →