← back to Cli Printing Press

fix(cli): scorer respects runtime MCP surface selection (refs #516 WU-A4) (#519)

358e46a67a8e961d6096f0290d32429fd6cd2960 · 2026-05-02 19:14:46 -0700 · Trevin Chow

Pre-fix: estimateMCPTokens unconditionally read internal/mcp/tools.go.
For CLIs that opt into mcp.orchestration: code AND default
DUB_MCP_SURFACE (or equivalent) to "thin" in main.go, the agent loads
internal/mcp/code_orch.go's 2-tool surface — NOT tools.go's typed
endpoint mirror. The scorer was reporting mcp_token_efficiency 4/10
on Dub despite the runtime catalog being ~1K tokens (2 short tools)
instead of ~30K (53 verbose tools).

Adds canonicalMCPSurfacePath(dir) which:
  - returns code_orch.go when it exists AND main.go's surface-selection
    default is "thin"
  - falls back to tools.go in every other case (no regression for CLIs
    that don't opt into orchestration)

Stub detection extended to accept both RegisterTools and
RegisterCodeOrchestrationTools.

Real-world impact (verified):
  - dub:        91→92  (mcp_token_efficiency 4/10 → 7/10)
  - open-meteo: 82→82  (unchanged — no code_orch.go)
  - craigslist: 85→85  (unchanged — no code_orch.go)

Refs #516 (umbrella). The other 3 sub-WUs in #516 do NOT match the
umbrella's framing and are addressed separately on the issue.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files touched

• M internal/pipeline/mcp_size.go
• M internal/pipeline/mcp_size_test.go

Diff

commit 358e46a67a8e961d6096f0290d32429fd6cd2960
Author: Trevin Chow <trevin@trevinchow.com>
Date:   Sat May 2 19:14:46 2026 -0700

    fix(cli): scorer respects runtime MCP surface selection (refs #516 WU-A4) (#519)
    
    Pre-fix: estimateMCPTokens unconditionally read internal/mcp/tools.go.
    For CLIs that opt into mcp.orchestration: code AND default
    DUB_MCP_SURFACE (or equivalent) to "thin" in main.go, the agent loads
    internal/mcp/code_orch.go's 2-tool surface — NOT tools.go's typed
    endpoint mirror. The scorer was reporting mcp_token_efficiency 4/10
    on Dub despite the runtime catalog being ~1K tokens (2 short tools)
    instead of ~30K (53 verbose tools).
    
    Adds canonicalMCPSurfacePath(dir) which:
      - returns code_orch.go when it exists AND main.go's surface-selection
        default is "thin"
      - falls back to tools.go in every other case (no regression for CLIs
        that don't opt into orchestration)
    
    Stub detection extended to accept both RegisterTools and
    RegisterCodeOrchestrationTools.
    
    Real-world impact (verified):
      - dub:        91→92  (mcp_token_efficiency 4/10 → 7/10)
      - open-meteo: 82→82  (unchanged — no code_orch.go)
      - craigslist: 85→85  (unchanged — no code_orch.go)
    
    Refs #516 (umbrella). The other 3 sub-WUs in #516 do NOT match the
    umbrella's framing and are addressed separately on the issue.
    
    Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 internal/pipeline/mcp_size.go      |  67 +++++++++++++++++-
 internal/pipeline/mcp_size_test.go | 135 +++++++++++++++++++++++++++++++++++++
 2 files changed, 200 insertions(+), 2 deletions(-)

diff --git a/internal/pipeline/mcp_size.go b/internal/pipeline/mcp_size.go
index 83f2ed15..14db3806 100644
--- a/internal/pipeline/mcp_size.go
+++ b/internal/pipeline/mcp_size.go
@@ -35,17 +35,80 @@ func mcpToolsPath(dir string) string {
 	return filepath.Join(dir, "internal", "mcp", "tools.go")
 }
 
+// mcpCodeOrchPath is the conventional location of the code-orchestration
+// thin-surface tool registrations (search + execute pair) in a printed CLI
+// that opted into mcp.orchestration: code.
+func mcpCodeOrchPath(dir string) string {
+	return filepath.Join(dir, "internal", "mcp", "code_orch.go")
+}
+
+// runtimeSurfaceDefault matches the default branch of `<API>_MCP_SURFACE`
+// selection logic in a printed CLI's MCP main.go. The API prefix varies
+// per CLI (DUB_, NOTION_, etc.) but the env-var suffix is stable. Captures
+// the literal default value (typically "thin", "full", or "both").
+var runtimeSurfaceDefault = regexp.MustCompile(`os\.Getenv\("[A-Z0-9_]+_MCP_SURFACE"\)[\s\S]{0,200}?surface\s*=\s*"([^"]+)"`)
+
+// canonicalMCPSurfacePath returns the file the scorer should read for token
+// efficiency: the file containing the tool registrations the agent actually
+// loads under the runtime default surface.
+//
+// When the printed CLI opts into mcp.orchestration: code AND its main.go
+// defaults DUB_MCP_SURFACE (or equivalent) to "thin", the agent loads
+// internal/mcp/code_orch.go (the search+execute pair), NOT
+// internal/mcp/tools.go (the typed endpoint mirrors). The scorer should
+// reflect that.
+//
+// Falls back to internal/mcp/tools.go when:
+//   - code_orch.go does not exist (CLI didn't opt into orchestration),
+//   - main.go has no surface-selection logic (older CLIs with no env var),
+//   - default surface is "full" or "both" (tools.go IS the agent surface).
+//
+// This is the architectural fix for retro umbrella issue #516 WU-A4: the
+// scorer must evaluate what the agent sees, not what the static templates
+// emit. Two surfaces co-exist in code_orch CLIs; the runtime default
+// dictates which one drives the catalog tax.
+func canonicalMCPSurfacePath(dir string) string {
+	toolsPath := mcpToolsPath(dir)
+	codeOrchPath := mcpCodeOrchPath(dir)
+
+	if _, err := os.Stat(codeOrchPath); err != nil {
+		return toolsPath
+	}
+	mainPath := mcpMainPath(dir)
+	if mainPath == "" {
+		return toolsPath
+	}
+	mainSrc, err := os.ReadFile(mainPath)
+	if err != nil {
+		return toolsPath
+	}
+	match := runtimeSurfaceDefault.FindStringSubmatch(string(mainSrc))
+	if len(match) < 2 {
+		return toolsPath
+	}
+	if match[1] == "thin" {
+		return codeOrchPath
+	}
+	return toolsPath
+}
+
 // estimateMCPTokens reads the generated MCP tool surface and returns an
 // estimate of its total token weight, plus per-tool breakdown when
 // individual tool definitions can be isolated. Returns a zero-valued
 // estimate (TotalTokens == 0) if no MCP surface exists.
+//
+// Reads canonicalMCPSurfacePath, which respects runtime surface selection
+// in main.go (e.g., DUB_MCP_SURFACE defaulting to "thin" loads
+// code_orch.go's 2-tool surface, not tools.go's 53-tool endpoint mirror).
+// Pre-#516 behavior was unconditional read of tools.go; that mis-scored
+// any CLI whose default surface was the thin pair.
 func estimateMCPTokens(dir string) MCPTokenEstimate {
-	content, err := os.ReadFile(mcpToolsPath(dir))
+	content, err := os.ReadFile(canonicalMCPSurfacePath(dir))
 	if err != nil {
 		return MCPTokenEstimate{}
 	}
 	src := string(content)
-	if !strings.Contains(src, "RegisterTools") {
+	if !strings.Contains(src, "RegisterTools") && !strings.Contains(src, "RegisterCodeOrchestrationTools") {
 		// File exists but is a stub — treat as no MCP surface.
 		return MCPTokenEstimate{}
 	}
diff --git a/internal/pipeline/mcp_size_test.go b/internal/pipeline/mcp_size_test.go
index a62b2e8f..2a3cb72e 100644
--- a/internal/pipeline/mcp_size_test.go
+++ b/internal/pipeline/mcp_size_test.go
@@ -118,3 +118,138 @@ func TestScoreMCPTokenEfficiency_PartialCreditMedium(t *testing.T) {
 	assert.True(t, scored)
 	assert.Equal(t, 7, score)
 }
+
+// TestEstimateMCPTokens_RuntimeSurfaceSelection captures WU-A4 from
+// retro umbrella #516: when the MCP main.go has surface-selection logic
+// defaulting to a thin code-orchestration surface, the agent loads the
+// thin surface (e.g., dub_search + dub_execute), NOT the full
+// endpoint-mirror in tools.go. The token-efficiency score must reflect
+// the actual default surface, not the static tools.go.
+func TestEstimateMCPTokens_RuntimeSurfaceSelection(t *testing.T) {
+	t.Run("thin default selects code_orch.go for token counting", func(t *testing.T) {
+		dir := t.TempDir()
+		mcpDir := filepath.Join(dir, "internal", "mcp")
+		require.NoError(t, os.MkdirAll(mcpDir, 0o755))
+
+		// Write a "tools.go" with 3 verbose endpoint mirrors that would
+		// score badly on token efficiency if read directly.
+		toolsBody := `package mcp
+
+import mcplib "github.com/mark3labs/mcp-go/mcp"
+
+func RegisterTools(s *server.MCPServer) {
+	s.AddTool(mcplib.NewTool("verbose_endpoint_one", mcplib.WithDescription("` + strings.Repeat("x", 400) + `")), nil)
+	s.AddTool(mcplib.NewTool("verbose_endpoint_two", mcplib.WithDescription("` + strings.Repeat("y", 400) + `")), nil)
+	s.AddTool(mcplib.NewTool("verbose_endpoint_three", mcplib.WithDescription("` + strings.Repeat("z", 400) + `")), nil)
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(mcpDir, "tools.go"), []byte(toolsBody), 0o644))
+
+		// Write a "code_orch.go" with the thin 2-tool surface — short descriptions.
+		codeOrchBody := `package mcp
+
+import mcplib "github.com/mark3labs/mcp-go/mcp"
+
+func RegisterCodeOrchestrationTools(s *server.MCPServer) {
+	s.AddTool(mcplib.NewTool("dub_search", mcplib.WithDescription("Search the API.")), nil)
+	s.AddTool(mcplib.NewTool("dub_execute", mcplib.WithDescription("Execute one endpoint.")), nil)
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(mcpDir, "code_orch.go"), []byte(codeOrchBody), 0o644))
+
+		// Write a "main.go" that defaults DUB_MCP_SURFACE to "thin".
+		cmdDir := filepath.Join(dir, "cmd", "demo-pp-mcp")
+		require.NoError(t, os.MkdirAll(cmdDir, 0o755))
+		mainBody := `package main
+
+import (
+	"os"
+	mcptools "demo-pp-cli/internal/mcp"
+	"github.com/mark3labs/mcp-go/server"
+)
+
+func main() {
+	s := server.NewMCPServer("Demo", "1.0.0")
+	surface := os.Getenv("DUB_MCP_SURFACE")
+	if surface == "" {
+		surface = "thin"
+	}
+	switch surface {
+	case "thin":
+		mcptools.RegisterCodeOrchestrationTools(s)
+	case "full":
+		mcptools.RegisterTools(s)
+	}
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(cmdDir, "main.go"), []byte(mainBody), 0o644))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, ".printing-press.json"),
+			[]byte(`{"cli_name":"demo-pp-cli"}`), 0o644))
+
+		est := estimateMCPTokens(dir)
+		// EXPECT: scorer sees the thin surface (2 tools, short descriptions),
+		// NOT the full surface (3 tools with 400-char descriptions each).
+		require.Equal(t, 2, est.ToolCount, "should count code_orch.go's 2 tools, not tools.go's 3")
+		assert.Less(t, est.TotalTokens, 100, "thin surface tools have short descriptions; should be well under 100 total tokens")
+	})
+
+	t.Run("no surface selection falls through to tools.go (no regression)", func(t *testing.T) {
+		// Existing behavior preserved when there's no main.go surface
+		// selection: just read tools.go.
+		dir := writeMCPTools(t, `
+	s.AddTool(mcplib.NewTool("get_thing", mcplib.WithDescription("Retrieve a thing.")), nil)
+`)
+		est := estimateMCPTokens(dir)
+		require.Equal(t, 1, est.ToolCount)
+		assert.Equal(t, "get_thing", est.PerTool[0].Name)
+	})
+
+	t.Run("full default selects tools.go (DUB_MCP_SURFACE=full not thin)", func(t *testing.T) {
+		dir := t.TempDir()
+		mcpDir := filepath.Join(dir, "internal", "mcp")
+		require.NoError(t, os.MkdirAll(mcpDir, 0o755))
+
+		// tools.go has 1 tool (chosen by default since surface=full)
+		toolsBody := `package mcp
+
+import mcplib "github.com/mark3labs/mcp-go/mcp"
+
+func RegisterTools(s *server.MCPServer) {
+	s.AddTool(mcplib.NewTool("typed_tool", mcplib.WithDescription("Typed.")), nil)
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(mcpDir, "tools.go"), []byte(toolsBody), 0o644))
+
+		// code_orch.go exists but main.go default is "full"
+		codeOrchBody := `package mcp
+
+func RegisterCodeOrchestrationTools(s *server.MCPServer) {
+	s.AddTool(mcplib.NewTool("thin_search", mcplib.WithDescription("S.")), nil)
+	s.AddTool(mcplib.NewTool("thin_execute", mcplib.WithDescription("E.")), nil)
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(mcpDir, "code_orch.go"), []byte(codeOrchBody), 0o644))
+
+		cmdDir := filepath.Join(dir, "cmd", "demo-pp-mcp")
+		require.NoError(t, os.MkdirAll(cmdDir, 0o755))
+		mainBody := `package main
+
+import "os"
+
+func main() {
+	surface := os.Getenv("DUB_MCP_SURFACE")
+	if surface == "" {
+		surface = "full"
+	}
+	_ = surface
+}
+`
+		require.NoError(t, os.WriteFile(filepath.Join(cmdDir, "main.go"), []byte(mainBody), 0o644))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, ".printing-press.json"),
+			[]byte(`{"cli_name":"demo-pp-cli"}`), 0o644))
+
+		est := estimateMCPTokens(dir)
+		require.Equal(t, 1, est.ToolCount, "full default → tools.go")
+		assert.Equal(t, "typed_tool", est.PerTool[0].Name)
+	})
+}

← 83df8ae7 chore(main): release 3.6.1 (#512)  ·  back to Cli Printing Press  ·  fix(cli): MCP template ports — NoCache=true + codeOrch stopw 734e00d1 →