[object Object]

← back to Cli Printing Press

feat(websniff): add APISpec generator and sniff CLI command

4f4dd9c44a80f0d588249acffa8717a970770609 · 2026-03-28 21:35:49 -0700 · Matt Van Horn

Wires parser + classifier + schema inference into end-to-end pipeline:
- Analyze() produces a validated APISpec from HAR/enriched captures
- Resource grouping by URL path prefix
- Auth detection from captured headers (Bearer, API key, query param)
- Name derivation from target domain
- `printing-press sniff --har <file>` CLI command with --output, --name, --blocklist flags
- Spec cached at ~/.cache/printing-press/sniff/<name>-spec.yaml

Verified end-to-end: sniff --har testdata/sniff/sample-enriched.json
produces a valid hn-algolia spec with correct endpoint, auth, and schema.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 4f4dd9c44a80f0d588249acffa8717a970770609
Author: Matt Van Horn <mvanhorn@MacBook-Pro.local>
Date:   Sat Mar 28 21:35:49 2026 -0700

    feat(websniff): add APISpec generator and sniff CLI command
    
    Wires parser + classifier + schema inference into end-to-end pipeline:
    - Analyze() produces a validated APISpec from HAR/enriched captures
    - Resource grouping by URL path prefix
    - Auth detection from captured headers (Bearer, API key, query param)
    - Name derivation from target domain
    - `printing-press sniff --har <file>` CLI command with --output, --name, --blocklist flags
    - Spec cached at ~/.cache/printing-press/sniff/<name>-spec.yaml
    
    Verified end-to-end: sniff --har testdata/sniff/sample-enriched.json
    produces a valid hn-algolia spec with correct endpoint, auth, and schema.
    
    Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 internal/cli/root.go              |   1 +
 internal/cli/sniff.go             |  77 +++++++
 internal/websniff/classifier.go   |  13 +-
 internal/websniff/specgen.go      | 461 ++++++++++++++++++++++++++++++++++++++
 internal/websniff/specgen_test.go | 112 +++++++++
 5 files changed, 660 insertions(+), 4 deletions(-)

diff --git a/internal/cli/root.go b/internal/cli/root.go
index 347ed9da..f047d5c6 100644
--- a/internal/cli/root.go
+++ b/internal/cli/root.go
@@ -45,6 +45,7 @@ func Execute() error {
 	rootCmd.AddCommand(newVisionCmd())
 	rootCmd.AddCommand(newVersionCmd())
 	rootCmd.AddCommand(newPrintCmd())
+	rootCmd.AddCommand(newSniffCmd())
 
 	return rootCmd.Execute()
 }
diff --git a/internal/cli/sniff.go b/internal/cli/sniff.go
new file mode 100644
index 00000000..14920cb6
--- /dev/null
+++ b/internal/cli/sniff.go
@@ -0,0 +1,77 @@
+package cli
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mvanhorn/cli-printing-press/internal/websniff"
+	"github.com/spf13/cobra"
+)
+
+func newSniffCmd() *cobra.Command {
+	var harPath string
+	var outputPath string
+	var name string
+	var blocklist string
+
+	cmd := &cobra.Command{
+		Use:   "sniff",
+		Short: "Analyze captured web traffic to discover API endpoints and generate a spec",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			websniff.SetAdditionalBlocklist(splitCSV(blocklist))
+
+			apiSpec, err := websniff.Analyze(harPath)
+			if err != nil {
+				return fmt.Errorf("analyzing capture: %w", err)
+			}
+
+			if name != "" {
+				apiSpec.Name = name
+				apiSpec.Config.Path = fmt.Sprintf("~/.config/%s-pp-cli/config.toml", name)
+			}
+
+			if outputPath == "" {
+				outputPath = websniff.DefaultCachePath(apiSpec.Name)
+			}
+
+			if err := websniff.WriteSpec(apiSpec, outputPath); err != nil {
+				return fmt.Errorf("writing spec: %w", err)
+			}
+
+			endpoints := 0
+			for _, resource := range apiSpec.Resources {
+				endpoints += len(resource.Endpoints)
+			}
+
+			fmt.Printf("Spec written to %s (%d endpoints across %d resources)\n", outputPath, endpoints, len(apiSpec.Resources))
+			fmt.Printf("Run 'printing-press generate --spec %s' to build the CLI\n", outputPath)
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&harPath, "har", "", "Path to HAR or enriched capture file")
+	cmd.Flags().StringVar(&outputPath, "output", "", "Output path for generated spec YAML")
+	cmd.Flags().StringVar(&name, "name", "", "Override the auto-detected API name")
+	cmd.Flags().StringVar(&blocklist, "blocklist", "", "Comma-separated additional domains to filter")
+	_ = cmd.MarkFlagRequired("har")
+
+	return cmd
+}
+
+func splitCSV(value string) []string {
+	if strings.TrimSpace(value) == "" {
+		return nil
+	}
+
+	parts := strings.Split(value, ",")
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		out = append(out, part)
+	}
+
+	return out
+}
diff --git a/internal/websniff/classifier.go b/internal/websniff/classifier.go
index 3403b020..6bcb08d4 100644
--- a/internal/websniff/classifier.go
+++ b/internal/websniff/classifier.go
@@ -15,16 +15,17 @@ type EndpointGroup struct {
 }
 
 var (
-	uuidSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
-	hashSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{32,}$`)
-	numericPattern     = regexp.MustCompile(`^\d+$`)
+	uuidSegmentPattern  = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+	hashSegmentPattern  = regexp.MustCompile(`(?i)^[0-9a-f]{32,}$`)
+	numericPattern      = regexp.MustCompile(`^\d+$`)
+	additionalBlocklist []string
 )
 
 func ClassifyEntries(entries []EnrichedEntry) (api []EnrichedEntry, noise []EnrichedEntry) {
 	api = make([]EnrichedEntry, 0, len(entries))
 	noise = make([]EnrichedEntry, 0, len(entries))
 
-	blocklist := DefaultBlocklist()
+	blocklist := append(DefaultBlocklist(), additionalBlocklist...)
 	for _, entry := range entries {
 		score := scoreEntry(entry, blocklist)
 		classified := entry
@@ -43,6 +44,10 @@ func ClassifyEntries(entries []EnrichedEntry) (api []EnrichedEntry, noise []Enri
 	return api, noise
 }
 
+func SetAdditionalBlocklist(domains []string) {
+	additionalBlocklist = append([]string(nil), domains...)
+}
+
 func DefaultBlocklist() []string {
 	return []string{
 		"google-analytics.com",
diff --git a/internal/websniff/specgen.go b/internal/websniff/specgen.go
new file mode 100644
index 00000000..bc0ddfa8
--- /dev/null
+++ b/internal/websniff/specgen.go
@@ -0,0 +1,461 @@
+package websniff
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/mvanhorn/cli-printing-press/internal/spec"
+	"gopkg.in/yaml.v3"
+)
+
+func Analyze(capturePath string) (*spec.APISpec, error) {
+	entries, targetURL, err := ParseCapture(capturePath)
+	if err != nil {
+		return nil, err
+	}
+
+	apiEntries, _ := ClassifyEntries(entries)
+	groups := DeduplicateEndpoints(apiEntries)
+
+	resources := make(map[string]spec.Resource)
+	for _, group := range groups {
+		endpoint := buildEndpoint(group)
+		resourceKey, resourceName := deriveResourceKey(group.NormalizedPath)
+		if resourceKey == "" {
+			resourceKey = "default"
+			resourceName = "default"
+		}
+
+		resource := resources[resourceKey]
+		if resource.Description == "" {
+			resource.Description = fmt.Sprintf("Operations on %s", resourceName)
+		}
+		if resource.Endpoints == nil {
+			resource.Endpoints = make(map[string]spec.Endpoint)
+		}
+
+		name := deriveEndpointName(group.Method, group.NormalizedPath)
+		if _, exists := resource.Endpoints[name]; exists {
+			name = uniqueEndpointName(resource.Endpoints, name)
+		}
+		resource.Endpoints[name] = endpoint
+		resources[resourceKey] = resource
+	}
+
+	baseURL := mostCommonBaseURL(apiEntries)
+	if baseURL == "" {
+		baseURL = normalizeBaseURL(targetURL)
+	}
+
+	nameSource := targetURL
+	if nameSource == "" {
+		nameSource = baseURL
+	}
+	name := deriveNameFromURL(nameSource)
+
+	apiSpec := &spec.APISpec{
+		Name:        name,
+		Description: fmt.Sprintf("Discovered API spec for %s", name),
+		Version:     "0.1.0",
+		BaseURL:     baseURL,
+		Auth:        detectAuth(apiEntries, name),
+		Config: spec.ConfigSpec{
+			Format: "toml",
+			Path:   fmt.Sprintf("~/.config/%s-pp-cli/config.toml", name),
+		},
+		Resources: resources,
+		Types:     map[string]spec.TypeDef{},
+	}
+
+	if err := apiSpec.Validate(); err != nil {
+		if len(apiSpec.Resources) == 0 && len(groups) > 0 {
+			apiSpec.Resources["default"] = spec.Resource{
+				Description: "Discovered operations",
+				Endpoints:   map[string]spec.Endpoint{},
+			}
+		}
+		if apiSpec.Auth.Type == "" {
+			apiSpec.Auth = spec.AuthConfig{Type: "none"}
+		}
+		if validateErr := apiSpec.Validate(); validateErr != nil {
+			return nil, fmt.Errorf("validating generated spec: %w", validateErr)
+		}
+	}
+
+	return apiSpec, nil
+}
+
+func WriteSpec(apiSpec *spec.APISpec, outputPath string) error {
+	if apiSpec == nil {
+		return fmt.Errorf("api spec is required")
+	}
+
+	data, err := yaml.Marshal(apiSpec)
+	if err != nil {
+		return fmt.Errorf("marshaling spec yaml: %w", err)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
+		return fmt.Errorf("creating output directory: %w", err)
+	}
+
+	if err := os.WriteFile(outputPath, data, 0o644); err != nil {
+		return fmt.Errorf("writing spec yaml: %w", err)
+	}
+
+	return nil
+}
+
+func DefaultCachePath(name string) string {
+	home, err := os.UserHomeDir()
+	if err != nil || home == "" {
+		return filepath.Join(".cache", "printing-press", "sniff", name+"-spec.yaml")
+	}
+
+	return filepath.Join(home, ".cache", "printing-press", "sniff", name+"-spec.yaml")
+}
+
+func buildEndpoint(group EndpointGroup) spec.Endpoint {
+	responseBodies := make([]string, 0, len(group.Entries))
+	for _, entry := range group.Entries {
+		if strings.TrimSpace(entry.ResponseBody) != "" {
+			responseBodies = append(responseBodies, entry.ResponseBody)
+		}
+	}
+
+	body := inferRequestBody(group.Entries)
+	params := inferURLParams(group.Entries, group.NormalizedPath)
+	auth := detectAuth(group.Entries, "")
+	if auth.Type == "api_key" && strings.EqualFold(auth.In, "query") && auth.Header != "" {
+		params = filterAuthQueryParam(params, auth.Header)
+	}
+
+	responseType := inferResponseType(responseBodies)
+	responseFields := InferResponseSchema(responseBodies)
+	if len(params) == 0 && len(responseFields) > 0 {
+		params = responseFields
+	}
+
+	return spec.Endpoint{
+		Method:      group.Method,
+		Path:        group.NormalizedPath,
+		Description: fmt.Sprintf("%s %s", group.Method, group.NormalizedPath),
+		Params:      params,
+		Body:        body,
+		Response: spec.ResponseDef{
+			Type: responseType,
+			Item: deriveResponseItemName(group.NormalizedPath),
+		},
+	}
+}
+
+func inferRequestBody(entries []EnrichedEntry) []spec.Param {
+	for _, entry := range entries {
+		body := strings.TrimSpace(entry.RequestBody)
+		if body == "" {
+			continue
+		}
+
+		contentType := getHeaderValue(entry.RequestHeaders, "Content-Type")
+		params := InferRequestSchema(body, contentType)
+		if len(params) > 0 {
+			return params
+		}
+	}
+
+	return nil
+}
+
+func inferURLParams(entries []EnrichedEntry, normalizedPath string) []spec.Param {
+	paramsByName := make(map[string]spec.Param)
+
+	for _, segment := range strings.Split(normalizedPath, "/") {
+		if !strings.HasPrefix(segment, "{") || !strings.HasSuffix(segment, "}") {
+			continue
+		}
+
+		name := strings.TrimSuffix(strings.TrimPrefix(segment, "{"), "}")
+		paramsByName[name] = spec.Param{
+			Name:        name,
+			Type:        "string",
+			Required:    true,
+			Positional:  true,
+			Description: fmt.Sprintf("The %s path segment", name),
+		}
+	}
+
+	for _, entry := range entries {
+		parsed, err := url.Parse(entry.URL)
+		if err != nil {
+			continue
+		}
+
+		for key, values := range parsed.Query() {
+			if _, exists := paramsByName[key]; exists {
+				continue
+			}
+
+			value := ""
+			if len(values) > 0 {
+				value = values[0]
+			}
+
+			paramsByName[key] = spec.Param{
+				Name:        key,
+				Type:        inferScalarStringType(value),
+				Required:    false,
+				Description: "",
+			}
+		}
+	}
+
+	if len(paramsByName) == 0 {
+		return nil
+	}
+
+	names := make([]string, 0, len(paramsByName))
+	for name := range paramsByName {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	params := make([]spec.Param, 0, len(names))
+	for _, name := range names {
+		params = append(params, paramsByName[name])
+	}
+
+	return params
+}
+
+func detectAuth(entries []EnrichedEntry, name string) spec.AuthConfig {
+	envPrefix := strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
+	for _, entry := range entries {
+		for headerName, value := range entry.RequestHeaders {
+			lowerHeader := strings.ToLower(headerName)
+			switch {
+			case strings.EqualFold(headerName, "Authorization") && strings.HasPrefix(strings.TrimSpace(value), "Bearer "):
+				return spec.AuthConfig{
+					Type:    "bearer_token",
+					Header:  "Authorization",
+					EnvVars: envVarsOrNil(envPrefix, "TOKEN"),
+				}
+			case strings.Contains(lowerHeader, "api-key") || strings.Contains(lowerHeader, "api_key"):
+				return spec.AuthConfig{
+					Type:    "api_key",
+					Header:  headerName,
+					In:      "header",
+					EnvVars: envVarsOrNil(envPrefix, "API_KEY"),
+				}
+			}
+		}
+
+		parsed, err := url.Parse(entry.URL)
+		if err != nil {
+			continue
+		}
+		for key := range parsed.Query() {
+			lowerKey := strings.ToLower(key)
+			if strings.Contains(lowerKey, "key") || strings.Contains(lowerKey, "token") {
+				return spec.AuthConfig{
+					Type:    "api_key",
+					Header:  key,
+					In:      "query",
+					EnvVars: envVarsOrNil(envPrefix, "API_KEY"),
+				}
+			}
+		}
+	}
+
+	return spec.AuthConfig{Type: "none"}
+}
+
+func envVarsOrNil(prefix string, suffix string) []string {
+	if prefix == "" {
+		return nil
+	}
+
+	return []string{prefix + "_" + suffix}
+}
+
+func mostCommonBaseURL(entries []EnrichedEntry) string {
+	counts := make(map[string]int)
+	best := ""
+	bestCount := 0
+
+	for _, entry := range entries {
+		base := normalizeBaseURL(entry.URL)
+		if base == "" {
+			continue
+		}
+
+		counts[base]++
+		if counts[base] > bestCount {
+			best = base
+			bestCount = counts[base]
+		}
+	}
+
+	return best
+}
+
+func normalizeBaseURL(raw string) string {
+	parsed, err := url.Parse(raw)
+	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+		return ""
+	}
+
+	return parsed.Scheme + "://" + parsed.Host
+}
+
+func deriveResourceKey(path string) (string, string) {
+	segments := significantSegments(path)
+	if len(segments) == 0 {
+		return "", ""
+	}
+
+	if len(segments) > 3 {
+		segments = segments[:3]
+	}
+
+	return strings.Join(segments, "/"), segments[len(segments)-1]
+}
+
+func significantSegments(path string) []string {
+	parts := strings.Split(path, "/")
+	segments := make([]string, 0, len(parts))
+	for _, segment := range parts {
+		segment = strings.TrimSpace(segment)
+		if segment == "" {
+			continue
+		}
+		if strings.HasPrefix(segment, "{") && strings.HasSuffix(segment, "}") {
+			continue
+		}
+		if segment == "api" || isVersionSegment(segment) {
+			continue
+		}
+		segments = append(segments, segment)
+	}
+
+	return segments
+}
+
+func isVersionSegment(segment string) bool {
+	if len(segment) < 2 || segment[0] != 'v' {
+		return false
+	}
+	for _, r := range segment[1:] {
+		if r < '0' || r > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+func deriveEndpointName(method string, normalizedPath string) string {
+	resource := "endpoint"
+	segments := significantSegments(normalizedPath)
+	if len(segments) > 0 {
+		resource = strings.ReplaceAll(segments[len(segments)-1], "-", "_")
+	}
+
+	switch strings.ToUpper(method) {
+	case "GET":
+		if strings.Contains(normalizedPath, "{") {
+			return "get_" + resource
+		}
+		return "list_" + resource
+	case "POST":
+		return "create_" + resource
+	case "PUT", "PATCH":
+		return "update_" + resource
+	case "DELETE":
+		return "delete_" + resource
+	default:
+		return strings.ToLower(method) + "_" + resource
+	}
+}
+
+func uniqueEndpointName(endpoints map[string]spec.Endpoint, base string) string {
+	for i := 2; ; i++ {
+		name := fmt.Sprintf("%s_%d", base, i)
+		if _, exists := endpoints[name]; !exists {
+			return name
+		}
+	}
+}
+
+func inferResponseType(bodies []string) string {
+	for _, body := range bodies {
+		body = strings.TrimSpace(body)
+		if body == "" {
+			continue
+		}
+
+		var value any
+		if err := json.Unmarshal([]byte(body), &value); err != nil {
+			continue
+		}
+
+		switch value.(type) {
+		case []any:
+			return "array"
+		case map[string]any:
+			return "object"
+		}
+	}
+
+	return "object"
+}
+
+func deriveResponseItemName(path string) string {
+	segments := significantSegments(path)
+	if len(segments) == 0 {
+		return "response"
+	}
+
+	return strings.ReplaceAll(segments[len(segments)-1], "-", "_")
+}
+
+func filterAuthQueryParam(params []spec.Param, authParam string) []spec.Param {
+	filtered := make([]spec.Param, 0, len(params))
+	for _, param := range params {
+		if strings.EqualFold(param.Name, authParam) {
+			continue
+		}
+		filtered = append(filtered, param)
+	}
+	return filtered
+}
+
+func deriveNameFromURL(raw string) string {
+	parsed, err := url.Parse(raw)
+	if err != nil || parsed.Hostname() == "" {
+		return "api"
+	}
+
+	host := strings.TrimPrefix(strings.ToLower(parsed.Hostname()), "www.")
+	labels := strings.Split(host, ".")
+	if len(labels) == 0 {
+		return "api"
+	}
+	if len(labels) > 2 {
+		switch labels[0] {
+		case "api", "app", "developer", "developers":
+			labels = labels[1:]
+		}
+	}
+	if len(labels) > 1 {
+		labels = labels[:len(labels)-1]
+	}
+	if len(labels) == 0 {
+		return "api"
+	}
+
+	return strings.Join(labels, "-")
+}
diff --git a/internal/websniff/specgen_test.go b/internal/websniff/specgen_test.go
new file mode 100644
index 00000000..797597b6
--- /dev/null
+++ b/internal/websniff/specgen_test.go
@@ -0,0 +1,112 @@
+package websniff
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mvanhorn/cli-printing-press/internal/spec"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAnalyze(t *testing.T) {
+	t.Parallel()
+
+	apiSpec, err := Analyze(filepath.Join("..", "..", "testdata", "sniff", "sample-enriched.json"))
+	require.NoError(t, err)
+	require.NotNil(t, apiSpec)
+
+	assert.Equal(t, "hn-algolia", apiSpec.Name)
+	require.NotEmpty(t, apiSpec.Resources)
+
+	foundEndpointWithParams := false
+	for _, resource := range apiSpec.Resources {
+		require.NotEmpty(t, resource.Endpoints)
+		for _, endpoint := range resource.Endpoints {
+			if len(endpoint.Params) > 0 {
+				foundEndpointWithParams = true
+			}
+		}
+	}
+
+	assert.True(t, foundEndpointWithParams)
+	assert.NoError(t, apiSpec.Validate())
+}
+
+func TestWriteSpec(t *testing.T) {
+	t.Parallel()
+
+	apiSpec := &spec.APISpec{
+		Name:        "example",
+		Description: "Example API",
+		Version:     "0.1.0",
+		BaseURL:     "https://api.example.com",
+		Auth:        spec.AuthConfig{Type: "none"},
+		Config: spec.ConfigSpec{
+			Format: "toml",
+			Path:   "~/.config/example-pp-cli/config.toml",
+		},
+		Resources: map[string]spec.Resource{
+			"widgets": {
+				Description: "Operations on widgets",
+				Endpoints: map[string]spec.Endpoint{
+					"list_widgets": {
+						Method: "GET",
+						Path:   "/widgets",
+						Response: spec.ResponseDef{
+							Type: "object",
+							Item: "widgets",
+						},
+					},
+				},
+			},
+		},
+		Types: map[string]spec.TypeDef{},
+	}
+
+	outputPath := filepath.Join(t.TempDir(), "nested", "spec.yaml")
+	err := WriteSpec(apiSpec, outputPath)
+	require.NoError(t, err)
+
+	data, err := os.ReadFile(outputPath)
+	require.NoError(t, err)
+
+	parsed, err := spec.ParseBytes(data)
+	require.NoError(t, err)
+	assert.Equal(t, apiSpec.Name, parsed.Name)
+	assert.Equal(t, apiSpec.BaseURL, parsed.BaseURL)
+}
+
+func TestDeriveNameFromURL(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		raw  string
+		want string
+	}{
+		{
+			name: "strips www and tld",
+			raw:  "https://www.youtube.com",
+			want: "youtube",
+		},
+		{
+			name: "keeps meaningful subdomain",
+			raw:  "https://hn.algolia.com",
+			want: "hn-algolia",
+		},
+		{
+			name: "drops generic api prefix",
+			raw:  "https://api.example.com",
+			want: "example",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equal(t, tt.want, deriveNameFromURL(tt.raw))
+		})
+	}
+}

← f9b8e141 feat(websniff): add JSON schema inference from captured payl  ·  back to Cli Printing Press  ·  refactor(cli): decouple skills and binary from repo checkout dfc0eedc →