← back to Cli Printing Press
feat(websniff): add APISpec generator and sniff CLI command
4f4dd9c44a80f0d588249acffa8717a970770609 · 2026-03-28 21:35:49 -0700 · Matt Van Horn
Wires parser + classifier + schema inference into end-to-end pipeline:
- Analyze() produces a validated APISpec from HAR/enriched captures
- Resource grouping by URL path prefix
- Auth detection from captured headers (Bearer, API key, query param)
- Name derivation from target domain
- `printing-press sniff --har <file>` CLI command with --output, --name, --blocklist flags
- Spec cached at ~/.cache/printing-press/sniff/<name>-spec.yaml
Verified end-to-end: sniff --har testdata/sniff/sample-enriched.json
produces a valid hn-algolia spec with correct endpoint, auth, and schema.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
M internal/cli/root.goA internal/cli/sniff.goM internal/websniff/classifier.goA internal/websniff/specgen.goA internal/websniff/specgen_test.go
Diff
commit 4f4dd9c44a80f0d588249acffa8717a970770609
Author: Matt Van Horn <mvanhorn@MacBook-Pro.local>
Date: Sat Mar 28 21:35:49 2026 -0700
feat(websniff): add APISpec generator and sniff CLI command
Wires parser + classifier + schema inference into end-to-end pipeline:
- Analyze() produces a validated APISpec from HAR/enriched captures
- Resource grouping by URL path prefix
- Auth detection from captured headers (Bearer, API key, query param)
- Name derivation from target domain
- `printing-press sniff --har <file>` CLI command with --output, --name, --blocklist flags
- Spec cached at ~/.cache/printing-press/sniff/<name>-spec.yaml
Verified end-to-end: sniff --har testdata/sniff/sample-enriched.json
produces a valid hn-algolia spec with correct endpoint, auth, and schema.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/cli/root.go | 1 +
internal/cli/sniff.go | 77 +++++++
internal/websniff/classifier.go | 13 +-
internal/websniff/specgen.go | 461 ++++++++++++++++++++++++++++++++++++++
internal/websniff/specgen_test.go | 112 +++++++++
5 files changed, 660 insertions(+), 4 deletions(-)
diff --git a/internal/cli/root.go b/internal/cli/root.go
index 347ed9da..f047d5c6 100644
--- a/internal/cli/root.go
+++ b/internal/cli/root.go
@@ -45,6 +45,7 @@ func Execute() error {
rootCmd.AddCommand(newVisionCmd())
rootCmd.AddCommand(newVersionCmd())
rootCmd.AddCommand(newPrintCmd())
+ rootCmd.AddCommand(newSniffCmd())
return rootCmd.Execute()
}
diff --git a/internal/cli/sniff.go b/internal/cli/sniff.go
new file mode 100644
index 00000000..14920cb6
--- /dev/null
+++ b/internal/cli/sniff.go
@@ -0,0 +1,77 @@
+package cli
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/mvanhorn/cli-printing-press/internal/websniff"
+ "github.com/spf13/cobra"
+)
+
+func newSniffCmd() *cobra.Command {
+ var harPath string
+ var outputPath string
+ var name string
+ var blocklist string
+
+ cmd := &cobra.Command{
+ Use: "sniff",
+ Short: "Analyze captured web traffic to discover API endpoints and generate a spec",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ websniff.SetAdditionalBlocklist(splitCSV(blocklist))
+
+ apiSpec, err := websniff.Analyze(harPath)
+ if err != nil {
+ return fmt.Errorf("analyzing capture: %w", err)
+ }
+
+ if name != "" {
+ apiSpec.Name = name
+ apiSpec.Config.Path = fmt.Sprintf("~/.config/%s-pp-cli/config.toml", name)
+ }
+
+ if outputPath == "" {
+ outputPath = websniff.DefaultCachePath(apiSpec.Name)
+ }
+
+ if err := websniff.WriteSpec(apiSpec, outputPath); err != nil {
+ return fmt.Errorf("writing spec: %w", err)
+ }
+
+ endpoints := 0
+ for _, resource := range apiSpec.Resources {
+ endpoints += len(resource.Endpoints)
+ }
+
+ fmt.Printf("Spec written to %s (%d endpoints across %d resources)\n", outputPath, endpoints, len(apiSpec.Resources))
+ fmt.Printf("Run 'printing-press generate --spec %s' to build the CLI\n", outputPath)
+ return nil
+ },
+ }
+
+ cmd.Flags().StringVar(&harPath, "har", "", "Path to HAR or enriched capture file")
+ cmd.Flags().StringVar(&outputPath, "output", "", "Output path for generated spec YAML")
+ cmd.Flags().StringVar(&name, "name", "", "Override the auto-detected API name")
+ cmd.Flags().StringVar(&blocklist, "blocklist", "", "Comma-separated additional domains to filter")
+ _ = cmd.MarkFlagRequired("har")
+
+ return cmd
+}
+
+func splitCSV(value string) []string {
+ if strings.TrimSpace(value) == "" {
+ return nil
+ }
+
+ parts := strings.Split(value, ",")
+ out := make([]string, 0, len(parts))
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ out = append(out, part)
+ }
+
+ return out
+}
diff --git a/internal/websniff/classifier.go b/internal/websniff/classifier.go
index 3403b020..6bcb08d4 100644
--- a/internal/websniff/classifier.go
+++ b/internal/websniff/classifier.go
@@ -15,16 +15,17 @@ type EndpointGroup struct {
}
var (
- uuidSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
- hashSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{32,}$`)
- numericPattern = regexp.MustCompile(`^\d+$`)
+ uuidSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+ hashSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{32,}$`)
+ numericPattern = regexp.MustCompile(`^\d+$`)
+ additionalBlocklist []string
)
func ClassifyEntries(entries []EnrichedEntry) (api []EnrichedEntry, noise []EnrichedEntry) {
api = make([]EnrichedEntry, 0, len(entries))
noise = make([]EnrichedEntry, 0, len(entries))
- blocklist := DefaultBlocklist()
+ blocklist := append(DefaultBlocklist(), additionalBlocklist...)
for _, entry := range entries {
score := scoreEntry(entry, blocklist)
classified := entry
@@ -43,6 +44,10 @@ func ClassifyEntries(entries []EnrichedEntry) (api []EnrichedEntry, noise []Enri
return api, noise
}
+func SetAdditionalBlocklist(domains []string) {
+ additionalBlocklist = append([]string(nil), domains...)
+}
+
func DefaultBlocklist() []string {
return []string{
"google-analytics.com",
diff --git a/internal/websniff/specgen.go b/internal/websniff/specgen.go
new file mode 100644
index 00000000..bc0ddfa8
--- /dev/null
+++ b/internal/websniff/specgen.go
@@ -0,0 +1,461 @@
+package websniff
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/mvanhorn/cli-printing-press/internal/spec"
+ "gopkg.in/yaml.v3"
+)
+
+func Analyze(capturePath string) (*spec.APISpec, error) {
+ entries, targetURL, err := ParseCapture(capturePath)
+ if err != nil {
+ return nil, err
+ }
+
+ apiEntries, _ := ClassifyEntries(entries)
+ groups := DeduplicateEndpoints(apiEntries)
+
+ resources := make(map[string]spec.Resource)
+ for _, group := range groups {
+ endpoint := buildEndpoint(group)
+ resourceKey, resourceName := deriveResourceKey(group.NormalizedPath)
+ if resourceKey == "" {
+ resourceKey = "default"
+ resourceName = "default"
+ }
+
+ resource := resources[resourceKey]
+ if resource.Description == "" {
+ resource.Description = fmt.Sprintf("Operations on %s", resourceName)
+ }
+ if resource.Endpoints == nil {
+ resource.Endpoints = make(map[string]spec.Endpoint)
+ }
+
+ name := deriveEndpointName(group.Method, group.NormalizedPath)
+ if _, exists := resource.Endpoints[name]; exists {
+ name = uniqueEndpointName(resource.Endpoints, name)
+ }
+ resource.Endpoints[name] = endpoint
+ resources[resourceKey] = resource
+ }
+
+ baseURL := mostCommonBaseURL(apiEntries)
+ if baseURL == "" {
+ baseURL = normalizeBaseURL(targetURL)
+ }
+
+ nameSource := targetURL
+ if nameSource == "" {
+ nameSource = baseURL
+ }
+ name := deriveNameFromURL(nameSource)
+
+ apiSpec := &spec.APISpec{
+ Name: name,
+ Description: fmt.Sprintf("Discovered API spec for %s", name),
+ Version: "0.1.0",
+ BaseURL: baseURL,
+ Auth: detectAuth(apiEntries, name),
+ Config: spec.ConfigSpec{
+ Format: "toml",
+ Path: fmt.Sprintf("~/.config/%s-pp-cli/config.toml", name),
+ },
+ Resources: resources,
+ Types: map[string]spec.TypeDef{},
+ }
+
+ if err := apiSpec.Validate(); err != nil {
+ if len(apiSpec.Resources) == 0 && len(groups) > 0 {
+ apiSpec.Resources["default"] = spec.Resource{
+ Description: "Discovered operations",
+ Endpoints: map[string]spec.Endpoint{},
+ }
+ }
+ if apiSpec.Auth.Type == "" {
+ apiSpec.Auth = spec.AuthConfig{Type: "none"}
+ }
+ if validateErr := apiSpec.Validate(); validateErr != nil {
+ return nil, fmt.Errorf("validating generated spec: %w", validateErr)
+ }
+ }
+
+ return apiSpec, nil
+}
+
+func WriteSpec(apiSpec *spec.APISpec, outputPath string) error {
+ if apiSpec == nil {
+ return fmt.Errorf("api spec is required")
+ }
+
+ data, err := yaml.Marshal(apiSpec)
+ if err != nil {
+ return fmt.Errorf("marshaling spec yaml: %w", err)
+ }
+
+ if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
+ return fmt.Errorf("creating output directory: %w", err)
+ }
+
+ if err := os.WriteFile(outputPath, data, 0o644); err != nil {
+ return fmt.Errorf("writing spec yaml: %w", err)
+ }
+
+ return nil
+}
+
+func DefaultCachePath(name string) string {
+ home, err := os.UserHomeDir()
+ if err != nil || home == "" {
+ return filepath.Join(".cache", "printing-press", "sniff", name+"-spec.yaml")
+ }
+
+ return filepath.Join(home, ".cache", "printing-press", "sniff", name+"-spec.yaml")
+}
+
+func buildEndpoint(group EndpointGroup) spec.Endpoint {
+ responseBodies := make([]string, 0, len(group.Entries))
+ for _, entry := range group.Entries {
+ if strings.TrimSpace(entry.ResponseBody) != "" {
+ responseBodies = append(responseBodies, entry.ResponseBody)
+ }
+ }
+
+ body := inferRequestBody(group.Entries)
+ params := inferURLParams(group.Entries, group.NormalizedPath)
+ auth := detectAuth(group.Entries, "")
+ if auth.Type == "api_key" && strings.EqualFold(auth.In, "query") && auth.Header != "" {
+ params = filterAuthQueryParam(params, auth.Header)
+ }
+
+ responseType := inferResponseType(responseBodies)
+ responseFields := InferResponseSchema(responseBodies)
+ if len(params) == 0 && len(responseFields) > 0 {
+ params = responseFields
+ }
+
+ return spec.Endpoint{
+ Method: group.Method,
+ Path: group.NormalizedPath,
+ Description: fmt.Sprintf("%s %s", group.Method, group.NormalizedPath),
+ Params: params,
+ Body: body,
+ Response: spec.ResponseDef{
+ Type: responseType,
+ Item: deriveResponseItemName(group.NormalizedPath),
+ },
+ }
+}
+
+func inferRequestBody(entries []EnrichedEntry) []spec.Param {
+ for _, entry := range entries {
+ body := strings.TrimSpace(entry.RequestBody)
+ if body == "" {
+ continue
+ }
+
+ contentType := getHeaderValue(entry.RequestHeaders, "Content-Type")
+ params := InferRequestSchema(body, contentType)
+ if len(params) > 0 {
+ return params
+ }
+ }
+
+ return nil
+}
+
+func inferURLParams(entries []EnrichedEntry, normalizedPath string) []spec.Param {
+ paramsByName := make(map[string]spec.Param)
+
+ for _, segment := range strings.Split(normalizedPath, "/") {
+ if !strings.HasPrefix(segment, "{") || !strings.HasSuffix(segment, "}") {
+ continue
+ }
+
+ name := strings.TrimSuffix(strings.TrimPrefix(segment, "{"), "}")
+ paramsByName[name] = spec.Param{
+ Name: name,
+ Type: "string",
+ Required: true,
+ Positional: true,
+ Description: fmt.Sprintf("The %s path segment", name),
+ }
+ }
+
+ for _, entry := range entries {
+ parsed, err := url.Parse(entry.URL)
+ if err != nil {
+ continue
+ }
+
+ for key, values := range parsed.Query() {
+ if _, exists := paramsByName[key]; exists {
+ continue
+ }
+
+ value := ""
+ if len(values) > 0 {
+ value = values[0]
+ }
+
+ paramsByName[key] = spec.Param{
+ Name: key,
+ Type: inferScalarStringType(value),
+ Required: false,
+ Description: "",
+ }
+ }
+ }
+
+ if len(paramsByName) == 0 {
+ return nil
+ }
+
+ names := make([]string, 0, len(paramsByName))
+ for name := range paramsByName {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+
+ params := make([]spec.Param, 0, len(names))
+ for _, name := range names {
+ params = append(params, paramsByName[name])
+ }
+
+ return params
+}
+
+func detectAuth(entries []EnrichedEntry, name string) spec.AuthConfig {
+ envPrefix := strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
+ for _, entry := range entries {
+ for headerName, value := range entry.RequestHeaders {
+ lowerHeader := strings.ToLower(headerName)
+ switch {
+ case strings.EqualFold(headerName, "Authorization") && strings.HasPrefix(strings.TrimSpace(value), "Bearer "):
+ return spec.AuthConfig{
+ Type: "bearer_token",
+ Header: "Authorization",
+ EnvVars: envVarsOrNil(envPrefix, "TOKEN"),
+ }
+ case strings.Contains(lowerHeader, "api-key") || strings.Contains(lowerHeader, "api_key"):
+ return spec.AuthConfig{
+ Type: "api_key",
+ Header: headerName,
+ In: "header",
+ EnvVars: envVarsOrNil(envPrefix, "API_KEY"),
+ }
+ }
+ }
+
+ parsed, err := url.Parse(entry.URL)
+ if err != nil {
+ continue
+ }
+ for key := range parsed.Query() {
+ lowerKey := strings.ToLower(key)
+ if strings.Contains(lowerKey, "key") || strings.Contains(lowerKey, "token") {
+ return spec.AuthConfig{
+ Type: "api_key",
+ Header: key,
+ In: "query",
+ EnvVars: envVarsOrNil(envPrefix, "API_KEY"),
+ }
+ }
+ }
+ }
+
+ return spec.AuthConfig{Type: "none"}
+}
+
+func envVarsOrNil(prefix string, suffix string) []string {
+ if prefix == "" {
+ return nil
+ }
+
+ return []string{prefix + "_" + suffix}
+}
+
+func mostCommonBaseURL(entries []EnrichedEntry) string {
+ counts := make(map[string]int)
+ best := ""
+ bestCount := 0
+
+ for _, entry := range entries {
+ base := normalizeBaseURL(entry.URL)
+ if base == "" {
+ continue
+ }
+
+ counts[base]++
+ if counts[base] > bestCount {
+ best = base
+ bestCount = counts[base]
+ }
+ }
+
+ return best
+}
+
+func normalizeBaseURL(raw string) string {
+ parsed, err := url.Parse(raw)
+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+ return ""
+ }
+
+ return parsed.Scheme + "://" + parsed.Host
+}
+
+func deriveResourceKey(path string) (string, string) {
+ segments := significantSegments(path)
+ if len(segments) == 0 {
+ return "", ""
+ }
+
+ if len(segments) > 3 {
+ segments = segments[:3]
+ }
+
+ return strings.Join(segments, "/"), segments[len(segments)-1]
+}
+
+func significantSegments(path string) []string {
+ parts := strings.Split(path, "/")
+ segments := make([]string, 0, len(parts))
+ for _, segment := range parts {
+ segment = strings.TrimSpace(segment)
+ if segment == "" {
+ continue
+ }
+ if strings.HasPrefix(segment, "{") && strings.HasSuffix(segment, "}") {
+ continue
+ }
+ if segment == "api" || isVersionSegment(segment) {
+ continue
+ }
+ segments = append(segments, segment)
+ }
+
+ return segments
+}
+
+func isVersionSegment(segment string) bool {
+ if len(segment) < 2 || segment[0] != 'v' {
+ return false
+ }
+ for _, r := range segment[1:] {
+ if r < '0' || r > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+func deriveEndpointName(method string, normalizedPath string) string {
+ resource := "endpoint"
+ segments := significantSegments(normalizedPath)
+ if len(segments) > 0 {
+ resource = strings.ReplaceAll(segments[len(segments)-1], "-", "_")
+ }
+
+ switch strings.ToUpper(method) {
+ case "GET":
+ if strings.Contains(normalizedPath, "{") {
+ return "get_" + resource
+ }
+ return "list_" + resource
+ case "POST":
+ return "create_" + resource
+ case "PUT", "PATCH":
+ return "update_" + resource
+ case "DELETE":
+ return "delete_" + resource
+ default:
+ return strings.ToLower(method) + "_" + resource
+ }
+}
+
+func uniqueEndpointName(endpoints map[string]spec.Endpoint, base string) string {
+ for i := 2; ; i++ {
+ name := fmt.Sprintf("%s_%d", base, i)
+ if _, exists := endpoints[name]; !exists {
+ return name
+ }
+ }
+}
+
+func inferResponseType(bodies []string) string {
+ for _, body := range bodies {
+ body = strings.TrimSpace(body)
+ if body == "" {
+ continue
+ }
+
+ var value any
+ if err := json.Unmarshal([]byte(body), &value); err != nil {
+ continue
+ }
+
+ switch value.(type) {
+ case []any:
+ return "array"
+ case map[string]any:
+ return "object"
+ }
+ }
+
+ return "object"
+}
+
+func deriveResponseItemName(path string) string {
+ segments := significantSegments(path)
+ if len(segments) == 0 {
+ return "response"
+ }
+
+ return strings.ReplaceAll(segments[len(segments)-1], "-", "_")
+}
+
+func filterAuthQueryParam(params []spec.Param, authParam string) []spec.Param {
+ filtered := make([]spec.Param, 0, len(params))
+ for _, param := range params {
+ if strings.EqualFold(param.Name, authParam) {
+ continue
+ }
+ filtered = append(filtered, param)
+ }
+ return filtered
+}
+
+func deriveNameFromURL(raw string) string {
+ parsed, err := url.Parse(raw)
+ if err != nil || parsed.Hostname() == "" {
+ return "api"
+ }
+
+ host := strings.TrimPrefix(strings.ToLower(parsed.Hostname()), "www.")
+ labels := strings.Split(host, ".")
+ if len(labels) == 0 {
+ return "api"
+ }
+ if len(labels) > 2 {
+ switch labels[0] {
+ case "api", "app", "developer", "developers":
+ labels = labels[1:]
+ }
+ }
+ if len(labels) > 1 {
+ labels = labels[:len(labels)-1]
+ }
+ if len(labels) == 0 {
+ return "api"
+ }
+
+ return strings.Join(labels, "-")
+}
diff --git a/internal/websniff/specgen_test.go b/internal/websniff/specgen_test.go
new file mode 100644
index 00000000..797597b6
--- /dev/null
+++ b/internal/websniff/specgen_test.go
@@ -0,0 +1,112 @@
+package websniff
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/mvanhorn/cli-printing-press/internal/spec"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAnalyze(t *testing.T) {
+ t.Parallel()
+
+ apiSpec, err := Analyze(filepath.Join("..", "..", "testdata", "sniff", "sample-enriched.json"))
+ require.NoError(t, err)
+ require.NotNil(t, apiSpec)
+
+ assert.Equal(t, "hn-algolia", apiSpec.Name)
+ require.NotEmpty(t, apiSpec.Resources)
+
+ foundEndpointWithParams := false
+ for _, resource := range apiSpec.Resources {
+ require.NotEmpty(t, resource.Endpoints)
+ for _, endpoint := range resource.Endpoints {
+ if len(endpoint.Params) > 0 {
+ foundEndpointWithParams = true
+ }
+ }
+ }
+
+ assert.True(t, foundEndpointWithParams)
+ assert.NoError(t, apiSpec.Validate())
+}
+
+func TestWriteSpec(t *testing.T) {
+ t.Parallel()
+
+ apiSpec := &spec.APISpec{
+ Name: "example",
+ Description: "Example API",
+ Version: "0.1.0",
+ BaseURL: "https://api.example.com",
+ Auth: spec.AuthConfig{Type: "none"},
+ Config: spec.ConfigSpec{
+ Format: "toml",
+ Path: "~/.config/example-pp-cli/config.toml",
+ },
+ Resources: map[string]spec.Resource{
+ "widgets": {
+ Description: "Operations on widgets",
+ Endpoints: map[string]spec.Endpoint{
+ "list_widgets": {
+ Method: "GET",
+ Path: "/widgets",
+ Response: spec.ResponseDef{
+ Type: "object",
+ Item: "widgets",
+ },
+ },
+ },
+ },
+ },
+ Types: map[string]spec.TypeDef{},
+ }
+
+ outputPath := filepath.Join(t.TempDir(), "nested", "spec.yaml")
+ err := WriteSpec(apiSpec, outputPath)
+ require.NoError(t, err)
+
+ data, err := os.ReadFile(outputPath)
+ require.NoError(t, err)
+
+ parsed, err := spec.ParseBytes(data)
+ require.NoError(t, err)
+ assert.Equal(t, apiSpec.Name, parsed.Name)
+ assert.Equal(t, apiSpec.BaseURL, parsed.BaseURL)
+}
+
+func TestDeriveNameFromURL(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ raw string
+ want string
+ }{
+ {
+ name: "strips www and tld",
+ raw: "https://www.youtube.com",
+ want: "youtube",
+ },
+ {
+ name: "keeps meaningful subdomain",
+ raw: "https://hn.algolia.com",
+ want: "hn-algolia",
+ },
+ {
+ name: "drops generic api prefix",
+ raw: "https://api.example.com",
+ want: "example",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tt.want, deriveNameFromURL(tt.raw))
+ })
+ }
+}
← f9b8e141 feat(websniff): add JSON schema inference from captured payl
·
back to Cli Printing Press
·
refactor(cli): decouple skills and binary from repo checkout dfc0eedc →