← back to Cli Printing Press
feat(websniff): add API endpoint classifier with analytics blocklist
07f158f615a8abc32057986c77660ca7678c4d50 · 2026-03-28 21:29:02 -0700 · Matt Van Horn
Scoring heuristic classifies HAR entries as API vs noise. Includes
30-domain analytics blocklist, path parameter detection (numeric,
UUID, hex), and endpoint deduplication by normalized method+path.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Files touched
A internal/websniff/classifier.goA internal/websniff/classifier_test.go
Diff
commit 07f158f615a8abc32057986c77660ca7678c4d50
Author: Matt Van Horn <mvanhorn@MacBook-Pro.local>
Date: Sat Mar 28 21:29:02 2026 -0700
feat(websniff): add API endpoint classifier with analytics blocklist
Scoring heuristic classifies HAR entries as API vs noise. Includes
30-domain analytics blocklist, path parameter detection (numeric,
UUID, hex), and endpoint deduplication by normalized method+path.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
internal/websniff/classifier.go | 258 +++++++++++++++++++++++++++++++++++
internal/websniff/classifier_test.go | 236 ++++++++++++++++++++++++++++++++
2 files changed, 494 insertions(+)
diff --git a/internal/websniff/classifier.go b/internal/websniff/classifier.go
new file mode 100644
index 00000000..3403b020
--- /dev/null
+++ b/internal/websniff/classifier.go
@@ -0,0 +1,258 @@
+package websniff
+
+import (
+ "encoding/json"
+ "net"
+ "net/url"
+ "regexp"
+ "strings"
+)
+
+type EndpointGroup struct {
+ Method string
+ NormalizedPath string
+ Entries []EnrichedEntry
+}
+
+var (
+ uuidSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+ hashSegmentPattern = regexp.MustCompile(`(?i)^[0-9a-f]{32,}$`)
+ numericPattern = regexp.MustCompile(`^\d+$`)
+)
+
+func ClassifyEntries(entries []EnrichedEntry) (api []EnrichedEntry, noise []EnrichedEntry) {
+ api = make([]EnrichedEntry, 0, len(entries))
+ noise = make([]EnrichedEntry, 0, len(entries))
+
+ blocklist := DefaultBlocklist()
+ for _, entry := range entries {
+ score := scoreEntry(entry, blocklist)
+ classified := entry
+ if score > 0 {
+ classified.Classification = "api"
+ classified.IsNoise = false
+ api = append(api, classified)
+ continue
+ }
+
+ classified.Classification = "noise"
+ classified.IsNoise = true
+ noise = append(noise, classified)
+ }
+
+ return api, noise
+}
+
+func DefaultBlocklist() []string {
+ return []string{
+ "google-analytics.com",
+ "doubleclick.net",
+ "sentry.io",
+ "facebook.com",
+ "googlesyndication.com",
+ "googletagmanager.com",
+ "fonts.googleapis.com",
+ "gstatic.com",
+ "bat.bing.com",
+ "criteo.com",
+ "demdex.net",
+ "onetrust.com",
+ "cookielaw.org",
+ "amazon-adsystem.com",
+ "adsymptotic.com",
+ "improving.duckduckgo.com",
+ "lngtd.com",
+ "kargo.com",
+ "segment.io",
+ "api.segment.io",
+ "mixpanel.com",
+ "amplitude.com",
+ "hotjar.com",
+ "newrelic.com",
+ "fullstory.com",
+ "intercom.io",
+ "branch.io",
+ "stats.g.doubleclick.net",
+ "adservice.google.com",
+ "connect.facebook.net",
+ }
+}
+
+func DeduplicateEndpoints(entries []EnrichedEntry) []EndpointGroup {
+ groups := make([]EndpointGroup, 0)
+ indexByKey := make(map[string]int)
+
+ for _, entry := range entries {
+ method := strings.ToUpper(strings.TrimSpace(entry.Method))
+ normalizedPath := normalizeEntryPath(entry.URL)
+ key := method + " " + normalizedPath
+
+ if idx, ok := indexByKey[key]; ok {
+ groups[idx].Entries = append(groups[idx].Entries, entry)
+ continue
+ }
+
+ indexByKey[key] = len(groups)
+ groups = append(groups, EndpointGroup{
+ Method: method,
+ NormalizedPath: normalizedPath,
+ Entries: []EnrichedEntry{entry},
+ })
+ }
+
+ return groups
+}
+
+func scoreEntry(entry EnrichedEntry, blocklist []string) int {
+ score := 0
+ responseType := strings.ToLower(entry.ResponseContentType)
+ requestType := strings.ToLower(getHeaderValue(entry.RequestHeaders, "Content-Type"))
+ path := strings.ToLower(extractPath(entry.URL))
+ host := strings.ToLower(extractHost(entry.URL))
+ urlLower := strings.ToLower(entry.URL)
+
+ if strings.Contains(responseType, "application/json") {
+ score += 2
+ }
+
+ if strings.Contains(requestType, "application/json") || strings.Contains(requestType, "application/x-www-form-urlencoded") {
+ score++
+ }
+
+ for _, indicator := range []string{"/api/", "/v1/", "/v2/", "/v3/", "/graphql", "/data/", "/youtubei/"} {
+ if strings.Contains(path, indicator) {
+ score++
+ break
+ }
+ }
+
+ if isValidJSONBody(entry.ResponseBody) {
+ score++
+ }
+
+ if hostMatchesBlocklist(host, blocklist) {
+ score -= 3
+ }
+
+ for _, prefix := range []string{"image/", "text/css", "text/html", "application/javascript", "font/"} {
+ if strings.HasPrefix(responseType, prefix) {
+ score -= 2
+ break
+ }
+ }
+
+ for _, suffix := range []string{".js", ".css", ".png", ".jpg", ".woff", ".svg", ".ico"} {
+ if strings.HasSuffix(urlLower, suffix) {
+ score--
+ break
+ }
+ }
+
+ return score
+}
+
+func getHeaderValue(headers map[string]string, want string) string {
+ for key, value := range headers {
+ if strings.EqualFold(key, want) {
+ return value
+ }
+ }
+
+ return ""
+}
+
+func isValidJSONBody(body string) bool {
+ if strings.TrimSpace(body) == "" {
+ return false
+ }
+
+ var payload any
+ return json.Unmarshal([]byte(body), &payload) == nil
+}
+
+func hostMatchesBlocklist(host string, blocklist []string) bool {
+ if host == "" {
+ return false
+ }
+
+ for _, blocked := range blocklist {
+ blocked = strings.ToLower(blocked)
+ if host == blocked || strings.HasSuffix(host, "."+blocked) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func extractHost(rawURL string) string {
+ parsed, err := url.Parse(rawURL)
+ if err == nil && parsed.Host != "" {
+ return parsed.Hostname()
+ }
+
+ host := rawURL
+ if idx := strings.Index(host, "://"); idx >= 0 {
+ host = host[idx+3:]
+ }
+ if idx := strings.IndexAny(host, "/?"); idx >= 0 {
+ host = host[:idx]
+ }
+
+ host, _, err = net.SplitHostPort(host)
+ if err == nil {
+ return host
+ }
+
+ return host
+}
+
+func extractPath(rawURL string) string {
+ parsed, err := url.Parse(rawURL)
+ if err == nil && parsed.Path != "" {
+ return parsed.Path
+ }
+
+ path := rawURL
+ if idx := strings.Index(path, "://"); idx >= 0 {
+ path = path[idx+3:]
+ if slash := strings.Index(path, "/"); slash >= 0 {
+ path = path[slash:]
+ } else {
+ return "/"
+ }
+ }
+ if idx := strings.Index(path, "?"); idx >= 0 {
+ path = path[:idx]
+ }
+ if path == "" {
+ return "/"
+ }
+ if !strings.HasPrefix(path, "/") {
+ path = "/" + path
+ }
+
+ return path
+}
+
+func normalizeEntryPath(rawURL string) string {
+ path := extractPath(rawURL)
+ segments := strings.Split(path, "/")
+ for i, segment := range segments {
+ switch {
+ case numericPattern.MatchString(segment):
+ segments[i] = "{id}"
+ case uuidSegmentPattern.MatchString(segment):
+ segments[i] = "{uuid}"
+ case hashSegmentPattern.MatchString(segment):
+ segments[i] = "{hash}"
+ }
+ }
+
+ normalized := strings.Join(segments, "/")
+ if normalized == "" {
+ return "/"
+ }
+
+ return normalized
+}
diff --git a/internal/websniff/classifier_test.go b/internal/websniff/classifier_test.go
new file mode 100644
index 00000000..8d6583f1
--- /dev/null
+++ b/internal/websniff/classifier_test.go
@@ -0,0 +1,236 @@
+package websniff
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestClassifyEntries(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ entries []EnrichedEntry
+ wantAPIURLs []string
+ wantNoiseURLs []string
+ wantClassByURL map[string]string
+ wantIsNoiseByURL map[string]bool
+ }{
+ {
+ name: "json api and analytics tracker are separated",
+ entries: []EnrichedEntry{
+ {
+ Method: "GET",
+ URL: "https://example.com/api/users",
+ ResponseContentType: "application/json; charset=utf-8",
+ ResponseBody: `{"users":[{"id":1}]}`,
+ },
+ {
+ Method: "GET",
+ URL: "https://www.google-analytics.com/g/collect?v=2",
+ ResponseContentType: "text/html",
+ },
+ },
+ wantAPIURLs: []string{"https://example.com/api/users"},
+ wantNoiseURLs: []string{"https://www.google-analytics.com/g/collect?v=2"},
+ wantClassByURL: map[string]string{
+ "https://example.com/api/users": "api",
+ "https://www.google-analytics.com/g/collect?v=2": "noise",
+ },
+ wantIsNoiseByURL: map[string]bool{
+ "https://example.com/api/users": false,
+ "https://www.google-analytics.com/g/collect?v=2": true,
+ },
+ },
+ {
+ name: "google analytics is noise",
+ entries: []EnrichedEntry{
+ {
+ Method: "POST",
+ URL: "https://google-analytics.com/j/collect",
+ ResponseContentType: "application/json",
+ ResponseBody: `{}`,
+ },
+ },
+ wantNoiseURLs: []string{"https://google-analytics.com/j/collect"},
+ wantClassByURL: map[string]string{
+ "https://google-analytics.com/j/collect": "noise",
+ },
+ wantIsNoiseByURL: map[string]bool{
+ "https://google-analytics.com/j/collect": true,
+ },
+ },
+ {
+ name: "post form with json response is api",
+ entries: []EnrichedEntry{
+ {
+ Method: "POST",
+ URL: "https://example.com/session",
+ ResponseContentType: "application/json",
+ ResponseBody: `{"ok":true}`,
+ RequestHeaders: map[string]string{
+ "Content-Type": "application/x-www-form-urlencoded",
+ },
+ },
+ },
+ wantAPIURLs: []string{"https://example.com/session"},
+ wantClassByURL: map[string]string{
+ "https://example.com/session": "api",
+ },
+ wantIsNoiseByURL: map[string]bool{
+ "https://example.com/session": false,
+ },
+ },
+ {
+ name: "all noise entries produce empty api list",
+ entries: []EnrichedEntry{
+ {
+ Method: "GET",
+ URL: "https://cdn.example.com/styles.css",
+ ResponseContentType: "text/css",
+ },
+ {
+ Method: "GET",
+ URL: "https://cdn.example.com/logo.png",
+ ResponseContentType: "image/png",
+ },
+ },
+ wantNoiseURLs: []string{
+ "https://cdn.example.com/styles.css",
+ "https://cdn.example.com/logo.png",
+ },
+ wantClassByURL: map[string]string{
+ "https://cdn.example.com/styles.css": "noise",
+ "https://cdn.example.com/logo.png": "noise",
+ },
+ wantIsNoiseByURL: map[string]bool{
+ "https://cdn.example.com/styles.css": true,
+ "https://cdn.example.com/logo.png": true,
+ },
+ },
+ {
+ name: "youtube internal endpoint is api",
+ entries: []EnrichedEntry{
+ {
+ Method: "POST",
+ URL: "https://www.youtube.com/youtubei/v1/player?prettyPrint=false",
+ ResponseContentType: "application/json",
+ ResponseBody: `{"videoDetails":{"videoId":"abc123"}}`,
+ },
+ },
+ wantAPIURLs: []string{"https://www.youtube.com/youtubei/v1/player?prettyPrint=false"},
+ wantClassByURL: map[string]string{
+ "https://www.youtube.com/youtubei/v1/player?prettyPrint=false": "api",
+ },
+ wantIsNoiseByURL: map[string]bool{
+ "https://www.youtube.com/youtubei/v1/player?prettyPrint=false": false,
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ api, noise := ClassifyEntries(tt.entries)
+
+ assert.Equal(t, emptyStrings(tt.wantAPIURLs), entryURLs(api))
+ assert.Equal(t, emptyStrings(tt.wantNoiseURLs), entryURLs(noise))
+
+ for _, entry := range append(api, noise...) {
+ assert.Equal(t, tt.wantClassByURL[entry.URL], entry.Classification)
+ assert.Equal(t, tt.wantIsNoiseByURL[entry.URL], entry.IsNoise)
+ }
+ })
+ }
+}
+
+func TestDeduplicateEndpoints(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ entries []EnrichedEntry
+ wantMethods []string
+ wantNormalizedPaths []string
+ wantGroupSizes []int
+ }{
+ {
+ name: "numeric ids normalize to id placeholder",
+ entries: []EnrichedEntry{
+ {Method: "GET", URL: "https://example.com/users/123?expand=true"},
+ {Method: "GET", URL: "https://example.com/users/456"},
+ },
+ wantMethods: []string{"GET"},
+ wantNormalizedPaths: []string{"/users/{id}"},
+ wantGroupSizes: []int{2},
+ },
+ {
+ name: "uuid segment normalizes to uuid placeholder",
+ entries: []EnrichedEntry{
+ {Method: "GET", URL: "https://example.com/orders/550e8400-e29b-41d4-a716-446655440000"},
+ {Method: "GET", URL: "https://example.com/orders/123e4567-e89b-12d3-a456-426614174000?include=items"},
+ },
+ wantMethods: []string{"GET"},
+ wantNormalizedPaths: []string{"/orders/{uuid}"},
+ wantGroupSizes: []int{2},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ groups := DeduplicateEndpoints(tt.entries)
+
+ assert.Equal(t, tt.wantMethods, groupMethods(groups))
+ assert.Equal(t, tt.wantNormalizedPaths, groupPaths(groups))
+ assert.Equal(t, tt.wantGroupSizes, groupSizes(groups))
+ })
+ }
+}
+
+func entryURLs(entries []EnrichedEntry) []string {
+ urls := make([]string, 0, len(entries))
+ for _, entry := range entries {
+ urls = append(urls, entry.URL)
+ }
+
+ return urls
+}
+
+func groupMethods(groups []EndpointGroup) []string {
+ methods := make([]string, 0, len(groups))
+ for _, group := range groups {
+ methods = append(methods, group.Method)
+ }
+
+ return methods
+}
+
+func groupPaths(groups []EndpointGroup) []string {
+ paths := make([]string, 0, len(groups))
+ for _, group := range groups {
+ paths = append(paths, group.NormalizedPath)
+ }
+
+ return paths
+}
+
+func groupSizes(groups []EndpointGroup) []int {
+ sizes := make([]int, 0, len(groups))
+ for _, group := range groups {
+ sizes = append(sizes, len(group.Entries))
+ }
+
+ return sizes
+}
+
+func emptyStrings(values []string) []string {
+ if values == nil {
+ return []string{}
+ }
+
+ return values
+}
← 6c6c9cf5 feat(websniff): add HAR and enriched capture parser
·
back to Cli Printing Press
·
feat(websniff): add JSON schema inference from captured payl f9b8e141 →