← back to Cli Printing Press
fix(cli): surface API body in sync_error events and add --param passthrough (#1104)
cb3c0974a1c9ab449e50d7a3700a979cb3e0f82c · 2026-05-11 13:07:19 -0700 · Trevin Chow
* fix(cli): surface API body in sync_error events and add --param passthrough
When an OpenAPI spec marks a filter param optional but the API rejects
requests without it (YouTube channels.list requiring mine=true, GitHub
repo listings without a scope), users had two problems: (1) no way to
inject the missing filter at sync time without hand-editing generated
code, and (2) the per-resource 4xx response body was buried inside a
wrapped err.Error() string in JSON event mode, with the dependent-
resource path swallowing it entirely.
This patch adds:
- --param key=value (repeatable) injecting a query param into every
sync request, and --resource-param resource:key=value (repeatable)
for per-resource overrides. Both apply after spec-derived params
(cursor, since, limit, dates) so user flags win on conflict. Help
text cautions against overriding pagination keys (corrupts resume
state).
- syncErrorJSON helper that unwraps *client.APIError and emits the
sync_error event with structured status/method/path/body fields,
while keeping the original error key for log consumers that read
the human-readable form.
- A previously-silent dependent-resource error path now emits a
sync_error JSON event carrying both the parent ID and the API body.
The client import in helpers.go.tmpl is relaxed from
auth-required+data-layer to just data-layer so the new helper compiles
on no-auth + data-layer specs (which can still encounter 4xx).
channel_workflow.go.tmpl and auto_refresh.go.tmpl gate the new
userParams arg on isGraphQL so GraphQL specs (whose graphql_sync.go.tmpl
defines its own syncResource without userParams) still compile.
Refs #881
* test(cli): regenerate sync goldens for --param + structured sync_error
* fix(cli): validate --resource-param keys + tighten dep-path comment per Greptile
Files touched
A internal/generator/sync_param_passthrough_test.goM internal/generator/templates/auto_refresh.go.tmplM internal/generator/templates/channel_workflow.go.tmplM internal/generator/templates/helpers.go.tmplM internal/generator/templates/sync.go.tmplM testdata/golden/expected/generate-golden-api-rich-auth/printing-press-rich-auth/internal/cli/helpers.goM testdata/golden/expected/generate-golden-api/dogfood.jsonM testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/helpers.goM testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/sync.goM testdata/golden/expected/generate-sync-walker-api/sync-walker-golden/internal/cli/sync.goM testdata/golden/expected/generate-tier-routing-api/tier-routing-golden/internal/cli/sync.go
Diff
commit cb3c0974a1c9ab449e50d7a3700a979cb3e0f82c
Author: Trevin Chow <trevin@trevinchow.com>
Date: Mon May 11 13:07:19 2026 -0700
fix(cli): surface API body in sync_error events and add --param passthrough (#1104)
* fix(cli): surface API body in sync_error events and add --param passthrough
When an OpenAPI spec marks a filter param optional but the API rejects
requests without it (YouTube channels.list requiring mine=true, GitHub
repo listings without a scope), users had two problems: (1) no way to
inject the missing filter at sync time without hand-editing generated
code, and (2) the per-resource 4xx response body was buried inside a
wrapped err.Error() string in JSON event mode, with the dependent-
resource path swallowing it entirely.
This patch adds:
- --param key=value (repeatable) injecting a query param into every
sync request, and --resource-param resource:key=value (repeatable)
for per-resource overrides. Both apply after spec-derived params
(cursor, since, limit, dates) so user flags win on conflict. Help
text cautions against overriding pagination keys (corrupts resume
state).
- syncErrorJSON helper that unwraps *client.APIError and emits the
sync_error event with structured status/method/path/body fields,
while keeping the original error key for log consumers that read
the human-readable form.
- A previously-silent dependent-resource error path now emits a
sync_error JSON event carrying both the parent ID and the API body.
The client import in helpers.go.tmpl is relaxed from
auth-required+data-layer to just data-layer so the new helper compiles
on no-auth + data-layer specs (which can still encounter 4xx).
channel_workflow.go.tmpl and auto_refresh.go.tmpl gate the new
userParams arg on isGraphQL so GraphQL specs (whose graphql_sync.go.tmpl
defines its own syncResource without userParams) still compile.
Refs #881
* test(cli): regenerate sync goldens for --param + structured sync_error
* fix(cli): validate --resource-param keys + tighten dep-path comment per Greptile
---
internal/generator/sync_param_passthrough_test.go | 192 +++++++++++++++++++++
internal/generator/templates/auto_refresh.go.tmpl | 2 +-
.../generator/templates/channel_workflow.go.tmpl | 2 +-
internal/generator/templates/helpers.go.tmpl | 116 ++++++++++++-
internal/generator/templates/sync.go.tmpl | 62 ++++++-
.../internal/cli/helpers.go | 114 ++++++++++++
.../expected/generate-golden-api/dogfood.json | 2 +-
.../printing-press-golden/internal/cli/helpers.go | 114 ++++++++++++
.../printing-press-golden/internal/cli/sync.go | 60 ++++++-
.../sync-walker-golden/internal/cli/sync.go | 60 ++++++-
.../tier-routing-golden/internal/cli/sync.go | 41 ++++-
11 files changed, 729 insertions(+), 36 deletions(-)
diff --git a/internal/generator/sync_param_passthrough_test.go b/internal/generator/sync_param_passthrough_test.go
new file mode 100644
index 00000000..f23f80f9
--- /dev/null
+++ b/internal/generator/sync_param_passthrough_test.go
@@ -0,0 +1,192 @@
+// Copyright 2026 Anthropic, PBC. Licensed under Apache-2.0.
+
+package generator
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/mvanhorn/cli-printing-press/v4/internal/naming"
+ "github.com/mvanhorn/cli-printing-press/v4/internal/spec"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// TestGenerateSyncParamPassthrough verifies the sync template emits the
+// --param / --resource-param flags, parses them through parseSyncUserParams,
+// and applies them after spec-derived params. Some APIs mark filter params
+// optional in the spec but reject requests without them at runtime; without
+// this passthrough, the only workaround is hand-editing the generated client.
+func TestGenerateSyncParamPassthrough(t *testing.T) {
+ t.Parallel()
+
+ apiSpec := minimalSpec("syncparam")
+ outputDir := filepath.Join(t.TempDir(), naming.CLI(apiSpec.Name))
+ gen := New(apiSpec, outputDir)
+ require.NoError(t, gen.Generate())
+
+ syncGo, err := os.ReadFile(filepath.Join(outputDir, "internal", "cli", "sync.go"))
+ require.NoError(t, err)
+ syncSrc := string(syncGo)
+
+ // Flag declarations exist with the expected help text shape (operator
+ // has to know what to type without reading the source).
+ assert.Contains(t, syncSrc, `cmd.Flags().StringArrayVar(¶mFlags, "param"`,
+ "sync should expose a repeatable --param flag")
+ assert.Contains(t, syncSrc, `cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param"`,
+ "sync should expose a repeatable --resource-param flag")
+ assert.Contains(t, syncSrc, "key=value",
+ "--param help text should describe the key=value shape")
+ assert.Contains(t, syncSrc, "resource:key=value",
+ "--resource-param help text should describe the resource:key=value shape")
+
+ // Parsing runs before client construction so a malformed flag fails fast
+ // (and as usageErr, not a generic runtime error).
+ assert.Contains(t, syncSrc, "parseSyncUserParams(paramFlags, resourceParamFlags)",
+ "sync must parse user params at RunE entry")
+ parseIdx := strings.Index(syncSrc, "parseSyncUserParams(paramFlags, resourceParamFlags)")
+ newClientIdx := strings.Index(syncSrc, "flags.newClient()")
+ require.NotEqual(t, -1, parseIdx)
+ require.NotEqual(t, -1, newClientIdx)
+ assert.Less(t, parseIdx, newClientIdx,
+ "--param must parse before newClient so usage errors don't waste an HTTP handshake")
+
+ // userParams flows through to the syncResource worker. The exact call
+ // site differs by template branch (HasTierRouting vs not), so assert the
+ // last arg is userParams.
+ assert.Contains(t, syncSrc, ", userParams)",
+ "syncResource and syncDependentResources must receive userParams")
+
+ // applyTo is called in the page loop AFTER cursor/since/limit are set,
+ // so user flags win on conflict.
+ loopIdx := strings.Index(syncSrc, "params[pageSize.limitParam] = strconv.Itoa(pageSize.limit)")
+ applyIdx := strings.Index(syncSrc, "userParams.applyTo(resource, params)")
+ require.NotEqual(t, -1, loopIdx, "page loop should set the limit param")
+ require.NotEqual(t, -1, applyIdx, "syncResource should apply user params before c.Get")
+ assert.Less(t, loopIdx, applyIdx,
+ "user params must apply after spec-derived params so flags can override")
+
+ // --resource-param keys must be validated against known resources before
+ // sync starts, so a typo errors instead of silently no-op'ing.
+ assert.Contains(t, syncSrc, "userParams.validateResourceNames(knownSyncResourceNames())",
+ "sync should validate --resource-param keys against the known resource set")
+ assert.Contains(t, syncSrc, "func knownSyncResourceNames() []string",
+ "knownSyncResourceNames helper must be emitted alongside defaultSyncResources")
+}
+
+// TestGenerateSyncErrorJSONIncludesAPIBody verifies that the sync_error JSON
+// event surfaces the API response body as a structured field, not just
+// embedded inside an opaque err.Error() string. Without this surfacing, a
+// 4xx whose JSON event stream contains only `errored: 1` leaves operators
+// without the body needed to diagnose required-but-not-spec'd filter params.
+func TestGenerateSyncErrorJSONIncludesAPIBody(t *testing.T) {
+ t.Parallel()
+
+ apiSpec := minimalSpec("syncerr")
+ outputDir := filepath.Join(t.TempDir(), naming.CLI(apiSpec.Name))
+ gen := New(apiSpec, outputDir)
+ require.NoError(t, gen.Generate())
+
+ helpersGo, err := os.ReadFile(filepath.Join(outputDir, "internal", "cli", "helpers.go"))
+ require.NoError(t, err)
+ helpersSrc := string(helpersGo)
+
+ // The helper unwraps *client.APIError and populates Status/Method/Path/Body.
+ assert.Contains(t, helpersSrc, "func syncErrorJSON(resource, parent string, err error) string",
+ "syncErrorJSON helper must exist with the resource/parent/err signature")
+ assert.Contains(t, helpersSrc, "var apiErr *client.APIError",
+ "helper must extract *client.APIError for structured fields")
+ for _, snippet := range []string{
+ "payload.Status = apiErr.StatusCode",
+ "payload.Method = apiErr.Method",
+ "payload.Path = apiErr.Path",
+ "payload.Body = apiErr.Body",
+ } {
+ assert.Contains(t, helpersSrc, snippet,
+ "sync_error payload should surface %s from APIError", snippet)
+ }
+
+ // The flat path now uses the helper instead of a hand-rolled fmt.Fprintf
+ // that embedded the body inside err.Error(). Confirm the old form is
+ // gone (otherwise the body would still be lost in a wrapped string).
+ syncGo, err := os.ReadFile(filepath.Join(outputDir, "internal", "cli", "sync.go"))
+ require.NoError(t, err)
+ syncSrc := string(syncGo)
+ assert.NotContains(t, syncSrc, `{"event":"sync_error","resource":"%s","error":"%s"}`,
+ "sync_error event should be emitted via syncErrorJSON, not the legacy fmt.Fprintf shape")
+ assert.Contains(t, syncSrc, `syncErrorJSON(resource, "", err)`,
+ "syncResource flat path should emit sync_error via the helper")
+}
+
+// TestGenerateSyncDependentErrorNotSilent verifies the dependent-resource
+// error path emits a sync_error JSON event for non-warning failures. The
+// previous shape only emitted in human mode, so a 4xx on a parent request
+// was invisible in agent-driven runs — operators saw missing rows with no
+// diagnostic.
+func TestGenerateSyncDependentErrorNotSilent(t *testing.T) {
+ t.Parallel()
+
+ // Build a spec with a parent + child resource so syncDependentResource
+ // is actually emitted. The dependent-resource profiler requires paginated
+ // list endpoints (not bare GETs) for the parameterized child path to be
+ // classified as syncable; see profiler.detectDependentResources.
+ paginated := func(path string) spec.Endpoint {
+ return spec.Endpoint{
+ Method: "GET",
+ Path: path,
+ Response: spec.ResponseDef{Type: "array"},
+ Pagination: &spec.Pagination{Type: "cursor", LimitParam: "limit", CursorParam: "after"},
+ }
+ }
+ apiSpec := &spec.APISpec{
+ Name: "dependent-err",
+ Version: "0.1.0",
+ BaseURL: "https://api.example.test/v1",
+ Owner: "test-owner",
+ OwnerName: "Test Author",
+ Auth: spec.AuthConfig{
+ Type: "api_key",
+ Header: "Authorization",
+ Format: "Bearer {token}",
+ EnvVars: []string{"DEPENDENT_ERR_API_KEY"},
+ },
+ Config: spec.ConfigSpec{
+ Format: "toml",
+ Path: "~/.config/dependent-err-pp-cli/config.toml",
+ },
+ Resources: map[string]spec.Resource{
+ "teams": {
+ Description: "Teams",
+ Endpoints: map[string]spec.Endpoint{
+ "list": paginated("/teams"),
+ },
+ },
+ "players": {
+ Description: "Players (child of teams)",
+ Endpoints: map[string]spec.Endpoint{
+ "list": paginated("/teams/{team_id}/players"),
+ },
+ },
+ },
+ }
+
+ outputDir := filepath.Join(t.TempDir(), naming.CLI(apiSpec.Name))
+ gen := New(apiSpec, outputDir)
+ require.NoError(t, gen.Generate())
+
+ syncGo, err := os.ReadFile(filepath.Join(outputDir, "internal", "cli", "sync.go"))
+ require.NoError(t, err)
+ syncSrc := string(syncGo)
+
+ // syncDependentResource must actually be rendered for this spec.
+ require.Contains(t, syncSrc, "func syncDependentResource(",
+ "dependent-resource sync should render when the spec has a {parent_id} child path")
+
+ // Non-warning errors must reach syncErrorJSON in the dep path. The
+ // helper takes a non-empty parent ID so consumers can attribute the
+ // failure to a specific parent.
+ assert.Contains(t, syncSrc, "syncErrorJSON(dep.Name, parentID, err)",
+ "dependent-resource non-warning error must emit a sync_error JSON event with the parent ID")
+}
diff --git a/internal/generator/templates/auto_refresh.go.tmpl b/internal/generator/templates/auto_refresh.go.tmpl
index 0ed7f8a9..f9a0bb6e 100644
--- a/internal/generator/templates/auto_refresh.go.tmpl
+++ b/internal/generator/templates/auto_refresh.go.tmpl
@@ -189,7 +189,7 @@ func runAutoRefresh(ctx context.Context, flags *rootFlags, db *store.Store, reso
return ctx.Err()
default:
}
- result := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, db, resource, "", false, 1{{if .Pagination.DateRangeParam}}, ""{{end}})
+ result := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, db, resource, "", false, 1{{if .Pagination.DateRangeParam}}, ""{{end}}{{if not (isGraphQL .APISpec)}}, nil{{end}})
if result.Err != nil {
failures = append(failures, fmt.Sprintf("%s: %v", resource, result.Err))
}
diff --git a/internal/generator/templates/channel_workflow.go.tmpl b/internal/generator/templates/channel_workflow.go.tmpl
index b732062d..c63768cb 100644
--- a/internal/generator/templates/channel_workflow.go.tmpl
+++ b/internal/generator/templates/channel_workflow.go.tmpl
@@ -288,7 +288,7 @@ and full resync. After archiving, use 'search' for instant full-text search.`,
}
for _, resource := range resources {
- res := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, s, resource, "", full, 100{{if .Pagination.DateRangeParam}}, ""{{end}})
+ res := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, s, resource, "", full, 100{{if .Pagination.DateRangeParam}}, ""{{end}}{{if not (isGraphQL .APISpec)}}, nil{{end}})
if res.Err != nil {
fmt.Fprintf(cmd.ErrOrStderr(), " %s: error: %v\n", resource, res.Err)
continue
diff --git a/internal/generator/templates/helpers.go.tmpl b/internal/generator/templates/helpers.go.tmpl
index 629368db..952f6521 100644
--- a/internal/generator/templates/helpers.go.tmpl
+++ b/internal/generator/templates/helpers.go.tmpl
@@ -30,7 +30,7 @@ import (
{{- if and .Auth.Type (ne .Auth.Type "none")}}
"{{modulePath}}/internal/cliutil"
{{- end}}
-{{- if and (and .Auth.Type (ne .Auth.Type "none")) .HasDataLayer}}
+{{- if .HasDataLayer}}
"{{modulePath}}/internal/client"
{{- end}}
"github.com/spf13/cobra"
@@ -147,6 +147,120 @@ type accessWarning struct {
Reason string // "forbidden" | "insufficient_access" | "unauthenticated"
Message string // human-readable detail (the API's body or GraphQL error message)
}
+
+// syncErrorJSON returns a one-line JSON sync_error event. When err wraps a
+// *client.APIError, the structured status/method/path/body fields let
+// operators see the API's response body without parsing a wrapped error
+// string — required for diagnosing 4xx responses on endpoints whose OpenAPI
+// spec marks filter params optional but the API treats them as required.
+//
+// parent is non-empty only on the dependent-resource path, where it
+// identifies which parent ID's child request failed.
+func syncErrorJSON(resource, parent string, err error) string {
+ payload := struct {
+ Event string `json:"event"`
+ Resource string `json:"resource"`
+ Parent string `json:"parent,omitempty"`
+ Status int `json:"status,omitempty"`
+ Method string `json:"method,omitempty"`
+ Path string `json:"path,omitempty"`
+ Body string `json:"body,omitempty"`
+ Error string `json:"error"`
+ }{
+ Event: "sync_error",
+ Resource: resource,
+ Parent: parent,
+ Error: err.Error(),
+ }
+ var apiErr *client.APIError
+ if errors.As(err, &apiErr) {
+ payload.Status = apiErr.StatusCode
+ payload.Method = apiErr.Method
+ payload.Path = apiErr.Path
+ payload.Body = apiErr.Body
+ }
+ out, _ := json.Marshal(payload)
+ return string(out)
+}
+
+// syncUserParams carries user-supplied query parameters injected into every
+// sync HTTP request. perResource entries win over global on key conflict.
+type syncUserParams struct {
+ global map[string]string
+ perResource map[string]map[string]string
+}
+
+// parseSyncUserParams parses the repeatable --param key=value and
+// --resource-param resource:key=value flags. Returns usage errors keyed on the
+// specific invalid token so the user sees which entry was rejected.
+func parseSyncUserParams(globalFlags, perResourceFlags []string) (*syncUserParams, error) {
+ p := &syncUserParams{
+ global: map[string]string{},
+ perResource: map[string]map[string]string{},
+ }
+ for _, kv := range globalFlags {
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --param %q: expected key=value", kv)
+ }
+ p.global[k] = v
+ }
+ for _, spec := range perResourceFlags {
+ resource, kv, ok := strings.Cut(spec, ":")
+ if !ok || resource == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ if p.perResource[resource] == nil {
+ p.perResource[resource] = map[string]string{}
+ }
+ p.perResource[resource][k] = v
+ }
+ return p, nil
+}
+
+// applyTo merges user params into the request map. Called after spec-derived
+// params (cursor, since, page-size, dates) so user flags can override them.
+func (p *syncUserParams) applyTo(resource string, params map[string]string) {
+ if p == nil {
+ return
+ }
+ for k, v := range p.global {
+ params[k] = v
+ }
+ for k, v := range p.perResource[resource] {
+ params[k] = v
+ }
+}
+
+// validateResourceNames returns a usage-shaped error when any --resource-param
+// key targets a resource not in known. Without this check a typo
+// (e.g. --resource-param chanels:mine=true) is a silent no-op: the param
+// never matches a real resource and sync proceeds with missing rows.
+func (p *syncUserParams) validateResourceNames(known []string) error {
+ if p == nil || len(p.perResource) == 0 {
+ return nil
+ }
+ set := make(map[string]struct{}, len(known))
+ for _, r := range known {
+ set[r] = struct{}{}
+ }
+ var unknown []string
+ for r := range p.perResource {
+ if _, ok := set[r]; !ok {
+ unknown = append(unknown, r)
+ }
+ }
+ if len(unknown) == 0 {
+ return nil
+ }
+ sort.Strings(unknown)
+ return fmt.Errorf("--resource-param references unknown resource(s): %s (known: %s)",
+ strings.Join(unknown, ", "), strings.Join(known, ", "))
+}
{{- end}}
{{- if and (and .Auth.Type (ne .Auth.Type "none")) .HasDataLayer}}
diff --git a/internal/generator/templates/sync.go.tmpl b/internal/generator/templates/sync.go.tmpl
index e6a6fd16..26c7be21 100644
--- a/internal/generator/templates/sync.go.tmpl
+++ b/internal/generator/templates/sync.go.tmpl
@@ -49,6 +49,8 @@ func newSyncCmd(flags *rootFlags) *cobra.Command {
var maxPages int
var latestOnly bool
var strict bool
+ var paramFlags []string
+ var resourceParamFlags []string
{{- if .Pagination.DateRangeParam}}
var dates string
{{- end}}
@@ -95,6 +97,11 @@ Exit codes & warnings:
# Sync historical data for a date range
{{.Name}}-pp-cli sync --dates 20250101-20250131{{end}}`,
RunE: func(cmd *cobra.Command, args []string) error {
+ userParams, err := parseSyncUserParams(paramFlags, resourceParamFlags)
+ if err != nil {
+ return usageErr(err)
+ }
+
c, err := flags.newClient()
if err != nil {
return err
@@ -122,6 +129,15 @@ Exit codes & warnings:
resources = defaultSyncResources()
}
+ // Reject --resource-param keys that don't match a known resource.
+ // Validates against the full top-level + dependent set, not the
+ // user-filtered `resources` slice, so legitimate cases like
+ // "filter to A, but apply param to B if it gets synced" still
+ // catch typos without false positives.
+ if err := userParams.validateResourceNames(knownSyncResourceNames()); err != nil {
+ return usageErr(err)
+ }
+
// --full: clear all sync cursors before starting
if full {
for _, resource := range resources {
@@ -182,7 +198,7 @@ Exit codes & warnings:
go func() {
defer wg.Done()
for resource := range work {
- res := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, db, resource, sinceTS, full, maxPages{{if .Pagination.DateRangeParam}}, syncDates{{end}})
+ res := syncResource({{if .HasTierRouting}}syncClientForResource(c, resource){{else}}c{{end}}, db, resource, sinceTS, full, maxPages{{if .Pagination.DateRangeParam}}, syncDates{{end}}, userParams)
results <- res
}
}()
@@ -230,7 +246,7 @@ Exit codes & warnings:
{{- if .DependentSyncResources}}
// Sync dependent (parent-child) resources sequentially after flat resources.
- depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter{{if .Pagination.DateRangeParam}}, syncDates{{end}})
+ depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter{{if .Pagination.DateRangeParam}}, syncDates{{end}}, userParams)
for _, res := range depResults {
if res.Err != nil {
if humanFriendly {
@@ -314,6 +330,8 @@ Exit codes & warnings:
cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Maximum pages to fetch per resource (0 = unlimited; cap-hit emits a sync_warning event)")
cmd.Flags().BoolVar(&latestOnly, "latest-only", false, "Refresh head of each resource only; clears resume cursor and caps pages at 1. Mutually exclusive with --since (--since wins).")
cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any per-resource failure (default: only critical failures or all-resource failure exit non-zero).")
+ cmd.Flags().StringArrayVar(¶mFlags, "param", nil, "Extra query param to inject into every sync request (repeatable, key=value). Use for APIs whose spec marks a filter optional but the endpoint rejects calls without it (e.g. --param mine=true). Avoid pagination keys (limit/since/cursor) — overriding them corrupts resume state.")
+ cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param", nil, "Per-resource extra query param (repeatable, resource:key=value). Wins over --param when both define the same key.")
{{- if .Pagination.DateRangeParam}}
cmd.Flags().StringVar(&dates, "dates", "", "Date or date range to sync (passed as {{.Pagination.DateRangeParam}} query parameter)")
{{- end}}
@@ -328,7 +346,7 @@ Exit codes & warnings:
func syncResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, resource, sinceTS string, full bool, maxPages int{{if .Pagination.DateRangeParam}}, dates string{{end}}) syncResult {
+}, db *store.Store, resource, sinceTS string, full bool, maxPages int{{if .Pagination.DateRangeParam}}, dates string{{end}}, userParams *syncUserParams) syncResult {
started := time.Now()
if !humanFriendly {
@@ -444,6 +462,11 @@ func syncResource(c interface {
}
{{- end}}
+ // Apply user-supplied --param / --resource-param overrides last so they
+ // win over spec-derived defaults (e.g. forcing mine=true on a list
+ // endpoint whose OpenAPI spec marks the filter optional).
+ userParams.applyTo(resource, params)
+
data, err := c.Get(path, params)
if err != nil {
if w, ok := isSyncAccessWarning(err); ok {
@@ -454,7 +477,7 @@ func syncResource(c interface {
return syncResult{Resource: resource, Count: totalCount, Warn: fmt.Errorf("skipped %s: %s", resource, w.Reason), Duration: time.Since(started)}
}
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("fetching %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -470,7 +493,7 @@ func syncResource(c interface {
// Single object response - try to store as-is
if err := upsertSingleObject(db, resource, data); err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Err: err, Duration: time.Since(started)}
}
@@ -492,7 +515,7 @@ func syncResource(c interface {
stored, extractFailures, err := upsertResourceBatch(db, resource, items)
if err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("upserting batch for %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -1006,6 +1029,19 @@ func defaultSyncResources() []string {
}
}
+// knownSyncResourceNames returns every resource name sync will accept —
+// flat resources plus any parent-child dependents. Used by --resource-param
+// validation to reject misspellings before they become silent no-ops.
+func knownSyncResourceNames() []string {
+ names := defaultSyncResources()
+{{- if .DependentSyncResources}}
+ for _, dep := range dependentResourceDefs() {
+ names = append(names, dep.Name)
+ }
+{{- end}}
+ return names
+}
+
// syncResourcePath maps resource names to their actual API endpoint paths.
// For REST APIs this is typically "/<resource>". For non-REST APIs (e.g., Steam)
// this preserves the actual endpoint path like "/ISteamApps/GetAppList/v2".
@@ -1075,7 +1111,7 @@ func dependentResourceDefs() []dependentResourceDef {
func syncDependentResources(c {{if .HasTierRouting}}*client.Client{{else}}interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}{{end}}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string{{if .Pagination.DateRangeParam}}, dates string{{end}}) []syncResult {
+}{{end}}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string{{if .Pagination.DateRangeParam}}, dates string{{end}}, userParams *syncUserParams) []syncResult {
allow := make(map[string]bool, len(parentFilter))
for _, r := range parentFilter {
allow[r] = true
@@ -1085,7 +1121,7 @@ func syncDependentResources(c {{if .HasTierRouting}}*client.Client{{else}}interf
if len(allow) > 0 && !allow[dep.ParentTable] && !allow[dep.Name] {
continue
}
- res := syncDependentResource({{if .HasTierRouting}}syncClientForResource(c, dep.Name){{else}}c{{end}}, db, dep, sinceTS, full, maxPages{{if .Pagination.DateRangeParam}}, dates{{end}})
+ res := syncDependentResource({{if .HasTierRouting}}syncClientForResource(c, dep.Name){{else}}c{{end}}, db, dep, sinceTS, full, maxPages{{if .Pagination.DateRangeParam}}, dates{{end}}, userParams)
results = append(results, res)
}
return results
@@ -1095,7 +1131,7 @@ func syncDependentResources(c {{if .HasTierRouting}}*client.Client{{else}}interf
func syncDependentResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int{{if .Pagination.DateRangeParam}}, dates string{{end}}) syncResult {
+}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int{{if .Pagination.DateRangeParam}}, dates string{{end}}, userParams *syncUserParams) syncResult {
started := time.Now()
// Query parent table for the keys to substitute into the child path.
@@ -1173,6 +1209,9 @@ func syncDependentResource(c interface {
}
{{- end}}
+ // Apply user flags last so they win over spec-derived cursor/since/limit.
+ userParams.applyTo(dep.Name, params)
+
data, err := c.Get(path, params)
if err != nil {
// Non-fatal per parent: log and continue to next parent.
@@ -1191,6 +1230,11 @@ func syncDependentResource(c interface {
}
} else if humanFriendly {
fmt.Fprintf(os.Stderr, "\n %s: error for parent %s: %v\n", dep.Name, parentID, err)
+ } else {
+ // Non-warning failures were previously silent in JSON mode —
+ // operators only saw the missing rows. Emit a structured
+ // sync_error so the API body and status are inspectable.
+ fmt.Fprintln(os.Stdout, syncErrorJSON(dep.Name, parentID, err))
}
break
}
diff --git a/testdata/golden/expected/generate-golden-api-rich-auth/printing-press-rich-auth/internal/cli/helpers.go b/testdata/golden/expected/generate-golden-api-rich-auth/printing-press-rich-auth/internal/cli/helpers.go
index 9fec80c6..88a781db 100644
--- a/testdata/golden/expected/generate-golden-api-rich-auth/printing-press-rich-auth/internal/cli/helpers.go
+++ b/testdata/golden/expected/generate-golden-api-rich-auth/printing-press-rich-auth/internal/cli/helpers.go
@@ -132,6 +132,120 @@ type accessWarning struct {
Message string // human-readable detail (the API's body or GraphQL error message)
}
+// syncErrorJSON returns a one-line JSON sync_error event. When err wraps a
+// *client.APIError, the structured status/method/path/body fields let
+// operators see the API's response body without parsing a wrapped error
+// string — required for diagnosing 4xx responses on endpoints whose OpenAPI
+// spec marks filter params optional but the API treats them as required.
+//
+// parent is non-empty only on the dependent-resource path, where it
+// identifies which parent ID's child request failed.
+func syncErrorJSON(resource, parent string, err error) string {
+ payload := struct {
+ Event string `json:"event"`
+ Resource string `json:"resource"`
+ Parent string `json:"parent,omitempty"`
+ Status int `json:"status,omitempty"`
+ Method string `json:"method,omitempty"`
+ Path string `json:"path,omitempty"`
+ Body string `json:"body,omitempty"`
+ Error string `json:"error"`
+ }{
+ Event: "sync_error",
+ Resource: resource,
+ Parent: parent,
+ Error: err.Error(),
+ }
+ var apiErr *client.APIError
+ if errors.As(err, &apiErr) {
+ payload.Status = apiErr.StatusCode
+ payload.Method = apiErr.Method
+ payload.Path = apiErr.Path
+ payload.Body = apiErr.Body
+ }
+ out, _ := json.Marshal(payload)
+ return string(out)
+}
+
+// syncUserParams carries user-supplied query parameters injected into every
+// sync HTTP request. perResource entries win over global on key conflict.
+type syncUserParams struct {
+ global map[string]string
+ perResource map[string]map[string]string
+}
+
+// parseSyncUserParams parses the repeatable --param key=value and
+// --resource-param resource:key=value flags. Returns usage errors keyed on the
+// specific invalid token so the user sees which entry was rejected.
+func parseSyncUserParams(globalFlags, perResourceFlags []string) (*syncUserParams, error) {
+ p := &syncUserParams{
+ global: map[string]string{},
+ perResource: map[string]map[string]string{},
+ }
+ for _, kv := range globalFlags {
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --param %q: expected key=value", kv)
+ }
+ p.global[k] = v
+ }
+ for _, spec := range perResourceFlags {
+ resource, kv, ok := strings.Cut(spec, ":")
+ if !ok || resource == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ if p.perResource[resource] == nil {
+ p.perResource[resource] = map[string]string{}
+ }
+ p.perResource[resource][k] = v
+ }
+ return p, nil
+}
+
+// applyTo merges user params into the request map. Called after spec-derived
+// params (cursor, since, page-size, dates) so user flags can override them.
+func (p *syncUserParams) applyTo(resource string, params map[string]string) {
+ if p == nil {
+ return
+ }
+ for k, v := range p.global {
+ params[k] = v
+ }
+ for k, v := range p.perResource[resource] {
+ params[k] = v
+ }
+}
+
+// validateResourceNames returns a usage-shaped error when any --resource-param
+// key targets a resource not in known. Without this check a typo
+// (e.g. --resource-param chanels:mine=true) is a silent no-op: the param
+// never matches a real resource and sync proceeds with missing rows.
+func (p *syncUserParams) validateResourceNames(known []string) error {
+ if p == nil || len(p.perResource) == 0 {
+ return nil
+ }
+ set := make(map[string]struct{}, len(known))
+ for _, r := range known {
+ set[r] = struct{}{}
+ }
+ var unknown []string
+ for r := range p.perResource {
+ if _, ok := set[r]; !ok {
+ unknown = append(unknown, r)
+ }
+ }
+ if len(unknown) == 0 {
+ return nil
+ }
+ sort.Strings(unknown)
+ return fmt.Errorf("--resource-param references unknown resource(s): %s (known: %s)",
+ strings.Join(unknown, ", "), strings.Join(known, ", "))
+}
+
// accessDenialPatterns matches API error bodies that indicate the request was
// rejected for access-policy reasons rather than for input validity. Matching
// is case-insensitive and uses word boundaries so common substrings inside
diff --git a/testdata/golden/expected/generate-golden-api/dogfood.json b/testdata/golden/expected/generate-golden-api/dogfood.json
index c7ad4ac4..d36d06aa 100644
--- a/testdata/golden/expected/generate-golden-api/dogfood.json
+++ b/testdata/golden/expected/generate-golden-api/dogfood.json
@@ -16,7 +16,7 @@
},
"dead_functions": {
"dead": 0,
- "total": 54
+ "total": 56
},
"dir": "<ARTIFACT_DIR>/generate-golden-api/printing-press-golden",
"example_check": {
diff --git a/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/helpers.go b/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/helpers.go
index 7b15713f..50097e02 100644
--- a/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/helpers.go
+++ b/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/helpers.go
@@ -132,6 +132,120 @@ type accessWarning struct {
Message string // human-readable detail (the API's body or GraphQL error message)
}
+// syncErrorJSON returns a one-line JSON sync_error event. When err wraps a
+// *client.APIError, the structured status/method/path/body fields let
+// operators see the API's response body without parsing a wrapped error
+// string — required for diagnosing 4xx responses on endpoints whose OpenAPI
+// spec marks filter params optional but the API treats them as required.
+//
+// parent is non-empty only on the dependent-resource path, where it
+// identifies which parent ID's child request failed.
+func syncErrorJSON(resource, parent string, err error) string {
+ payload := struct {
+ Event string `json:"event"`
+ Resource string `json:"resource"`
+ Parent string `json:"parent,omitempty"`
+ Status int `json:"status,omitempty"`
+ Method string `json:"method,omitempty"`
+ Path string `json:"path,omitempty"`
+ Body string `json:"body,omitempty"`
+ Error string `json:"error"`
+ }{
+ Event: "sync_error",
+ Resource: resource,
+ Parent: parent,
+ Error: err.Error(),
+ }
+ var apiErr *client.APIError
+ if errors.As(err, &apiErr) {
+ payload.Status = apiErr.StatusCode
+ payload.Method = apiErr.Method
+ payload.Path = apiErr.Path
+ payload.Body = apiErr.Body
+ }
+ out, _ := json.Marshal(payload)
+ return string(out)
+}
+
+// syncUserParams carries user-supplied query parameters injected into every
+// sync HTTP request. perResource entries win over global on key conflict.
+type syncUserParams struct {
+ global map[string]string
+ perResource map[string]map[string]string
+}
+
+// parseSyncUserParams parses the repeatable --param key=value and
+// --resource-param resource:key=value flags. Returns usage errors keyed on the
+// specific invalid token so the user sees which entry was rejected.
+func parseSyncUserParams(globalFlags, perResourceFlags []string) (*syncUserParams, error) {
+ p := &syncUserParams{
+ global: map[string]string{},
+ perResource: map[string]map[string]string{},
+ }
+ for _, kv := range globalFlags {
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --param %q: expected key=value", kv)
+ }
+ p.global[k] = v
+ }
+ for _, spec := range perResourceFlags {
+ resource, kv, ok := strings.Cut(spec, ":")
+ if !ok || resource == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ k, v, ok := strings.Cut(kv, "=")
+ if !ok || k == "" {
+ return nil, fmt.Errorf("invalid --resource-param %q: expected resource:key=value", spec)
+ }
+ if p.perResource[resource] == nil {
+ p.perResource[resource] = map[string]string{}
+ }
+ p.perResource[resource][k] = v
+ }
+ return p, nil
+}
+
+// applyTo merges user params into the request map. Called after spec-derived
+// params (cursor, since, page-size, dates) so user flags can override them.
+func (p *syncUserParams) applyTo(resource string, params map[string]string) {
+ if p == nil {
+ return
+ }
+ for k, v := range p.global {
+ params[k] = v
+ }
+ for k, v := range p.perResource[resource] {
+ params[k] = v
+ }
+}
+
+// validateResourceNames returns a usage-shaped error when any --resource-param
+// key targets a resource not in known. Without this check a typo
+// (e.g. --resource-param chanels:mine=true) is a silent no-op: the param
+// never matches a real resource and sync proceeds with missing rows.
+func (p *syncUserParams) validateResourceNames(known []string) error {
+ if p == nil || len(p.perResource) == 0 {
+ return nil
+ }
+ set := make(map[string]struct{}, len(known))
+ for _, r := range known {
+ set[r] = struct{}{}
+ }
+ var unknown []string
+ for r := range p.perResource {
+ if _, ok := set[r]; !ok {
+ unknown = append(unknown, r)
+ }
+ }
+ if len(unknown) == 0 {
+ return nil
+ }
+ sort.Strings(unknown)
+ return fmt.Errorf("--resource-param references unknown resource(s): %s (known: %s)",
+ strings.Join(unknown, ", "), strings.Join(known, ", "))
+}
+
// accessDenialPatterns matches API error bodies that indicate the request was
// rejected for access-policy reasons rather than for input validity. Matching
// is case-insensitive and uses word boundaries so common substrings inside
diff --git a/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/sync.go b/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/sync.go
index abfee310..56f7ba5e 100644
--- a/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/sync.go
+++ b/testdata/golden/expected/generate-golden-api/printing-press-golden/internal/cli/sync.go
@@ -45,6 +45,8 @@ func newSyncCmd(flags *rootFlags) *cobra.Command {
var maxPages int
var latestOnly bool
var strict bool
+ var paramFlags []string
+ var resourceParamFlags []string
cmd := &cobra.Command{
Use: "sync",
@@ -84,6 +86,11 @@ Exit codes & warnings:
# Latest-only: refresh head of each resource, no historical backfill
printing-press-golden-pp-cli sync --latest-only`,
RunE: func(cmd *cobra.Command, args []string) error {
+ userParams, err := parseSyncUserParams(paramFlags, resourceParamFlags)
+ if err != nil {
+ return usageErr(err)
+ }
+
c, err := flags.newClient()
if err != nil {
return err
@@ -108,6 +115,15 @@ Exit codes & warnings:
resources = defaultSyncResources()
}
+ // Reject --resource-param keys that don't match a known resource.
+ // Validates against the full top-level + dependent set, not the
+ // user-filtered `resources` slice, so legitimate cases like
+ // "filter to A, but apply param to B if it gets synced" still
+ // catch typos without false positives.
+ if err := userParams.validateResourceNames(knownSyncResourceNames()); err != nil {
+ return usageErr(err)
+ }
+
// --full: clear all sync cursors before starting
if full {
for _, resource := range resources {
@@ -163,7 +179,7 @@ Exit codes & warnings:
go func() {
defer wg.Done()
for resource := range work {
- res := syncResource(c, db, resource, sinceTS, full, maxPages)
+ res := syncResource(c, db, resource, sinceTS, full, maxPages, userParams)
results <- res
}
}()
@@ -209,7 +225,7 @@ Exit codes & warnings:
}
}
// Sync dependent (parent-child) resources sequentially after flat resources.
- depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter)
+ depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter, userParams)
for _, res := range depResults {
if res.Err != nil {
if humanFriendly {
@@ -292,6 +308,8 @@ Exit codes & warnings:
cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Maximum pages to fetch per resource (0 = unlimited; cap-hit emits a sync_warning event)")
cmd.Flags().BoolVar(&latestOnly, "latest-only", false, "Refresh head of each resource only; clears resume cursor and caps pages at 1. Mutually exclusive with --since (--since wins).")
cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any per-resource failure (default: only critical failures or all-resource failure exit non-zero).")
+ cmd.Flags().StringArrayVar(¶mFlags, "param", nil, "Extra query param to inject into every sync request (repeatable, key=value). Use for APIs whose spec marks a filter optional but the endpoint rejects calls without it (e.g. --param mine=true). Avoid pagination keys (limit/since/cursor) — overriding them corrupts resume state.")
+ cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param", nil, "Per-resource extra query param (repeatable, resource:key=value). Wins over --param when both define the same key.")
return cmd
}
@@ -303,7 +321,7 @@ Exit codes & warnings:
func syncResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, resource, sinceTS string, full bool, maxPages int) syncResult {
+}, db *store.Store, resource, sinceTS string, full bool, maxPages int, userParams *syncUserParams) syncResult {
started := time.Now()
if !humanFriendly {
@@ -412,6 +430,11 @@ func syncResource(c interface {
params[sinceParam] = effectiveSince
}
+ // Apply user-supplied --param / --resource-param overrides last so they
+ // win over spec-derived defaults (e.g. forcing mine=true on a list
+ // endpoint whose OpenAPI spec marks the filter optional).
+ userParams.applyTo(resource, params)
+
data, err := c.Get(path, params)
if err != nil {
if w, ok := isSyncAccessWarning(err); ok {
@@ -422,7 +445,7 @@ func syncResource(c interface {
return syncResult{Resource: resource, Count: totalCount, Warn: fmt.Errorf("skipped %s: %s", resource, w.Reason), Duration: time.Since(started)}
}
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("fetching %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -438,7 +461,7 @@ func syncResource(c interface {
// Single object response - try to store as-is
if err := upsertSingleObject(db, resource, data); err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Err: err, Duration: time.Since(started)}
}
@@ -460,7 +483,7 @@ func syncResource(c interface {
stored, extractFailures, err := upsertResourceBatch(db, resource, items)
if err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("upserting batch for %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -929,6 +952,17 @@ func defaultSyncResources() []string {
}
}
+// knownSyncResourceNames returns every resource name sync will accept —
+// flat resources plus any parent-child dependents. Used by --resource-param
+// validation to reject misspellings before they become silent no-ops.
+func knownSyncResourceNames() []string {
+ names := defaultSyncResources()
+ for _, dep := range dependentResourceDefs() {
+ names = append(names, dep.Name)
+ }
+ return names
+}
+
// syncResourcePath maps resource names to their actual API endpoint paths.
// For REST APIs this is typically "/<resource>". For non-REST APIs (e.g., Steam)
// this preserves the actual endpoint path like "/ISteamApps/GetAppList/v2".
@@ -969,7 +1003,7 @@ func dependentResourceDefs() []dependentResourceDef {
func syncDependentResources(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string) []syncResult {
+}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string, userParams *syncUserParams) []syncResult {
allow := make(map[string]bool, len(parentFilter))
for _, r := range parentFilter {
allow[r] = true
@@ -979,7 +1013,7 @@ func syncDependentResources(c interface {
if len(allow) > 0 && !allow[dep.ParentTable] && !allow[dep.Name] {
continue
}
- res := syncDependentResource(c, db, dep, sinceTS, full, maxPages)
+ res := syncDependentResource(c, db, dep, sinceTS, full, maxPages, userParams)
results = append(results, res)
}
return results
@@ -989,7 +1023,7 @@ func syncDependentResources(c interface {
func syncDependentResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int) syncResult {
+}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int, userParams *syncUserParams) syncResult {
started := time.Now()
// Query parent table for the keys to substitute into the child path.
@@ -1062,6 +1096,9 @@ func syncDependentResource(c interface {
params[depSinceParam] = depSinceTS
}
+ // Apply user flags last so they win over spec-derived cursor/since/limit.
+ userParams.applyTo(dep.Name, params)
+
data, err := c.Get(path, params)
if err != nil {
// Non-fatal per parent: log and continue to next parent.
@@ -1080,6 +1117,11 @@ func syncDependentResource(c interface {
}
} else if humanFriendly {
fmt.Fprintf(os.Stderr, "\n %s: error for parent %s: %v\n", dep.Name, parentID, err)
+ } else {
+ // Non-warning failures were previously silent in JSON mode —
+ // operators only saw the missing rows. Emit a structured
+ // sync_error so the API body and status are inspectable.
+ fmt.Fprintln(os.Stdout, syncErrorJSON(dep.Name, parentID, err))
}
break
}
diff --git a/testdata/golden/expected/generate-sync-walker-api/sync-walker-golden/internal/cli/sync.go b/testdata/golden/expected/generate-sync-walker-api/sync-walker-golden/internal/cli/sync.go
index 04e9c926..782a636e 100644
--- a/testdata/golden/expected/generate-sync-walker-api/sync-walker-golden/internal/cli/sync.go
+++ b/testdata/golden/expected/generate-sync-walker-api/sync-walker-golden/internal/cli/sync.go
@@ -45,6 +45,8 @@ func newSyncCmd(flags *rootFlags) *cobra.Command {
var maxPages int
var latestOnly bool
var strict bool
+ var paramFlags []string
+ var resourceParamFlags []string
cmd := &cobra.Command{
Use: "sync",
@@ -84,6 +86,11 @@ Exit codes & warnings:
# Latest-only: refresh head of each resource, no historical backfill
sync-walker-golden-pp-cli sync --latest-only`,
RunE: func(cmd *cobra.Command, args []string) error {
+ userParams, err := parseSyncUserParams(paramFlags, resourceParamFlags)
+ if err != nil {
+ return usageErr(err)
+ }
+
c, err := flags.newClient()
if err != nil {
return err
@@ -108,6 +115,15 @@ Exit codes & warnings:
resources = defaultSyncResources()
}
+ // Reject --resource-param keys that don't match a known resource.
+ // Validates against the full top-level + dependent set, not the
+ // user-filtered `resources` slice, so legitimate cases like
+ // "filter to A, but apply param to B if it gets synced" still
+ // catch typos without false positives.
+ if err := userParams.validateResourceNames(knownSyncResourceNames()); err != nil {
+ return usageErr(err)
+ }
+
// --full: clear all sync cursors before starting
if full {
for _, resource := range resources {
@@ -163,7 +179,7 @@ Exit codes & warnings:
go func() {
defer wg.Done()
for resource := range work {
- res := syncResource(c, db, resource, sinceTS, full, maxPages)
+ res := syncResource(c, db, resource, sinceTS, full, maxPages, userParams)
results <- res
}
}()
@@ -209,7 +225,7 @@ Exit codes & warnings:
}
}
// Sync dependent (parent-child) resources sequentially after flat resources.
- depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter)
+ depResults := syncDependentResources(c, db, sinceTS, full, maxPages, parentFilter, userParams)
for _, res := range depResults {
if res.Err != nil {
if humanFriendly {
@@ -292,6 +308,8 @@ Exit codes & warnings:
cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Maximum pages to fetch per resource (0 = unlimited; cap-hit emits a sync_warning event)")
cmd.Flags().BoolVar(&latestOnly, "latest-only", false, "Refresh head of each resource only; clears resume cursor and caps pages at 1. Mutually exclusive with --since (--since wins).")
cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any per-resource failure (default: only critical failures or all-resource failure exit non-zero).")
+ cmd.Flags().StringArrayVar(¶mFlags, "param", nil, "Extra query param to inject into every sync request (repeatable, key=value). Use for APIs whose spec marks a filter optional but the endpoint rejects calls without it (e.g. --param mine=true). Avoid pagination keys (limit/since/cursor) — overriding them corrupts resume state.")
+ cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param", nil, "Per-resource extra query param (repeatable, resource:key=value). Wins over --param when both define the same key.")
return cmd
}
@@ -303,7 +321,7 @@ Exit codes & warnings:
func syncResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, resource, sinceTS string, full bool, maxPages int) syncResult {
+}, db *store.Store, resource, sinceTS string, full bool, maxPages int, userParams *syncUserParams) syncResult {
started := time.Now()
if !humanFriendly {
@@ -412,6 +430,11 @@ func syncResource(c interface {
params[sinceParam] = effectiveSince
}
+ // Apply user-supplied --param / --resource-param overrides last so they
+ // win over spec-derived defaults (e.g. forcing mine=true on a list
+ // endpoint whose OpenAPI spec marks the filter optional).
+ userParams.applyTo(resource, params)
+
data, err := c.Get(path, params)
if err != nil {
if w, ok := isSyncAccessWarning(err); ok {
@@ -422,7 +445,7 @@ func syncResource(c interface {
return syncResult{Resource: resource, Count: totalCount, Warn: fmt.Errorf("skipped %s: %s", resource, w.Reason), Duration: time.Since(started)}
}
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("fetching %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -438,7 +461,7 @@ func syncResource(c interface {
// Single object response - try to store as-is
if err := upsertSingleObject(db, resource, data); err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Err: err, Duration: time.Since(started)}
}
@@ -460,7 +483,7 @@ func syncResource(c interface {
stored, extractFailures, err := upsertResourceBatch(db, resource, items)
if err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("upserting batch for %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -924,6 +947,17 @@ func defaultSyncResources() []string {
}
}
+// knownSyncResourceNames returns every resource name sync will accept —
+// flat resources plus any parent-child dependents. Used by --resource-param
+// validation to reject misspellings before they become silent no-ops.
+func knownSyncResourceNames() []string {
+ names := defaultSyncResources()
+ for _, dep := range dependentResourceDefs() {
+ names = append(names, dep.Name)
+ }
+ return names
+}
+
// syncResourcePath maps resource names to their actual API endpoint paths.
// For REST APIs this is typically "/<resource>". For non-REST APIs (e.g., Steam)
// this preserves the actual endpoint path like "/ISteamApps/GetAppList/v2".
@@ -963,7 +997,7 @@ func dependentResourceDefs() []dependentResourceDef {
func syncDependentResources(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string) []syncResult {
+}, db *store.Store, sinceTS string, full bool, maxPages int, parentFilter []string, userParams *syncUserParams) []syncResult {
allow := make(map[string]bool, len(parentFilter))
for _, r := range parentFilter {
allow[r] = true
@@ -973,7 +1007,7 @@ func syncDependentResources(c interface {
if len(allow) > 0 && !allow[dep.ParentTable] && !allow[dep.Name] {
continue
}
- res := syncDependentResource(c, db, dep, sinceTS, full, maxPages)
+ res := syncDependentResource(c, db, dep, sinceTS, full, maxPages, userParams)
results = append(results, res)
}
return results
@@ -983,7 +1017,7 @@ func syncDependentResources(c interface {
func syncDependentResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int) syncResult {
+}, db *store.Store, dep dependentResourceDef, sinceTS string, full bool, maxPages int, userParams *syncUserParams) syncResult {
started := time.Now()
// Query parent table for the keys to substitute into the child path.
@@ -1056,6 +1090,9 @@ func syncDependentResource(c interface {
params[depSinceParam] = depSinceTS
}
+ // Apply user flags last so they win over spec-derived cursor/since/limit.
+ userParams.applyTo(dep.Name, params)
+
data, err := c.Get(path, params)
if err != nil {
// Non-fatal per parent: log and continue to next parent.
@@ -1074,6 +1111,11 @@ func syncDependentResource(c interface {
}
} else if humanFriendly {
fmt.Fprintf(os.Stderr, "\n %s: error for parent %s: %v\n", dep.Name, parentID, err)
+ } else {
+ // Non-warning failures were previously silent in JSON mode —
+ // operators only saw the missing rows. Emit a structured
+ // sync_error so the API body and status are inspectable.
+ fmt.Fprintln(os.Stdout, syncErrorJSON(dep.Name, parentID, err))
}
break
}
diff --git a/testdata/golden/expected/generate-tier-routing-api/tier-routing-golden/internal/cli/sync.go b/testdata/golden/expected/generate-tier-routing-api/tier-routing-golden/internal/cli/sync.go
index 82cdd353..973de6a2 100644
--- a/testdata/golden/expected/generate-tier-routing-api/tier-routing-golden/internal/cli/sync.go
+++ b/testdata/golden/expected/generate-tier-routing-api/tier-routing-golden/internal/cli/sync.go
@@ -46,6 +46,8 @@ func newSyncCmd(flags *rootFlags) *cobra.Command {
var maxPages int
var latestOnly bool
var strict bool
+ var paramFlags []string
+ var resourceParamFlags []string
cmd := &cobra.Command{
Use: "sync",
@@ -85,6 +87,11 @@ Exit codes & warnings:
# Latest-only: refresh head of each resource, no historical backfill
tier-routing-golden-pp-cli sync --latest-only`,
RunE: func(cmd *cobra.Command, args []string) error {
+ userParams, err := parseSyncUserParams(paramFlags, resourceParamFlags)
+ if err != nil {
+ return usageErr(err)
+ }
+
c, err := flags.newClient()
if err != nil {
return err
@@ -106,6 +113,15 @@ Exit codes & warnings:
resources = defaultSyncResources()
}
+ // Reject --resource-param keys that don't match a known resource.
+ // Validates against the full top-level + dependent set, not the
+ // user-filtered `resources` slice, so legitimate cases like
+ // "filter to A, but apply param to B if it gets synced" still
+ // catch typos without false positives.
+ if err := userParams.validateResourceNames(knownSyncResourceNames()); err != nil {
+ return usageErr(err)
+ }
+
// --full: clear all sync cursors before starting
if full {
for _, resource := range resources {
@@ -161,7 +177,7 @@ Exit codes & warnings:
go func() {
defer wg.Done()
for resource := range work {
- res := syncResource(syncClientForResource(c, resource), db, resource, sinceTS, full, maxPages)
+ res := syncResource(syncClientForResource(c, resource), db, resource, sinceTS, full, maxPages, userParams)
results <- res
}
}()
@@ -266,6 +282,8 @@ Exit codes & warnings:
cmd.Flags().IntVar(&maxPages, "max-pages", 100, "Maximum pages to fetch per resource (0 = unlimited; cap-hit emits a sync_warning event)")
cmd.Flags().BoolVar(&latestOnly, "latest-only", false, "Refresh head of each resource only; clears resume cursor and caps pages at 1. Mutually exclusive with --since (--since wins).")
cmd.Flags().BoolVar(&strict, "strict", false, "Exit non-zero on any per-resource failure (default: only critical failures or all-resource failure exit non-zero).")
+ cmd.Flags().StringArrayVar(¶mFlags, "param", nil, "Extra query param to inject into every sync request (repeatable, key=value). Use for APIs whose spec marks a filter optional but the endpoint rejects calls without it (e.g. --param mine=true). Avoid pagination keys (limit/since/cursor) — overriding them corrupts resume state.")
+ cmd.Flags().StringArrayVar(&resourceParamFlags, "resource-param", nil, "Per-resource extra query param (repeatable, resource:key=value). Wins over --param when both define the same key.")
return cmd
}
@@ -277,7 +295,7 @@ Exit codes & warnings:
func syncResource(c interface {
Get(string, map[string]string) (json.RawMessage, error)
RateLimit() float64
-}, db *store.Store, resource, sinceTS string, full bool, maxPages int) syncResult {
+}, db *store.Store, resource, sinceTS string, full bool, maxPages int, userParams *syncUserParams) syncResult {
started := time.Now()
if !humanFriendly {
@@ -386,6 +404,11 @@ func syncResource(c interface {
params[sinceParam] = effectiveSince
}
+ // Apply user-supplied --param / --resource-param overrides last so they
+ // win over spec-derived defaults (e.g. forcing mine=true on a list
+ // endpoint whose OpenAPI spec marks the filter optional).
+ userParams.applyTo(resource, params)
+
data, err := c.Get(path, params)
if err != nil {
if w, ok := isSyncAccessWarning(err); ok {
@@ -396,7 +419,7 @@ func syncResource(c interface {
return syncResult{Resource: resource, Count: totalCount, Warn: fmt.Errorf("skipped %s: %s", resource, w.Reason), Duration: time.Since(started)}
}
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("fetching %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -412,7 +435,7 @@ func syncResource(c interface {
// Single object response - try to store as-is
if err := upsertSingleObject(db, resource, data); err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Err: err, Duration: time.Since(started)}
}
@@ -434,7 +457,7 @@ func syncResource(c interface {
stored, extractFailures, err := upsertResourceBatch(db, resource, items)
if err != nil {
if !humanFriendly {
- fmt.Fprintf(os.Stdout, `{"event":"sync_error","resource":"%s","error":"%s"}`+"\n", resource, strings.ReplaceAll(err.Error(), `"`, `\"`))
+ fmt.Fprintln(os.Stdout, syncErrorJSON(resource, "", err))
}
return syncResult{Resource: resource, Count: totalCount, Err: fmt.Errorf("upserting batch for %s: %w", resource, err), Duration: time.Since(started)}
}
@@ -896,6 +919,14 @@ func defaultSyncResources() []string {
}
}
+// knownSyncResourceNames returns every resource name sync will accept —
+// flat resources plus any parent-child dependents. Used by --resource-param
+// validation to reject misspellings before they become silent no-ops.
+func knownSyncResourceNames() []string {
+ names := defaultSyncResources()
+ return names
+}
+
// syncResourcePath maps resource names to their actual API endpoint paths.
// For REST APIs this is typically "/<resource>". For non-REST APIs (e.g., Steam)
// this preserves the actual endpoint path like "/ISteamApps/GetAppList/v2".
← cba06db1 fix(cli): gate OAuth refresh-token params on x-oauth-refresh
·
back to Cli Printing Press
·
fix(cli): honor Spec.BasePath in generated client URL constr c6420747 →