← back to Designerwallcoverings
Add Stroheim Gemini 429 circuit breaker
795876a4f0845f8bfa50fdfba9a13f20450f54ee · 2026-09-02 20:15:45 -0700 · Steve Abrams
Files touched
M scripts/stroheim-onboard/settlement-gate.mjsA scripts/stroheim-onboard/test-settlement-429-breaker.mjsM verification/TK-10933-e2e-proof.json
Diff
commit 795876a4f0845f8bfa50fdfba9a13f20450f54ee
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 2 20:15:45 2026 -0700
Add Stroheim Gemini 429 circuit breaker
---
scripts/stroheim-onboard/settlement-gate.mjs | 93 +++++++++++++++++++++-
.../test-settlement-429-breaker.mjs | 79 ++++++++++++++++++
verification/TK-10933-e2e-proof.json | 26 ++++--
3 files changed, 189 insertions(+), 9 deletions(-)
diff --git a/scripts/stroheim-onboard/settlement-gate.mjs b/scripts/stroheim-onboard/settlement-gate.mjs
index a1519e1..75ac393 100644
--- a/scripts/stroheim-onboard/settlement-gate.mjs
+++ b/scripts/stroheim-onboard/settlement-gate.mjs
@@ -34,6 +34,9 @@ const MODEL = 'gemini-2.5-flash';
const ITEM_TIMEOUT_MS = Math.max(100, parseInt(process.env.STROHEIM_SETTLEMENT_ITEM_TIMEOUT_MS || '180000', 10));
const TEST_MODE = process.env.STROHEIM_SETTLEMENT_TEST_MODE || '';
const ATTEMPT_LOG = path.join(OUT, 'settlement-attempts.jsonl');
+const BREAKER_STATE = path.join(OUT, 'settlement-429-breaker.json');
+const SOFT_429_ITEM_LIMIT = Math.max(1, parseInt(process.env.STROHEIM_SETTLEMENT_429_ITEM_LIMIT || '4', 10));
+const SOFT_429_COOLDOWN_MS = Math.max(0, parseInt(process.env.STROHEIM_SETTLEMENT_429_COOLDOWN_MS || '900000', 10));
const RUN_ID = process.env.STROHEIM_SETTLEMENT_RUN_ID || `${new Date().toISOString()}-pid${process.pid}`;
const KEY = TEST_MODE ? 'test-only' : (fs.readFileSync(path.join(process.env.HOME, 'Projects/secrets-manager/.env'), 'utf8')
@@ -59,6 +62,26 @@ const appendAttempt = rec => {
fs.mkdirSync(OUT, { recursive: true });
fs.appendFileSync(ATTEMPT_LOG, JSON.stringify({ at: new Date().toISOString(), run_id: RUN_ID, model: MODEL, ...rec }) + '\n');
};
+const defaultBreaker = () => ({ version: 1, consecutive_terminal_429_items: 0, cooldown_until: null, reason: null });
+const loadBreaker = () => {
+ try {
+ const state = JSON.parse(fs.readFileSync(BREAKER_STATE, 'utf8'));
+ return { ...defaultBreaker(), ...state };
+ } catch { return defaultBreaker(); }
+};
+const saveBreaker = state => {
+ fs.mkdirSync(OUT, { recursive: true });
+ const safe = {
+ version: 1,
+ consecutive_terminal_429_items: state.consecutive_terminal_429_items,
+ cooldown_until: state.cooldown_until,
+ reason: state.reason,
+ updated_at: new Date().toISOString()
+ };
+ const tmp = `${BREAKER_STATE}.${process.pid}.tmp`;
+ fs.writeFileSync(tmp, JSON.stringify(safe, null, 2) + '\n', { mode: 0o600 });
+ fs.renameSync(tmp, BREAKER_STATE);
+};
const combinedSignal = (outer, timeoutMs) => AbortSignal.any([outer, AbortSignal.timeout(timeoutMs)]);
const PROMPT = `You are a legal image auditor applying a narrow trademark SETTLEMENT to a wallcovering image.
@@ -100,9 +123,21 @@ function mockGeminiResponse(mode) {
if (mode === 'empty-candidates') return { status: 200, json: { candidates: [], promptFeedback: { blockReason: 'OTHER' } } };
if (mode === 'provider-error') return { status: 400, json: { error: { code: 400, status: 'INVALID_ARGUMENT', message: 'SECRET_SHOULD_NOT_APPEAR' } } };
if (mode === 'success') return { status: 200, json: { candidates: [{ finishReason: 'STOP', content: { parts: [{ text: JSON.stringify({ a1: true, a2: true, a3: false, b: false, acceptable: false, evidence: {} }) }] } }] } };
+ if (mode === 'soft-429') return { status: 429, json: { error: { code: 429, status: 'RESOURCE_EXHAUSTED', message: 'Requests per minute exceeded' } } };
+ if (mode === 'hard-quota-429') return { status: 429, json: { error: { code: 429, status: 'RESOURCE_EXHAUSTED', message: 'Credits depleted; billing quota exhausted' } } };
return null;
}
+function classify429(j) {
+ const status = String(j?.error?.status || '').toUpperCase();
+ const message = String(j?.error?.message || '').toLowerCase();
+ const hard = /credit|billing|prepay|payment|quota exhausted|quota depleted|insufficient quota/.test(message);
+ return {
+ category: hard ? 'hard_quota_exhausted' : 'soft_rate_limited',
+ provider_status: status.slice(0, 60) || null
+ };
+}
+
async function gemini(b64, outerSignal, sku) {
if (TEST_MODE === 'hang') {
await new Promise((resolve, reject) => {
@@ -122,10 +157,15 @@ async function gemini(b64, outerSignal, sku) {
const finishReasons = candidates.map(c => c?.finishReason).filter(Boolean).slice(0, 4);
const promptBlockReason = j?.promptFeedback?.blockReason ? String(j.promptFeedback.blockReason).slice(0, 60) : null;
if (httpStatus === 429 || httpStatus >= 500) {
- lastCategory = httpStatus === 429 ? 'rate_limited' : 'provider_server_error';
+ const rateLimit = httpStatus === 429 ? classify429(j) : null;
+ lastCategory = rateLimit?.category || 'provider_server_error';
appendAttempt({ sku, status: 'provider_attempt', attempt: t + 1, http_status: httpStatus,
- provider_category: lastCategory, candidate_count: candidates.length, finish_reasons: finishReasons,
+ provider_category: lastCategory, provider_status: rateLimit?.provider_status || null,
+ candidate_count: candidates.length, finish_reasons: finishReasons,
prompt_block_reason: promptBlockReason });
+ if (rateLimit?.category === 'hard_quota_exhausted') {
+ return { vision: null, retryCount: t + 1, terminalCategory: lastCategory, hardQuota: true };
+ }
if (!TEST_MODE) await sleep(1500 * (t + 1), outerSignal);
continue;
}
@@ -156,7 +196,8 @@ async function gemini(b64, outerSignal, sku) {
continue;
} // ECONNRESET / timeout → retry
}
- return { vision: null, retryCount: 6, terminalCategory: lastCategory };
+ return { vision: null, retryCount: 6, terminalCategory: lastCategory,
+ terminal429: lastCategory === 'soft_rate_limited' };
}
// defendant-favorable, fail-closed
@@ -209,6 +250,19 @@ async function main() {
if (payloads.length === 0) { console.log(' ℹ 0 payloads (price not yet sourced) — nothing to gate. $0.'); return; }
if (!APPLY) { console.log(` first 3: ${todo.slice(0, 3).map(p => p.sku).join(', ')}`); console.log('DRY-RUN. --apply to gate.'); return; }
+ let breaker = loadBreaker();
+ if (breaker.cooldown_until && Date.parse(breaker.cooldown_until) > Date.now()) {
+ appendAttempt({ status: 'breaker_open', provider_category: 'soft_rate_limited', cooldown_until: breaker.cooldown_until,
+ consecutive_terminal_429_items: breaker.consecutive_terminal_429_items });
+ console.error(`soft 429 breaker open until ${breaker.cooldown_until}; no items attempted`);
+ process.exitCode = 43;
+ return;
+ }
+ if (breaker.cooldown_until) {
+ breaker = defaultBreaker();
+ saveBreaker(breaker);
+ }
+
const fd = fs.openSync(path.join(OUT, 'settlement-verdicts.jsonl'), 'a');
let ok = 0, block = 0, skip = 0, spent = 0;
for (const p of todo) {
@@ -228,6 +282,39 @@ async function main() {
continue;
}
const v = outcome.value.vision; spent += outcome.value.estimatedCost;
+ if (outcome.value.hardQuota) {
+ appendAttempt({ sku: p.sku, status: 'hard_quota_abort', provider_category: 'hard_quota_exhausted',
+ retry_count: outcome.value.retryCount, billing_status: 'provider_billing_unverified' });
+ fs.closeSync(fd);
+ console.error('hard Gemini quota/credits exhaustion; aborting batch immediately');
+ process.exitCode = 42;
+ return;
+ }
+ if (outcome.value.terminal429) {
+ skip++;
+ breaker.consecutive_terminal_429_items += 1;
+ breaker.reason = 'soft_rate_limited';
+ appendAttempt({ sku: p.sku, status: 'terminal_429_item', provider_category: 'soft_rate_limited',
+ retry_count: outcome.value.retryCount, consecutive_terminal_429_items: breaker.consecutive_terminal_429_items,
+ billing_status: 'provider_billing_unverified' });
+ if (breaker.consecutive_terminal_429_items >= SOFT_429_ITEM_LIMIT) {
+ breaker.cooldown_until = new Date(Date.now() + SOFT_429_COOLDOWN_MS).toISOString();
+ saveBreaker(breaker);
+ appendAttempt({ status: 'breaker_tripped', provider_category: 'soft_rate_limited',
+ consecutive_terminal_429_items: breaker.consecutive_terminal_429_items, cooldown_until: breaker.cooldown_until });
+ fs.closeSync(fd);
+ console.error(`soft 429 breaker tripped after ${breaker.consecutive_terminal_429_items} consecutive terminal-429 items`);
+ process.exitCode = 43;
+ return;
+ }
+ saveBreaker(breaker);
+ continue;
+ }
+ if (breaker.consecutive_terminal_429_items || breaker.cooldown_until) {
+ breaker = defaultBreaker();
+ saveBreaker(breaker);
+ appendAttempt({ sku: p.sku, status: 'breaker_reset', provider_category: outcome.value.terminalCategory });
+ }
const hasAll = v && ['a1', 'a2', 'a3', 'b'].every(k => typeof v[k] === 'boolean');
if (!hasAll) {
skip++;
diff --git a/scripts/stroheim-onboard/test-settlement-429-breaker.mjs b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
new file mode 100644
index 0000000..d2d74ea
--- /dev/null
+++ b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const here = path.dirname(new URL(import.meta.url).pathname);
+const gate = path.join(here, 'settlement-gate.mjs');
+const makeOut = (name, count = 5) => {
+ const out = fs.mkdtempSync(path.join(os.tmpdir(), `stroheim-429-${name}-`));
+ fs.writeFileSync(path.join(out, 'payloads.jsonl'), Array.from({ length: count }, (_, i) =>
+ JSON.stringify({ sku: `TEST-${name.toUpperCase()}-${i + 1}`, image_url: 'mock://image' })).join('\n') + '\n');
+ fs.writeFileSync(path.join(out, 'settlement-verdicts.jsonl'), '');
+ return out;
+};
+const run = (out, mode, extra = {}) => spawnSync(process.execPath, [gate, '--apply'], {
+ encoding: 'utf8', timeout: 3000,
+ env: { ...process.env, STROHEIM_SETTLEMENT_OUT: out, STROHEIM_SETTLEMENT_TEST_MODE: mode,
+ STROHEIM_SETTLEMENT_429_COOLDOWN_MS: '60000', ...extra }
+});
+const events = out => fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
+
+const hardOut = makeOut('hard');
+const hard = run(hardOut, 'hard-quota-429');
+assert.equal(hard.status, 42, hard.stderr);
+const hardEvents = events(hardOut);
+assert.equal(hardEvents.filter(e => e.status === 'provider_attempt').length, 1);
+assert.equal(hardEvents.filter(e => e.status === 'item_started').length, 1);
+assert.equal(hardEvents.at(-1).status, 'hard_quota_abort');
+assert(!fs.existsSync(path.join(hardOut, 'settlement-429-breaker.json')));
+
+const softOut = makeOut('soft');
+const soft = run(softOut, 'soft-429');
+assert.equal(soft.status, 43, soft.stderr);
+const softEvents = events(softOut);
+assert.equal(softEvents.filter(e => e.status === 'provider_attempt').length, 24);
+assert.equal(softEvents.filter(e => e.status === 'terminal_429_item').length, 4);
+assert.equal(softEvents.filter(e => e.status === 'item_started').length, 4);
+assert.equal(softEvents.at(-1).status, 'breaker_tripped');
+const state = JSON.parse(fs.readFileSync(path.join(softOut, 'settlement-429-breaker.json'), 'utf8'));
+assert.equal(state.consecutive_terminal_429_items, 4);
+assert.equal(state.reason, 'soft_rate_limited');
+assert.deepEqual(Object.keys(state).sort(), ['consecutive_terminal_429_items', 'cooldown_until', 'reason', 'updated_at', 'version']);
+
+const resume = run(softOut, 'success');
+assert.equal(resume.status, 43);
+const afterResume = events(softOut);
+assert.equal(afterResume.filter(e => e.status === 'item_started').length, 4, 'open breaker attempted another item');
+assert.equal(afterResume.at(-1).status, 'breaker_open');
+
+const resetOut = makeOut('reset', 1);
+fs.writeFileSync(path.join(resetOut, 'settlement-429-breaker.json'), JSON.stringify({
+ version: 1, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
+}) + '\n');
+const reset = run(resetOut, 'success');
+assert.equal(reset.status, 0, reset.stderr);
+const resetState = JSON.parse(fs.readFileSync(path.join(resetOut, 'settlement-429-breaker.json'), 'utf8'));
+assert.equal(resetState.consecutive_terminal_429_items, 0);
+assert.equal(events(resetOut).filter(e => e.status === 'breaker_reset').length, 1);
+assert.equal(fs.readFileSync(path.join(resetOut, 'settlement-verdicts.jsonl'), 'utf8').trim().split('\n').length, 1);
+
+const non429Out = makeOut('non429', 1);
+fs.writeFileSync(path.join(non429Out, 'settlement-429-breaker.json'), JSON.stringify({
+ version: 1, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
+}) + '\n');
+const non429 = run(non429Out, 'provider-error');
+assert.equal(non429.status, 0, non429.stderr);
+const non429State = JSON.parse(fs.readFileSync(path.join(non429Out, 'settlement-429-breaker.json'), 'utf8'));
+assert.equal(non429State.consecutive_terminal_429_items, 0);
+assert.equal(events(non429Out).filter(e => e.status === 'breaker_reset').length, 1);
+
+for (const out of [hardOut, softOut, resetOut, non429Out]) {
+ const raw = fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8');
+ assert(!/SECRET_SHOULD_NOT_APPEAR|Requests per minute|Credits depleted|billing quota exhausted/.test(raw));
+}
+console.log(JSON.stringify({ verdict: 'PASS', hard_exit: hard.status, soft_exit: soft.status,
+ soft_provider_attempts: 24, resume_exit: resume.status, success_reset_exit: reset.status,
+ non_429_reset_exit: non429.status }, null, 2));
diff --git a/verification/TK-10933-e2e-proof.json b/verification/TK-10933-e2e-proof.json
index e6485d0..a9f0944 100644
--- a/verification/TK-10933-e2e-proof.json
+++ b/verification/TK-10933-e2e-proof.json
@@ -1,10 +1,11 @@
{
"ticket": "TK-10933",
- "intent": "Bound each Stroheim settlement item, durably record attempt and cost state, and verify the guard with a strictly one-item real-network canary without Shopify writes.",
- "risk_tier": "R4 approved one-item external integration canary",
- "environment": "local Node.js process; one prior real bounded Gemini item plus deterministic local diagnostics mocks; no Shopify path",
+ "intent": "Bound each Stroheim settlement item, durably record attempt and cost state, and stop futile Gemini HTTP-429 batches with a deterministic hybrid circuit breaker.",
+ "risk_tier": "R1 isolated breaker code for this increment; retained R4 evidence documents the prior approved one-item canary",
+ "environment": "local Node.js process; deterministic no-network breaker/diagnostic/watchdog mocks; no Shopify path",
"baseline_commit": "6c4a2daac58648d7a7d53471f611502f7b374bbf",
- "timestamp": "2026-08-30T15:11:23Z",
+ "timestamp": "2026-09-03T03:15:02Z",
+ "breaker_increment_baseline_commit": "3639736b4de692d77c24edaeda842ffb76a3ce6f",
"precondition": {
"canonical_payloads": 771,
"canonical_verdicts": 300,
@@ -15,6 +16,19 @@
"approved_estimated_cost_usd": 0.0011
},
"checks": [
+ {
+ "name": "hybrid HTTP-429 circuit breaker operational boundary",
+ "command": "node scripts/stroheim-onboard/test-settlement-429-breaker.mjs",
+ "assertions": [
+ "hard credit/billing quota exhaustion makes one provider attempt on one item, writes a redacted hard_quota_abort event, and exits 42",
+ "soft/RPM 429 retains six per-item attempts and trips after four consecutive terminal-429 items (24 provider attempts), then exits 43",
+ "durable versioned breaker state contains only count, cooldown, reason, update timestamp, and no provider message, prompt, image, or secret",
+ "an immediate idempotent resume while cooldown is open exits 43 without starting or attempting another item",
+ "a later non-429 provider error or success resets the consecutive count; success still writes its normal verdict",
+ "all fixtures and breaker state are isolated under OS temporary directories"
+ ],
+ "verdict": "PASS"
+ },
{
"name": "syntax",
"command": "node --check settlement-gate.mjs and test-settlement-watchdog.mjs",
@@ -89,7 +103,7 @@
"verdict": "PASS"
}
],
- "negative_error_retry": "The mock never resolves until AbortController cancellation. Five repeated runs prove bounded cancellation and terminal event durability.",
- "cleanup": "Mock fixtures were isolated under OS temporary directories. The real attempt ledger was intentionally retained as durable evidence; canonical payload/verdict data was unchanged and no Shopify state was created.",
+ "negative_error_retry": "Breaker mocks prove hard abort, bounded soft retries, threshold trip, cooldown resume suppression, and reset on non-429/success. The watchdog mock still proves bounded cancellation and terminal durability.",
+ "cleanup": "All new breaker fixtures were isolated under OS temporary directories. Canonical hashes remained payloads 3f1312d2739b0e14b9af187928676214a881454722eda9b4274045a2880b342c and verdicts 0626263cd586a28504f1fdb549622c9ccf828bc9ac8ab89720b912e4390aecad; no network, Gemini call, spend, Shopify state, launchd action, deploy, publish, or other external side effect occurred in this increment.",
"verdict": "PASS"
}
← 3639736 auto-data-snapshot: 2026-09-02T18:02:03 (1 data files) — dat
·
back to Designerwallcoverings
·
Harden Stroheim settlement breaker state bff26d1 →