← back to Designerwallcoverings
Harden Stroheim settlement breaker state
bff26d16d5a1b70f7b192ec57be8d2ecc75ce124 · 2026-09-02 20:21:29 -0700 · Steve Abrams
Files touched
M scripts/stroheim-onboard/settlement-gate.mjsM scripts/stroheim-onboard/test-settlement-429-breaker.mjsM verification/TK-10933-e2e-proof.json
Diff
commit bff26d16d5a1b70f7b192ec57be8d2ecc75ce124
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 2 20:21:29 2026 -0700
Harden Stroheim settlement breaker state
---
scripts/stroheim-onboard/settlement-gate.mjs | 73 +++++++++++++++++++++-
.../test-settlement-429-breaker.mjs | 55 +++++++++++++---
verification/TK-10933-e2e-proof.json | 8 ++-
3 files changed, 123 insertions(+), 13 deletions(-)
diff --git a/scripts/stroheim-onboard/settlement-gate.mjs b/scripts/stroheim-onboard/settlement-gate.mjs
index 75ac393..4dd2849 100644
--- a/scripts/stroheim-onboard/settlement-gate.mjs
+++ b/scripts/stroheim-onboard/settlement-gate.mjs
@@ -35,6 +35,7 @@ const ITEM_TIMEOUT_MS = Math.max(100, parseInt(process.env.STROHEIM_SETTLEMENT_I
const TEST_MODE = process.env.STROHEIM_SETTLEMENT_TEST_MODE || '';
const ATTEMPT_LOG = path.join(OUT, 'settlement-attempts.jsonl');
const BREAKER_STATE = path.join(OUT, 'settlement-429-breaker.json');
+const RUN_LOCK = path.join(OUT, 'settlement-gate.lock');
const SOFT_429_ITEM_LIMIT = Math.max(1, parseInt(process.env.STROHEIM_SETTLEMENT_429_ITEM_LIMIT || '4', 10));
const SOFT_429_COOLDOWN_MS = Math.max(0, parseInt(process.env.STROHEIM_SETTLEMENT_429_COOLDOWN_MS || '900000', 10));
const RUN_ID = process.env.STROHEIM_SETTLEMENT_RUN_ID || `${new Date().toISOString()}-pid${process.pid}`;
@@ -62,17 +63,32 @@ const appendAttempt = rec => {
fs.mkdirSync(OUT, { recursive: true });
fs.appendFileSync(ATTEMPT_LOG, JSON.stringify({ at: new Date().toISOString(), run_id: RUN_ID, model: MODEL, ...rec }) + '\n');
};
-const defaultBreaker = () => ({ version: 1, consecutive_terminal_429_items: 0, cooldown_until: null, reason: null });
+const defaultBreaker = () => ({ version: 1, hard_quota_open: false, consecutive_terminal_429_items: 0, cooldown_until: null, reason: null });
const loadBreaker = () => {
try {
const state = JSON.parse(fs.readFileSync(BREAKER_STATE, 'utf8'));
+ if (state?.version !== 1 || typeof state.hard_quota_open !== 'boolean' ||
+ !Number.isInteger(state.consecutive_terminal_429_items) || state.consecutive_terminal_429_items < 0 ||
+ !(state.cooldown_until === null || typeof state.cooldown_until === 'string') ||
+ !(state.reason === null || ['soft_rate_limited', 'hard_quota_exhausted'].includes(state.reason))) {
+ throw new Error('invalid breaker state schema');
+ }
return { ...defaultBreaker(), ...state };
- } catch { return defaultBreaker(); }
+ } catch (error) {
+ if (error?.code === 'ENOENT') return defaultBreaker();
+ const quarantine = `${BREAKER_STATE}.invalid.${Date.now()}.${process.pid}`;
+ try { fs.renameSync(BREAKER_STATE, quarantine); } catch { /* preserve original failure */ }
+ const wrapped = new Error('breaker state unreadable or invalid; quarantined and refusing work');
+ wrapped.code = 'BREAKER_STATE_INVALID';
+ wrapped.quarantine = quarantine;
+ throw wrapped;
+ }
};
const saveBreaker = state => {
fs.mkdirSync(OUT, { recursive: true });
const safe = {
version: 1,
+ hard_quota_open: Boolean(state.hard_quota_open),
consecutive_terminal_429_items: state.consecutive_terminal_429_items,
cooldown_until: state.cooldown_until,
reason: state.reason,
@@ -82,6 +98,33 @@ const saveBreaker = state => {
fs.writeFileSync(tmp, JSON.stringify(safe, null, 2) + '\n', { mode: 0o600 });
fs.renameSync(tmp, BREAKER_STATE);
};
+const acquireRunLock = () => {
+ fs.mkdirSync(OUT, { recursive: true });
+ try {
+ fs.mkdirSync(RUN_LOCK, { mode: 0o700 });
+ try {
+ fs.writeFileSync(path.join(RUN_LOCK, 'owner.json'), JSON.stringify({ pid: process.pid, run_id: RUN_ID,
+ started_at: new Date().toISOString() }) + '\n', { mode: 0o600 });
+ } catch (error) {
+ try { fs.rmdirSync(RUN_LOCK); } catch { /* original error is controlling */ }
+ throw error;
+ }
+ return true;
+ } catch (error) {
+ if (error?.code === 'EEXIST') return false;
+ throw error;
+ }
+};
+let lockHeld = false;
+const releaseRunLock = () => {
+ if (!lockHeld) return;
+ try { fs.unlinkSync(path.join(RUN_LOCK, 'owner.json')); } catch { /* best effort on process exit */ }
+ try { fs.rmdirSync(RUN_LOCK); } catch { /* fail closed next run if cleanup is incomplete */ }
+ lockHeld = false;
+};
+process.on('exit', releaseRunLock);
+process.once('SIGINT', () => { releaseRunLock(); process.exit(130); });
+process.once('SIGTERM', () => { releaseRunLock(); process.exit(143); });
const combinedSignal = (outer, timeoutMs) => AbortSignal.any([outer, AbortSignal.timeout(timeoutMs)]);
const PROMPT = `You are a legal image auditor applying a narrow trademark SETTLEMENT to a wallcovering image.
@@ -250,7 +293,29 @@ async function main() {
if (payloads.length === 0) { console.log(' ℹ 0 payloads (price not yet sourced) — nothing to gate. $0.'); return; }
if (!APPLY) { console.log(` first 3: ${todo.slice(0, 3).map(p => p.sku).join(', ')}`); console.log('DRY-RUN. --apply to gate.'); return; }
- let breaker = loadBreaker();
+ lockHeld = acquireRunLock();
+ if (!lockHeld) {
+ console.error('settlement gate already running; refusing concurrent work');
+ process.exitCode = 45;
+ return;
+ }
+
+ let breaker;
+ try {
+ breaker = loadBreaker();
+ } catch (error) {
+ appendAttempt({ status: 'breaker_state_invalid', provider_category: 'local_state_error',
+ quarantine_file: path.basename(error.quarantine || '') });
+ console.error(error.message);
+ process.exitCode = 44;
+ return;
+ }
+ if (breaker.hard_quota_open) {
+ appendAttempt({ status: 'hard_quota_open', provider_category: 'hard_quota_exhausted' });
+ console.error('hard Gemini quota breaker open; no items attempted');
+ process.exitCode = 42;
+ return;
+ }
if (breaker.cooldown_until && Date.parse(breaker.cooldown_until) > Date.now()) {
appendAttempt({ status: 'breaker_open', provider_category: 'soft_rate_limited', cooldown_until: breaker.cooldown_until,
consecutive_terminal_429_items: breaker.consecutive_terminal_429_items });
@@ -283,6 +348,8 @@ async function main() {
}
const v = outcome.value.vision; spent += outcome.value.estimatedCost;
if (outcome.value.hardQuota) {
+ breaker = { ...defaultBreaker(), hard_quota_open: true, reason: 'hard_quota_exhausted' };
+ saveBreaker(breaker);
appendAttempt({ sku: p.sku, status: 'hard_quota_abort', provider_category: 'hard_quota_exhausted',
retry_count: outcome.value.retryCount, billing_status: 'provider_billing_unverified' });
fs.closeSync(fd);
diff --git a/scripts/stroheim-onboard/test-settlement-429-breaker.mjs b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
index d2d74ea..bade4b9 100644
--- a/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
+++ b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
-import { spawnSync } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
const here = path.dirname(new URL(import.meta.url).pathname);
const gate = path.join(here, 'settlement-gate.mjs');
@@ -28,7 +28,14 @@ const hardEvents = events(hardOut);
assert.equal(hardEvents.filter(e => e.status === 'provider_attempt').length, 1);
assert.equal(hardEvents.filter(e => e.status === 'item_started').length, 1);
assert.equal(hardEvents.at(-1).status, 'hard_quota_abort');
-assert(!fs.existsSync(path.join(hardOut, 'settlement-429-breaker.json')));
+const hardState = JSON.parse(fs.readFileSync(path.join(hardOut, 'settlement-429-breaker.json'), 'utf8'));
+assert.equal(hardState.hard_quota_open, true);
+const hardResume = run(hardOut, 'success');
+assert.equal(hardResume.status, 42);
+const hardResumeEvents = events(hardOut);
+assert.equal(hardResumeEvents.filter(e => e.status === 'provider_attempt').length, 1);
+assert.equal(hardResumeEvents.filter(e => e.status === 'item_started').length, 1);
+assert.equal(hardResumeEvents.at(-1).status, 'hard_quota_open');
const softOut = makeOut('soft');
const soft = run(softOut, 'soft-429');
@@ -41,7 +48,7 @@ assert.equal(softEvents.at(-1).status, 'breaker_tripped');
const state = JSON.parse(fs.readFileSync(path.join(softOut, 'settlement-429-breaker.json'), 'utf8'));
assert.equal(state.consecutive_terminal_429_items, 4);
assert.equal(state.reason, 'soft_rate_limited');
-assert.deepEqual(Object.keys(state).sort(), ['consecutive_terminal_429_items', 'cooldown_until', 'reason', 'updated_at', 'version']);
+assert.deepEqual(Object.keys(state).sort(), ['consecutive_terminal_429_items', 'cooldown_until', 'hard_quota_open', 'reason', 'updated_at', 'version']);
const resume = run(softOut, 'success');
assert.equal(resume.status, 43);
@@ -51,7 +58,7 @@ assert.equal(afterResume.at(-1).status, 'breaker_open');
const resetOut = makeOut('reset', 1);
fs.writeFileSync(path.join(resetOut, 'settlement-429-breaker.json'), JSON.stringify({
- version: 1, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
+ version: 1, hard_quota_open: false, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
}) + '\n');
const reset = run(resetOut, 'success');
assert.equal(reset.status, 0, reset.stderr);
@@ -62,7 +69,7 @@ assert.equal(fs.readFileSync(path.join(resetOut, 'settlement-verdicts.jsonl'), '
const non429Out = makeOut('non429', 1);
fs.writeFileSync(path.join(non429Out, 'settlement-429-breaker.json'), JSON.stringify({
- version: 1, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
+ version: 1, hard_quota_open: false, consecutive_terminal_429_items: 3, cooldown_until: null, reason: 'soft_rate_limited'
}) + '\n');
const non429 = run(non429Out, 'provider-error');
assert.equal(non429.status, 0, non429.stderr);
@@ -70,10 +77,42 @@ const non429State = JSON.parse(fs.readFileSync(path.join(non429Out, 'settlement-
assert.equal(non429State.consecutive_terminal_429_items, 0);
assert.equal(events(non429Out).filter(e => e.status === 'breaker_reset').length, 1);
-for (const out of [hardOut, softOut, resetOut, non429Out]) {
+const corruptOut = makeOut('corrupt', 1);
+fs.writeFileSync(path.join(corruptOut, 'settlement-429-breaker.json'), '{broken SECRET_SHOULD_NOT_APPEAR');
+const corrupt = run(corruptOut, 'success');
+assert.equal(corrupt.status, 44);
+const corruptEvents = events(corruptOut);
+assert.equal(corruptEvents.at(-1).status, 'breaker_state_invalid');
+assert.equal(corruptEvents.filter(e => e.status === 'item_started').length, 0);
+assert.equal(corruptEvents.filter(e => e.status === 'provider_attempt').length, 0);
+assert.equal(fs.readFileSync(path.join(corruptOut, 'settlement-verdicts.jsonl'), 'utf8'), '');
+const quarantined = fs.readdirSync(corruptOut).filter(name => name.startsWith('settlement-429-breaker.json.invalid.'));
+assert.equal(quarantined.length, 1);
+assert(!fs.readFileSync(path.join(corruptOut, 'settlement-attempts.jsonl'), 'utf8').includes('SECRET_SHOULD_NOT_APPEAR'));
+
+const concurrentOut = makeOut('concurrent', 2);
+const spawnRun = () => new Promise(resolve => {
+ const child = spawn(process.execPath, [gate, '--apply'], { env: { ...process.env,
+ STROHEIM_SETTLEMENT_OUT: concurrentOut, STROHEIM_SETTLEMENT_TEST_MODE: 'hang',
+ STROHEIM_SETTLEMENT_ITEM_TIMEOUT_MS: '300' } });
+ child.on('close', code => resolve(code));
+});
+const firstPromise = spawnRun();
+await new Promise(resolve => setTimeout(resolve, 50));
+const secondPromise = spawnRun();
+const concurrentCodes = await Promise.all([firstPromise, secondPromise]);
+assert.deepEqual([...concurrentCodes].sort((a, b) => a - b), [0, 45]);
+const concurrentEvents = events(concurrentOut);
+assert.equal(concurrentEvents.filter(e => e.status === 'item_started').length, 2);
+assert.equal(concurrentEvents.filter(e => e.status === 'gemini_started').length, 2);
+assert.equal(concurrentEvents.filter(e => e.status === 'timeout').length, 2);
+assert(!fs.existsSync(path.join(concurrentOut, 'settlement-gate.lock')));
+
+for (const out of [hardOut, softOut, resetOut, non429Out, corruptOut, concurrentOut]) {
const raw = fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8');
assert(!/SECRET_SHOULD_NOT_APPEAR|Requests per minute|Credits depleted|billing quota exhausted/.test(raw));
}
console.log(JSON.stringify({ verdict: 'PASS', hard_exit: hard.status, soft_exit: soft.status,
- soft_provider_attempts: 24, resume_exit: resume.status, success_reset_exit: reset.status,
- non_429_reset_exit: non429.status }, null, 2));
+ hard_resume_exit: hardResume.status, soft_provider_attempts: 24, resume_exit: resume.status,
+ corrupt_exit: corrupt.status, concurrent_exits: concurrentCodes,
+ success_reset_exit: reset.status, non_429_reset_exit: non429.status }, null, 2));
diff --git a/verification/TK-10933-e2e-proof.json b/verification/TK-10933-e2e-proof.json
index a9f0944..c18033b 100644
--- a/verification/TK-10933-e2e-proof.json
+++ b/verification/TK-10933-e2e-proof.json
@@ -4,7 +4,7 @@
"risk_tier": "R1 isolated breaker code for this increment; retained R4 evidence documents the prior approved one-item canary",
"environment": "local Node.js process; deterministic no-network breaker/diagnostic/watchdog mocks; no Shopify path",
"baseline_commit": "6c4a2daac58648d7a7d53471f611502f7b374bbf",
- "timestamp": "2026-09-03T03:15:02Z",
+ "timestamp": "2026-09-03T03:24:00Z",
"breaker_increment_baseline_commit": "3639736b4de692d77c24edaeda842ffb76a3ce6f",
"precondition": {
"canonical_payloads": 771,
@@ -12,6 +12,7 @@
"canonical_verdict_sha256": "0626263cd586a28504f1fdb549622c9ccf828bc9ac8ab89720b912e4390aecad",
"shopify_writes_authorized_for_test": false,
"external_gemini_calls_authorized_for_test": true,
+ "external_gemini_calls_authorized_for_current_increment": false,
"approved_item_limit": 1,
"approved_estimated_cost_usd": 0.0011
},
@@ -22,8 +23,11 @@
"assertions": [
"hard credit/billing quota exhaustion makes one provider attempt on one item, writes a redacted hard_quota_abort event, and exits 42",
"soft/RPM 429 retains six per-item attempts and trips after four consecutive terminal-429 items (24 provider attempts), then exits 43",
- "durable versioned breaker state contains only count, cooldown, reason, update timestamp, and no provider message, prompt, image, or secret",
+ "durable versioned breaker state contains only hard-open flag, count, cooldown, reason, update timestamp, and no provider message, prompt, image, or secret",
+ "hard-open state survives restart: the second invocation exits 42 without starting an item or making another provider attempt",
"an immediate idempotent resume while cooldown is open exits 43 without starting or attempting another item",
+ "unreadable or invalid state is quarantined, records only a redacted local-state event, starts no item, writes no verdict, and exits 44",
+ "atomic run exclusion permits one process to handle two mocked watchdog items while its concurrent peer exits 45; no duplicate attempts occur and normal exit removes the lock",
"a later non-429 provider error or success resets the consecutive count; success still writes its normal verdict",
"all fixtures and breaker state are isolated under OS temporary directories"
],
← 795876a Add Stroheim Gemini 429 circuit breaker
·
back to Designerwallcoverings
·
Keep Stroheim breaker failures fail closed fd91592 →