← back to Designerwallcoverings
Keep Stroheim breaker failures fail closed
fd91592ef7c9620f13e0aea7a5db1e71f4ba4d88 · 2026-09-02 20:25:51 -0700 · Steve Abrams
Files touched
M scripts/stroheim-onboard/settlement-gate.mjsM scripts/stroheim-onboard/test-settlement-429-breaker.mjsM verification/TK-10933-e2e-proof.json
Diff
commit fd91592ef7c9620f13e0aea7a5db1e71f4ba4d88
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 2 20:25:51 2026 -0700
Keep Stroheim breaker failures fail closed
---
scripts/stroheim-onboard/settlement-gate.mjs | 28 ++++++++++++++++-
.../test-settlement-429-breaker.mjs | 36 +++++++++++++++++++---
verification/TK-10933-e2e-proof.json | 7 +++--
3 files changed, 63 insertions(+), 8 deletions(-)
diff --git a/scripts/stroheim-onboard/settlement-gate.mjs b/scripts/stroheim-onboard/settlement-gate.mjs
index 4dd2849..2e63f6e 100644
--- a/scripts/stroheim-onboard/settlement-gate.mjs
+++ b/scripts/stroheim-onboard/settlement-gate.mjs
@@ -35,7 +35,9 @@ const ITEM_TIMEOUT_MS = Math.max(100, parseInt(process.env.STROHEIM_SETTLEMENT_I
const TEST_MODE = process.env.STROHEIM_SETTLEMENT_TEST_MODE || '';
const ATTEMPT_LOG = path.join(OUT, 'settlement-attempts.jsonl');
const BREAKER_STATE = path.join(OUT, 'settlement-429-breaker.json');
+const BREAKER_INVALID = path.join(OUT, 'settlement-429-breaker.invalid');
const RUN_LOCK = path.join(OUT, 'settlement-gate.lock');
+const LOCK_REFUSAL_LOG = path.join(OUT, 'settlement-lock-refusals.jsonl');
const SOFT_429_ITEM_LIMIT = Math.max(1, parseInt(process.env.STROHEIM_SETTLEMENT_429_ITEM_LIMIT || '4', 10));
const SOFT_429_COOLDOWN_MS = Math.max(0, parseInt(process.env.STROHEIM_SETTLEMENT_429_COOLDOWN_MS || '900000', 10));
const RUN_ID = process.env.STROHEIM_SETTLEMENT_RUN_ID || `${new Date().toISOString()}-pid${process.pid}`;
@@ -66,6 +68,11 @@ const appendAttempt = rec => {
const defaultBreaker = () => ({ version: 1, hard_quota_open: false, consecutive_terminal_429_items: 0, cooldown_until: null, reason: null });
const loadBreaker = () => {
try {
+ if (fs.existsSync(BREAKER_INVALID)) {
+ const error = new Error('breaker state is marked invalid; explicit local repair required');
+ error.code = 'BREAKER_STATE_INVALID';
+ throw error;
+ }
const state = JSON.parse(fs.readFileSync(BREAKER_STATE, 'utf8'));
if (state?.version !== 1 || typeof state.hard_quota_open !== 'boolean' ||
!Number.isInteger(state.consecutive_terminal_429_items) || state.consecutive_terminal_429_items < 0 ||
@@ -76,14 +83,32 @@ const loadBreaker = () => {
return { ...defaultBreaker(), ...state };
} catch (error) {
if (error?.code === 'ENOENT') return defaultBreaker();
+ if (error?.code === 'BREAKER_STATE_INVALID') throw error;
const quarantine = `${BREAKER_STATE}.invalid.${Date.now()}.${process.pid}`;
try { fs.renameSync(BREAKER_STATE, quarantine); } catch { /* preserve original failure */ }
+ fs.writeFileSync(BREAKER_INVALID, JSON.stringify({ version: 1, status: 'invalid',
+ repair_required: true, detected_at: new Date().toISOString() }) + '\n', { mode: 0o600 });
const wrapped = new Error('breaker state unreadable or invalid; quarantined and refusing work');
wrapped.code = 'BREAKER_STATE_INVALID';
wrapped.quarantine = quarantine;
throw wrapped;
}
};
+const lockOwnerState = () => {
+ try {
+ const owner = JSON.parse(fs.readFileSync(path.join(RUN_LOCK, 'owner.json'), 'utf8'));
+ const pid = Number.isInteger(owner?.pid) && owner.pid > 0 ? owner.pid : null;
+ if (!pid) return { owner_state: 'unknown', owner_pid: null };
+ try { process.kill(pid, 0); return { owner_state: 'live', owner_pid: pid }; }
+ catch (error) { return { owner_state: error?.code === 'ESRCH' ? 'dead' : 'unknown', owner_pid: pid }; }
+ } catch { return { owner_state: 'unknown', owner_pid: null }; }
+};
+const appendLockRefusal = () => {
+ const owner = lockOwnerState();
+ fs.appendFileSync(LOCK_REFUSAL_LOG, JSON.stringify({ at: new Date().toISOString(), run_id: RUN_ID,
+ status: 'lock_refused', ...owner }) + '\n', { mode: 0o600 });
+ return owner;
+};
const saveBreaker = state => {
fs.mkdirSync(OUT, { recursive: true });
const safe = {
@@ -295,7 +320,8 @@ async function main() {
lockHeld = acquireRunLock();
if (!lockHeld) {
- console.error('settlement gate already running; refusing concurrent work');
+ const owner = appendLockRefusal();
+ console.error(`settlement gate lock held (${owner.owner_state} owner); refusing concurrent work`);
process.exitCode = 45;
return;
}
diff --git a/scripts/stroheim-onboard/test-settlement-429-breaker.mjs b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
index bade4b9..21f7483 100644
--- a/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
+++ b/scripts/stroheim-onboard/test-settlement-429-breaker.mjs
@@ -19,7 +19,11 @@ const run = (out, mode, extra = {}) => spawnSync(process.execPath, [gate, '--app
env: { ...process.env, STROHEIM_SETTLEMENT_OUT: out, STROHEIM_SETTLEMENT_TEST_MODE: mode,
STROHEIM_SETTLEMENT_429_COOLDOWN_MS: '60000', ...extra }
});
-const events = out => fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
+const events = out => {
+ const file = path.join(out, 'settlement-attempts.jsonl');
+ if (!fs.existsSync(file)) return [];
+ return fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse);
+};
const hardOut = makeOut('hard');
const hard = run(hardOut, 'hard-quota-429');
@@ -89,6 +93,13 @@ assert.equal(fs.readFileSync(path.join(corruptOut, 'settlement-verdicts.jsonl'),
const quarantined = fs.readdirSync(corruptOut).filter(name => name.startsWith('settlement-429-breaker.json.invalid.'));
assert.equal(quarantined.length, 1);
assert(!fs.readFileSync(path.join(corruptOut, 'settlement-attempts.jsonl'), 'utf8').includes('SECRET_SHOULD_NOT_APPEAR'));
+assert.deepEqual(JSON.parse(fs.readFileSync(path.join(corruptOut, 'settlement-429-breaker.invalid'), 'utf8')).repair_required, true);
+const corruptResume = run(corruptOut, 'success');
+assert.equal(corruptResume.status, 44);
+const corruptResumeEvents = events(corruptOut);
+assert.equal(corruptResumeEvents.filter(e => e.status === 'item_started').length, 0);
+assert.equal(corruptResumeEvents.filter(e => e.status === 'provider_attempt').length, 0);
+assert.equal(fs.readFileSync(path.join(corruptOut, 'settlement-verdicts.jsonl'), 'utf8'), '');
const concurrentOut = makeOut('concurrent', 2);
const spawnRun = () => new Promise(resolve => {
@@ -107,12 +118,29 @@ assert.equal(concurrentEvents.filter(e => e.status === 'item_started').length, 2
assert.equal(concurrentEvents.filter(e => e.status === 'gemini_started').length, 2);
assert.equal(concurrentEvents.filter(e => e.status === 'timeout').length, 2);
assert(!fs.existsSync(path.join(concurrentOut, 'settlement-gate.lock')));
+const concurrentRefusals = fs.readFileSync(path.join(concurrentOut, 'settlement-lock-refusals.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
+assert.equal(concurrentRefusals.length, 1);
+assert.equal(concurrentRefusals[0].status, 'lock_refused');
+assert.equal(concurrentRefusals[0].owner_state, 'live');
+
+const staleOut = makeOut('stale', 1);
+fs.mkdirSync(path.join(staleOut, 'settlement-gate.lock'));
+fs.writeFileSync(path.join(staleOut, 'settlement-gate.lock', 'owner.json'), JSON.stringify({ pid: 2147483647 }) + '\n');
+const stale1 = run(staleOut, 'success');
+const stale2 = run(staleOut, 'success');
+assert.deepEqual([stale1.status, stale2.status], [45, 45]);
+const staleRefusals = fs.readFileSync(path.join(staleOut, 'settlement-lock-refusals.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
+assert.equal(staleRefusals.length, 2);
+assert(staleRefusals.every(e => e.status === 'lock_refused' && e.owner_state === 'dead' && e.owner_pid === 2147483647));
+assert.equal(events(staleOut).filter(e => e.status === 'item_started' || e.status === 'provider_attempt').length, 0);
-for (const out of [hardOut, softOut, resetOut, non429Out, corruptOut, concurrentOut]) {
- const raw = fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8');
+for (const out of [hardOut, softOut, resetOut, non429Out, corruptOut, concurrentOut, staleOut]) {
+ const attemptFile = path.join(out, 'settlement-attempts.jsonl');
+ const raw = fs.existsSync(attemptFile) ? fs.readFileSync(attemptFile, 'utf8') : '';
assert(!/SECRET_SHOULD_NOT_APPEAR|Requests per minute|Credits depleted|billing quota exhausted/.test(raw));
}
console.log(JSON.stringify({ verdict: 'PASS', hard_exit: hard.status, soft_exit: soft.status,
hard_resume_exit: hardResume.status, soft_provider_attempts: 24, resume_exit: resume.status,
- corrupt_exit: corrupt.status, concurrent_exits: concurrentCodes,
+ corrupt_exits: [corrupt.status, corruptResume.status], concurrent_exits: concurrentCodes,
+ stale_lock_exits: [stale1.status, stale2.status],
success_reset_exit: reset.status, non_429_reset_exit: non429.status }, null, 2));
diff --git a/verification/TK-10933-e2e-proof.json b/verification/TK-10933-e2e-proof.json
index c18033b..98b4979 100644
--- a/verification/TK-10933-e2e-proof.json
+++ b/verification/TK-10933-e2e-proof.json
@@ -4,7 +4,7 @@
"risk_tier": "R1 isolated breaker code for this increment; retained R4 evidence documents the prior approved one-item canary",
"environment": "local Node.js process; deterministic no-network breaker/diagnostic/watchdog mocks; no Shopify path",
"baseline_commit": "6c4a2daac58648d7a7d53471f611502f7b374bbf",
- "timestamp": "2026-09-03T03:24:00Z",
+ "timestamp": "2026-09-03T03:31:00Z",
"breaker_increment_baseline_commit": "3639736b4de692d77c24edaeda842ffb76a3ce6f",
"precondition": {
"canonical_payloads": 771,
@@ -26,8 +26,9 @@
"durable versioned breaker state contains only hard-open flag, count, cooldown, reason, update timestamp, and no provider message, prompt, image, or secret",
"hard-open state survives restart: the second invocation exits 42 without starting an item or making another provider attempt",
"an immediate idempotent resume while cooldown is open exits 43 without starting or attempting another item",
- "unreadable or invalid state is quarantined, records only a redacted local-state event, starts no item, writes no verdict, and exits 44",
- "atomic run exclusion permits one process to handle two mocked watchdog items while its concurrent peer exits 45; no duplicate attempts occur and normal exit removes the lock",
+ "unreadable or invalid state is quarantined and creates a durable repair-required sentinel; both the initial run and its resume exit 44 with no item, provider attempt, or verdict",
+ "atomic run exclusion permits one process to handle two mocked watchdog items while its concurrent peer exits 45; no duplicate attempts occur, a separate redacted refusal ledger identifies the owner as live, and normal exit removes the lock",
+ "a deliberately stale lock remains fail-closed: two successive runs exit 45 with zero work and append two redacted refusal events identifying the recorded owner PID as dead; the lock is not auto-removed",
"a later non-429 provider error or success resets the consecutive count; success still writes its normal verdict",
"all fixtures and breaker state are isolated under OS temporary directories"
],
← bff26d1 Harden Stroheim settlement breaker state
·
back to Designerwallcoverings
·
auto-data-snapshot: 2026-09-02T21:48:54 (2 data files) — scr 9ad2e76 →