← back to Designerwallcoverings
fix Stroheim settlement hangs with bounded watchdog
2211117bc67d9c104b53b8c3b5cdc400c82299c7 · 2026-08-29 17:33:27 -0700 · Steve Abrams
Files touched
M scripts/stroheim-onboard/settlement-gate.mjsA scripts/stroheim-onboard/test-settlement-watchdog.mjsA verification/TK-10933-e2e-proof.json
Diff
commit 2211117bc67d9c104b53b8c3b5cdc400c82299c7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Aug 29 17:33:27 2026 -0700
fix Stroheim settlement hangs with bounded watchdog
---
scripts/stroheim-onboard/settlement-gate.mjs | 113 +++++++++++++++++----
.../stroheim-onboard/test-settlement-watchdog.mjs | 45 ++++++++
verification/TK-10933-e2e-proof.json | 63 ++++++++++++
3 files changed, 204 insertions(+), 17 deletions(-)
diff --git a/scripts/stroheim-onboard/settlement-gate.mjs b/scripts/stroheim-onboard/settlement-gate.mjs
index 383e5ed..a00e650 100644
--- a/scripts/stroheim-onboard/settlement-gate.mjs
+++ b/scripts/stroheim-onboard/settlement-gate.mjs
@@ -23,22 +23,42 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
const HERE = path.dirname(fileURLToPath(import.meta.url));
-const OUT = path.join(HERE, 'out');
+const OUT = process.env.STROHEIM_SETTLEMENT_OUT || path.join(HERE, 'out');
const APPLY = process.argv.includes('--apply');
const LIMIT = parseInt((process.argv.find(a => a.startsWith('--limit=')) || '').split('=')[1] || '0', 10);
const ONLY = (process.argv.find(a => a.startsWith('--sku=')) || '').split('=')[1];
const ONLY_SET = ONLY ? new Set(ONLY.split(',').map(s => s.trim().toUpperCase())) : null;
const COST_PER_IMG = 0.0011;
const MODEL = 'gemini-2.5-flash';
+const ITEM_TIMEOUT_MS = Math.max(100, parseInt(process.env.STROHEIM_SETTLEMENT_ITEM_TIMEOUT_MS || '180000', 10));
+const TEST_MODE = process.env.STROHEIM_SETTLEMENT_TEST_MODE || '';
+const ATTEMPT_LOG = path.join(OUT, 'settlement-attempts.jsonl');
+const RUN_ID = process.env.STROHEIM_SETTLEMENT_RUN_ID || `${new Date().toISOString()}-pid${process.pid}`;
-const KEY = (fs.readFileSync(path.join(process.env.HOME, 'Projects/secrets-manager/.env'), 'utf8')
+const KEY = TEST_MODE ? 'test-only' : (fs.readFileSync(path.join(process.env.HOME, 'Projects/secrets-manager/.env'), 'utf8')
.match(/^GEMINI_API_KEY=(.+)$/m) || [])[1]?.trim();
if (!KEY) { console.error('no GEMINI_API_KEY in ~/Projects/secrets-manager/.env'); process.exit(1); }
const URL = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${KEY}`;
const loadJsonl = f => fs.existsSync(path.join(OUT, f))
? fs.readFileSync(path.join(OUT, f), 'utf8').trim().split('\n').filter(Boolean).map(l => JSON.parse(l)) : [];
-const sleep = ms => new Promise(r => setTimeout(r, ms));
+const sleep = (ms, signal) => new Promise((resolve, reject) => {
+ if (signal?.aborted) return reject(signal.reason);
+ const timer = setTimeout(() => {
+ signal?.removeEventListener('abort', onAbort);
+ resolve();
+ }, ms);
+ const onAbort = () => {
+ clearTimeout(timer);
+ reject(signal.reason);
+ };
+ signal?.addEventListener('abort', onAbort, { once: true });
+});
+const appendAttempt = rec => {
+ fs.mkdirSync(OUT, { recursive: true });
+ fs.appendFileSync(ATTEMPT_LOG, JSON.stringify({ at: new Date().toISOString(), run_id: RUN_ID, model: MODEL, ...rec }) + '\n');
+};
+const combinedSignal = (outer, timeoutMs) => AbortSignal.any([outer, AbortSignal.timeout(timeoutMs)]);
const PROMPT = `You are a legal image auditor applying a narrow trademark SETTLEMENT to a wallcovering image.
Answer ONLY these five yes/no questions about the LEAF / FOLIAGE layer of the pattern, then return strict JSON.
@@ -50,31 +70,44 @@ Acceptable (carveout): Does it clearly contain tree trunks, clearly-drawn branch
Respond with ONLY this JSON, booleans true/false (never null), plus a 3-6 word evidence string each:
{"a1":bool,"a2":bool,"a3":bool,"b":bool,"acceptable":bool,"evidence":{"a1":"","a2":"","a3":"","b":"","acceptable":""}}`;
-async function fetchB64(url) {
+async function fetchB64(url, outerSignal) {
+ if (TEST_MODE === 'hang') return { mime: 'image/jpeg', data: 'dGVzdA==' };
for (let t = 0; t < 4; t++) {
try {
- const r = await fetch(url, { signal: AbortSignal.timeout(30000) });
+ const r = await fetch(url, { signal: combinedSignal(outerSignal, 30000) });
if (!r.ok) throw new Error(`img ${r.status}`);
const ct = r.headers.get('content-type') || 'image/jpeg';
const buf = Buffer.from(await r.arrayBuffer());
return { mime: ct.split(';')[0], data: buf.toString('base64') };
- } catch (e) { if (t === 3) throw e; await sleep(800 * (t + 1)); } // retry transient img-fetch (ECONNRESET/timeout)
+ } catch (e) {
+ if (outerSignal.aborted || t === 3) throw e;
+ await sleep(800 * (t + 1), outerSignal);
+ } // retry transient img-fetch (ECONNRESET/timeout)
}
}
-async function gemini(b64) {
+async function gemini(b64, outerSignal) {
+ if (TEST_MODE === 'hang') {
+ await new Promise((resolve, reject) => {
+ outerSignal.addEventListener('abort', () => reject(outerSignal.reason), { once: true });
+ });
+ }
const body = { contents: [{ parts: [{ text: PROMPT }, { inline_data: { mime_type: b64.mime, data: b64.data } }] }],
generationConfig: { temperature: 0, responseMimeType: 'application/json', thinkingConfig: { thinkingBudget: 0 } } };
for (let t = 0; t < 6; t++) {
try {
- const r = await fetch(URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: AbortSignal.timeout(45000) });
- if (r.status === 429 || r.status >= 500) { await sleep(1500 * (t + 1)); continue; }
+ const r = await fetch(URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: combinedSignal(outerSignal, 45000) });
+ if (r.status === 429 || r.status >= 500) { await sleep(1500 * (t + 1), outerSignal); continue; }
const j = await r.json();
const txt = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
let parsed = null; try { parsed = JSON.parse(txt); } catch { parsed = null; }
if (parsed) return parsed;
- await sleep(1000 * (t + 1)); continue; // empty/unparseable (transient) → retry, do NOT fail-closed on first blip
- } catch (e) { await sleep(1000 * (t + 1)); continue; } // ECONNRESET / timeout → retry
+ await sleep(1000 * (t + 1), outerSignal); continue; // empty/unparseable (transient) → retry, do NOT fail-closed on first blip
+ } catch (e) {
+ if (outerSignal.aborted) throw e;
+ await sleep(1000 * (t + 1), outerSignal);
+ continue;
+ } // ECONNRESET / timeout → retry
}
return null;
}
@@ -90,6 +123,34 @@ function verdict(v) {
return { verdict: 'OK', reason: 'no violation (Part B absent or Part A partial)', ...v };
}
+async function processItem(p, signal, progress) {
+ const b64 = await fetchB64(p.image_url, signal);
+ progress.geminiStarted = true;
+ appendAttempt({ sku: p.sku, status: 'gemini_started', estimated_cost: COST_PER_IMG });
+ const vision = await gemini(b64, signal);
+ return { vision, estimatedCost: COST_PER_IMG };
+}
+
+async function processItemBounded(p) {
+ const controller = new AbortController();
+ const progress = { geminiStarted: false };
+ let timer;
+ const work = processItem(p, controller.signal, progress).then(
+ value => ({ kind: 'result', value }),
+ error => ({ kind: 'error', error })
+ );
+ const timeout = new Promise(resolve => {
+ timer = setTimeout(() => {
+ controller.abort(new Error(`item watchdog exceeded ${ITEM_TIMEOUT_MS}ms`));
+ resolve({ kind: 'timeout' });
+ }, ITEM_TIMEOUT_MS);
+ });
+ const outcome = await Promise.race([work, timeout]);
+ clearTimeout(timer);
+ if (outcome.kind !== 'result') controller.abort(outcome.error);
+ return { ...outcome, progress };
+}
+
async function main() {
const payloads = loadJsonl('payloads.jsonl');
const done = new Map(loadJsonl('settlement-verdicts.jsonl').map(r => [r.sku.toUpperCase(), r]));
@@ -97,21 +158,39 @@ async function main() {
if (ONLY_SET) todo = todo.filter(p => ONLY_SET.has(p.sku.toUpperCase()));
if (LIMIT > 0) todo = todo.slice(0, LIMIT);
- console.log(`settlement-gate · model ${MODEL} · payloads ${payloads.length} · already-gated ${done.size} · to gate ${todo.length} · est $${(todo.length * COST_PER_IMG).toFixed(4)} · ${APPLY ? 'APPLY' : 'DRY-RUN'}`);
+ console.log(`settlement-gate · model ${MODEL} · payloads ${payloads.length} · already-gated ${done.size} · to gate ${todo.length} · est $${(todo.length * COST_PER_IMG).toFixed(4)} · item-watchdog ${ITEM_TIMEOUT_MS}ms · ${APPLY ? 'APPLY' : 'DRY-RUN'}`);
if (payloads.length === 0) { console.log(' ℹ 0 payloads (price not yet sourced) — nothing to gate. $0.'); return; }
if (!APPLY) { console.log(` first 3: ${todo.slice(0, 3).map(p => p.sku).join(', ')}`); console.log('DRY-RUN. --apply to gate.'); return; }
const fd = fs.openSync(path.join(OUT, 'settlement-verdicts.jsonl'), 'a');
let ok = 0, block = 0, skip = 0, spent = 0;
for (const p of todo) {
- let b64;
- try { b64 = await fetchB64(p.image_url); }
- catch (e) { skip++; await sleep(400); continue; } // img-fetch failed after retries → skip, retry next pass
- const v = await gemini(b64); spent += COST_PER_IMG;
+ appendAttempt({ sku: p.sku, status: 'item_started', estimated_cost: 0, timeout_ms: ITEM_TIMEOUT_MS });
+ const outcome = await processItemBounded(p);
+ if (outcome.kind === 'timeout') {
+ skip++;
+ appendAttempt({ sku: p.sku, status: 'timeout', estimated_cost: outcome.progress.geminiStarted ? COST_PER_IMG : 0,
+ billing_status: outcome.progress.geminiStarted ? 'unknown_after_abort' : 'not_started', error: `item watchdog exceeded ${ITEM_TIMEOUT_MS}ms` });
+ continue;
+ }
+ if (outcome.kind === 'error') {
+ skip++;
+ appendAttempt({ sku: p.sku, status: 'error', estimated_cost: outcome.progress.geminiStarted ? COST_PER_IMG : 0,
+ billing_status: outcome.progress.geminiStarted ? 'unknown_after_error' : 'not_started', error: String(outcome.error?.message || outcome.error) });
+ await sleep(400);
+ continue;
+ }
+ const v = outcome.value.vision; spent += outcome.value.estimatedCost;
const hasAll = v && ['a1', 'a2', 'a3', 'b'].every(k => typeof v[k] === 'boolean');
- if (!hasAll) { skip++; await sleep(400); continue; } // vision null/malformed → SKIP (not BLOCK), retry next pass
+ if (!hasAll) {
+ skip++;
+ appendAttempt({ sku: p.sku, status: 'vision_unparseable', estimated_cost: outcome.value.estimatedCost });
+ await sleep(400);
+ continue;
+ }
const rec = { sku: p.sku, ...verdict(v), gated_at: new Date().toISOString() };
fs.writeSync(fd, JSON.stringify(rec) + '\n');
+ appendAttempt({ sku: p.sku, status: rec.verdict.toLowerCase(), estimated_cost: outcome.value.estimatedCost });
if (rec.verdict === 'OK') ok++; else block++;
if ((ok + block + skip) % 20 === 0) console.log(` ...${ok + block + skip}/${todo.length} (OK ${ok}, BLOCK ${block}, skip ${skip}) · $${spent.toFixed(4)}`);
await sleep(400);
diff --git a/scripts/stroheim-onboard/test-settlement-watchdog.mjs b/scripts/stroheim-onboard/test-settlement-watchdog.mjs
new file mode 100644
index 0000000..298b78c
--- /dev/null
+++ b/scripts/stroheim-onboard/test-settlement-watchdog.mjs
@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const here = path.dirname(new URL(import.meta.url).pathname);
+const out = fs.mkdtempSync(path.join(os.tmpdir(), 'stroheim-watchdog-'));
+const payloads = [
+ { sku: 'TEST-HANG-1', image_url: 'mock://hang-1' },
+ { sku: 'TEST-HANG-2', image_url: 'mock://hang-2' }
+];
+fs.writeFileSync(path.join(out, 'payloads.jsonl'), payloads.map(JSON.stringify).join('\n') + '\n');
+fs.writeFileSync(path.join(out, 'settlement-verdicts.jsonl'), '');
+
+const started = Date.now();
+const run = spawnSync(process.execPath, [path.join(here, 'settlement-gate.mjs'), '--apply', '--limit=2'], {
+ encoding: 'utf8',
+ timeout: 3000,
+ env: {
+ ...process.env,
+ STROHEIM_SETTLEMENT_OUT: out,
+ STROHEIM_SETTLEMENT_TEST_MODE: 'hang',
+ STROHEIM_SETTLEMENT_ITEM_TIMEOUT_MS: '100'
+ }
+});
+const elapsed = Date.now() - started;
+assert.equal(run.status, 0, `process failed: ${run.stderr}`);
+assert(elapsed < 1500, `watchdog run was not bounded: ${elapsed}ms`);
+assert.match(run.stdout, /skipped\(retry-next-pass\) 2/);
+
+const attempts = fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8')
+ .trim().split('\n').map(JSON.parse);
+for (const sku of payloads.map(p => p.sku)) {
+ const events = attempts.filter(a => a.sku === sku);
+ assert.deepEqual(events.map(e => e.status), ['item_started', 'gemini_started', 'timeout']);
+ assert.equal(events[0].estimated_cost, 0);
+ assert.equal(events[1].estimated_cost, 0.0011);
+ assert.equal(events[2].estimated_cost, 0.0011);
+ assert.equal(events[2].billing_status, 'unknown_after_abort');
+ assert.match(events[2].error, /watchdog exceeded 100ms/);
+}
+assert.equal(fs.readFileSync(path.join(out, 'settlement-verdicts.jsonl'), 'utf8'), '');
+console.log(JSON.stringify({ verdict: 'PASS', elapsed_ms: elapsed, attempts: attempts.length, out }, null, 2));
diff --git a/verification/TK-10933-e2e-proof.json b/verification/TK-10933-e2e-proof.json
new file mode 100644
index 0000000..871a3fa
--- /dev/null
+++ b/verification/TK-10933-e2e-proof.json
@@ -0,0 +1,63 @@
+{
+ "ticket": "TK-10933",
+ "intent": "Bound each Stroheim settlement item, abort hung work, and durably record attempt and cost state without external spend or Shopify writes.",
+ "risk_tier": "R3 local proof of an R4 external integration guard",
+ "environment": "local deterministic mock; Node.js; no network calls",
+ "baseline_commit": "6c4a2daac58648d7a7d53471f611502f7b374bbf",
+ "timestamp": "2026-08-30T00:32:40Z",
+ "precondition": {
+ "canonical_payloads": 771,
+ "canonical_verdicts": 300,
+ "canonical_verdict_sha256": "0626263cd586a28504f1fdb549622c9ccf828bc9ac8ab89720b912e4390aecad",
+ "shopify_writes_authorized_for_test": false,
+ "external_gemini_calls_authorized_for_test": false
+ },
+ "checks": [
+ {
+ "name": "syntax",
+ "command": "node --check settlement-gate.mjs and test-settlement-watchdog.mjs",
+ "verdict": "PASS"
+ },
+ {
+ "name": "hung item watchdog and batch advancement",
+ "command": "node scripts/stroheim-onboard/test-settlement-watchdog.mjs (5 repetitions)",
+ "assertions": [
+ "two independently hung items each abort at a 100ms outer deadline",
+ "the batch advances from the first hung item to the second",
+ "the process exits 0 in 238-246ms",
+ "no settlement verdict is written"
+ ],
+ "verdict": "PASS"
+ },
+ {
+ "name": "durable attempt and cost state",
+ "assertions": [
+ "each item writes item_started before work",
+ "each item writes gemini_started with estimated_cost 0.0011 before the mocked call",
+ "each item writes timeout with billing_status unknown_after_abort and the watchdog error"
+ ],
+ "verdict": "PASS"
+ },
+ {
+ "name": "canonical negative boundary",
+ "command": "node settlement-gate.mjs --sku=TK-10933-NONEXISTENT",
+ "assertions": [
+ "DRY-RUN reports zero selected items",
+ "canonical settlement verdict SHA-256 remains unchanged"
+ ],
+ "verdict": "PASS"
+ },
+ {
+ "name": "external side effects",
+ "assertions": [
+ "Gemini calls: 0",
+ "Shopify products/variants created: 0/0",
+ "activation/deploy: 0"
+ ],
+ "verdict": "PASS"
+ }
+ ],
+ "negative_error_retry": "The mock never resolves until AbortController cancellation. Five repeated runs prove bounded cancellation and terminal event durability.",
+ "cleanup": "Fixtures and ledgers were isolated under OS temporary directories; canonical verdict data was unchanged.",
+ "verdict": "PASS"
+}
← 6c4a2da auto-data-snapshot: 2026-08-29T14:38:58 (1 data files) — dat
·
back to Designerwallcoverings
·
auto-data-snapshot: 2026-08-29T20:09:38 (1 data files) — dat b1d7b3c →