← back to Designerwallcoverings
add redacted Stroheim Gemini diagnostics
68ea201ac27dacab1e328b15c4ccf24248be42fe · 2026-08-30 08:11:54 -0700 · Steve Abrams
Files touched
M scripts/stroheim-onboard/settlement-gate.mjsA scripts/stroheim-onboard/test-settlement-diagnostics.mjsM verification/TK-10933-e2e-proof.json
Diff
commit 68ea201ac27dacab1e328b15c4ccf24248be42fe
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sun Aug 30 08:11:54 2026 -0700
add redacted Stroheim Gemini diagnostics
---
scripts/stroheim-onboard/settlement-gate.mjs | 81 ++++++++++++++++++----
.../test-settlement-diagnostics.mjs | 59 ++++++++++++++++
verification/TK-10933-e2e-proof.json | 17 ++++-
3 files changed, 140 insertions(+), 17 deletions(-)
diff --git a/scripts/stroheim-onboard/settlement-gate.mjs b/scripts/stroheim-onboard/settlement-gate.mjs
index a00e650..a1519e1 100644
--- a/scripts/stroheim-onboard/settlement-gate.mjs
+++ b/scripts/stroheim-onboard/settlement-gate.mjs
@@ -20,6 +20,7 @@
*/
import fs from 'node:fs';
import path from 'node:path';
+import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -71,7 +72,7 @@ Respond with ONLY this JSON, booleans true/false (never null), plus a 3-6 word e
{"a1":bool,"a2":bool,"a3":bool,"b":bool,"acceptable":bool,"evidence":{"a1":"","a2":"","a3":"","b":"","acceptable":""}}`;
async function fetchB64(url, outerSignal) {
- if (TEST_MODE === 'hang') return { mime: 'image/jpeg', data: 'dGVzdA==' };
+ if (TEST_MODE) return { mime: 'image/jpeg', data: 'dGVzdA==' };
for (let t = 0; t < 4; t++) {
try {
const r = await fetch(url, { signal: combinedSignal(outerSignal, 30000) });
@@ -86,7 +87,23 @@ async function fetchB64(url, outerSignal) {
}
}
-async function gemini(b64, outerSignal) {
+function responseDiagnostic(text, parseError) {
+ return {
+ response_chars: text.length,
+ response_sha256_prefix: crypto.createHash('sha256').update(text).digest('hex').slice(0, 12),
+ parse_error: parseError ? String(parseError.name || 'SyntaxError').slice(0, 40) : null
+ };
+}
+
+function mockGeminiResponse(mode) {
+ if (mode === 'malformed-json') return { status: 200, json: { candidates: [{ finishReason: 'STOP', content: { parts: [{ text: '{not-json SECRET_SHOULD_NOT_APPEAR' }] } }] } };
+ if (mode === 'empty-candidates') return { status: 200, json: { candidates: [], promptFeedback: { blockReason: 'OTHER' } } };
+ if (mode === 'provider-error') return { status: 400, json: { error: { code: 400, status: 'INVALID_ARGUMENT', message: 'SECRET_SHOULD_NOT_APPEAR' } } };
+ if (mode === 'success') return { status: 200, json: { candidates: [{ finishReason: 'STOP', content: { parts: [{ text: JSON.stringify({ a1: true, a2: true, a3: false, b: false, acceptable: false, evidence: {} }) }] } }] } };
+ return null;
+}
+
+async function gemini(b64, outerSignal, sku) {
if (TEST_MODE === 'hang') {
await new Promise((resolve, reject) => {
outerSignal.addEventListener('abort', () => reject(outerSignal.reason), { once: true });
@@ -94,22 +111,52 @@ async function gemini(b64, outerSignal) {
}
const body = { contents: [{ parts: [{ text: PROMPT }, { inline_data: { mime_type: b64.mime, data: b64.data } }] }],
generationConfig: { temperature: 0, responseMimeType: 'application/json', thinkingConfig: { thinkingBudget: 0 } } };
+ let lastCategory = 'not_started';
for (let t = 0; t < 6; t++) {
try {
- const r = await fetch(URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: combinedSignal(outerSignal, 45000) });
- if (r.status === 429 || r.status >= 500) { await sleep(1500 * (t + 1), outerSignal); continue; }
- const j = await r.json();
+ const mock = mockGeminiResponse(TEST_MODE);
+ const r = mock || await fetch(URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), signal: combinedSignal(outerSignal, 45000) });
+ const httpStatus = r.status;
+ const j = mock ? mock.json : await r.json().catch(() => ({}));
+ const candidates = Array.isArray(j?.candidates) ? j.candidates : [];
+ const finishReasons = candidates.map(c => c?.finishReason).filter(Boolean).slice(0, 4);
+ const promptBlockReason = j?.promptFeedback?.blockReason ? String(j.promptFeedback.blockReason).slice(0, 60) : null;
+ if (httpStatus === 429 || httpStatus >= 500) {
+ lastCategory = httpStatus === 429 ? 'rate_limited' : 'provider_server_error';
+ appendAttempt({ sku, status: 'provider_attempt', attempt: t + 1, http_status: httpStatus,
+ provider_category: lastCategory, candidate_count: candidates.length, finish_reasons: finishReasons,
+ prompt_block_reason: promptBlockReason });
+ if (!TEST_MODE) await sleep(1500 * (t + 1), outerSignal);
+ continue;
+ }
+ if (httpStatus >= 400) {
+ lastCategory = `provider_error:${String(j?.error?.status || `http_${httpStatus}`).slice(0, 60)}`;
+ appendAttempt({ sku, status: 'provider_attempt', attempt: t + 1, http_status: httpStatus,
+ provider_category: lastCategory, provider_error_code: j?.error?.code ?? null,
+ candidate_count: candidates.length, finish_reasons: finishReasons, prompt_block_reason: promptBlockReason });
+ return { vision: null, retryCount: t + 1, terminalCategory: lastCategory };
+ }
const txt = j?.candidates?.[0]?.content?.parts?.[0]?.text || '';
- let parsed = null; try { parsed = JSON.parse(txt); } catch { parsed = null; }
- if (parsed) return parsed;
- await sleep(1000 * (t + 1), outerSignal); continue; // empty/unparseable (transient) → retry, do NOT fail-closed on first blip
+ let parsed = null;
+ let parseError = null;
+ try { parsed = txt ? JSON.parse(txt) : null; } catch (error) { parseError = error; }
+ lastCategory = parsed ? 'strict_json' : (txt ? 'malformed_json' : 'empty_candidate');
+ appendAttempt({ sku, status: 'provider_attempt', attempt: t + 1, http_status: httpStatus,
+ provider_category: lastCategory, candidate_count: candidates.length, finish_reasons: finishReasons,
+ prompt_block_reason: promptBlockReason, ...responseDiagnostic(txt, parseError) });
+ if (parsed) return { vision: parsed, retryCount: t + 1, terminalCategory: lastCategory };
+ if (!TEST_MODE) await sleep(1000 * (t + 1), outerSignal);
+ continue; // empty/unparseable (transient) → retry, do NOT fail-closed on first blip
} catch (e) {
if (outerSignal.aborted) throw e;
- await sleep(1000 * (t + 1), outerSignal);
+ lastCategory = 'network_or_decode_error';
+ appendAttempt({ sku, status: 'provider_attempt', attempt: t + 1, http_status: null,
+ provider_category: lastCategory, error_name: String(e?.name || 'Error').slice(0, 40) });
+ if (!TEST_MODE) await sleep(1000 * (t + 1), outerSignal);
continue;
} // ECONNRESET / timeout → retry
}
- return null;
+ return { vision: null, retryCount: 6, terminalCategory: lastCategory };
}
// defendant-favorable, fail-closed
@@ -127,8 +174,8 @@ async function processItem(p, signal, progress) {
const b64 = await fetchB64(p.image_url, signal);
progress.geminiStarted = true;
appendAttempt({ sku: p.sku, status: 'gemini_started', estimated_cost: COST_PER_IMG });
- const vision = await gemini(b64, signal);
- return { vision, estimatedCost: COST_PER_IMG };
+ const result = await gemini(b64, signal, p.sku);
+ return { ...result, estimatedCost: COST_PER_IMG };
}
async function processItemBounded(p) {
@@ -184,20 +231,24 @@ async function main() {
const hasAll = v && ['a1', 'a2', 'a3', 'b'].every(k => typeof v[k] === 'boolean');
if (!hasAll) {
skip++;
- appendAttempt({ sku: p.sku, status: 'vision_unparseable', estimated_cost: outcome.value.estimatedCost });
+ appendAttempt({ sku: p.sku, status: 'vision_unparseable', estimated_cost: outcome.value.estimatedCost,
+ billing_status: 'provider_billing_unverified', retry_count: outcome.value.retryCount,
+ provider_category: outcome.value.terminalCategory });
await sleep(400);
continue;
}
const rec = { sku: p.sku, ...verdict(v), gated_at: new Date().toISOString() };
fs.writeSync(fd, JSON.stringify(rec) + '\n');
- appendAttempt({ sku: p.sku, status: rec.verdict.toLowerCase(), estimated_cost: outcome.value.estimatedCost });
+ appendAttempt({ sku: p.sku, status: rec.verdict.toLowerCase(), estimated_cost: outcome.value.estimatedCost,
+ billing_status: 'provider_billing_unverified', retry_count: outcome.value.retryCount,
+ provider_category: outcome.value.terminalCategory });
if (rec.verdict === 'OK') ok++; else block++;
if ((ok + block + skip) % 20 === 0) console.log(` ...${ok + block + skip}/${todo.length} (OK ${ok}, BLOCK ${block}, skip ${skip}) · $${spent.toFixed(4)}`);
await sleep(400);
}
fs.closeSync(fd);
console.log(` pass result: OK ${ok} · BLOCK ${block} · skipped(retry-next-pass) ${skip}`);
- console.log(`\nDONE. OK=${ok} BLOCK=${block} skip=${skip} of ${todo.length} · actual ~$${spent.toFixed(4)} (Gemini ${MODEL} vision)`);
+ console.log(`\nDONE. OK=${ok} BLOCK=${block} skip=${skip} of ${todo.length} · estimated attempted ~$${spent.toFixed(4)} (provider billing unverified; Gemini ${MODEL} vision)`);
if (block) console.log(` ⚠ ${block} BLOCK/held — will NOT be created/activated; review out/settlement-verdicts.jsonl (grep BLOCK).`);
}
main().catch(e => { console.error(e); process.exit(1); });
diff --git a/scripts/stroheim-onboard/test-settlement-diagnostics.mjs b/scripts/stroheim-onboard/test-settlement-diagnostics.mjs
new file mode 100644
index 0000000..37e888f
--- /dev/null
+++ b/scripts/stroheim-onboard/test-settlement-diagnostics.mjs
@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const here = path.dirname(new URL(import.meta.url).pathname);
+const modes = ['malformed-json', 'empty-candidates', 'provider-error', 'success'];
+const results = [];
+
+for (const mode of modes) {
+ const out = fs.mkdtempSync(path.join(os.tmpdir(), `stroheim-${mode}-`));
+ const sku = `TEST-${mode.toUpperCase()}`;
+ fs.writeFileSync(path.join(out, 'payloads.jsonl'), JSON.stringify({ sku, image_url: 'mock://image' }) + '\n');
+ fs.writeFileSync(path.join(out, 'settlement-verdicts.jsonl'), '');
+ const run = spawnSync(process.execPath, [path.join(here, 'settlement-gate.mjs'), '--apply', '--limit=1'], {
+ encoding: 'utf8', timeout: 3000,
+ env: { ...process.env, STROHEIM_SETTLEMENT_OUT: out, STROHEIM_SETTLEMENT_TEST_MODE: mode }
+ });
+ assert.equal(run.status, 0, `${mode} failed: ${run.stderr}`);
+ const raw = fs.readFileSync(path.join(out, 'settlement-attempts.jsonl'), 'utf8');
+ assert(!raw.includes('SECRET_SHOULD_NOT_APPEAR'), `${mode} leaked raw provider content`);
+ assert(!raw.includes('inline_data') && !raw.includes('dGVzdA==') && !raw.includes('You are a legal image auditor'), `${mode} leaked request content`);
+ const events = raw.trim().split('\n').map(JSON.parse);
+ const provider = events.filter(e => e.status === 'provider_attempt');
+ const terminal = events.at(-1);
+ assert(provider.length >= 1, `${mode} lacks provider diagnostics`);
+ assert(provider.every(e => 'http_status' in e && e.provider_category), `${mode} lacks status/category`);
+ assert.equal(terminal.billing_status, 'provider_billing_unverified');
+ assert.equal(terminal.retry_count, provider.length);
+ if (mode === 'malformed-json') {
+ assert.equal(provider.length, 6);
+ assert(provider.every(e => e.provider_category === 'malformed_json' && e.parse_error === 'SyntaxError'));
+ assert(provider.every(e => e.response_chars > 0 && /^[a-f0-9]{12}$/.test(e.response_sha256_prefix)));
+ assert.equal(terminal.status, 'vision_unparseable');
+ } else if (mode === 'empty-candidates') {
+ assert.equal(provider.length, 6);
+ assert(provider.every(e => e.provider_category === 'empty_candidate' && e.candidate_count === 0));
+ assert(provider.every(e => e.prompt_block_reason === 'OTHER'));
+ assert.equal(terminal.status, 'vision_unparseable');
+ } else if (mode === 'provider-error') {
+ assert.equal(provider.length, 1);
+ assert.equal(provider[0].http_status, 400);
+ assert.equal(provider[0].provider_category, 'provider_error:INVALID_ARGUMENT');
+ assert.equal(provider[0].provider_error_code, 400);
+ assert.equal(terminal.status, 'vision_unparseable');
+ } else {
+ assert.equal(provider.length, 1);
+ assert.equal(provider[0].provider_category, 'strict_json');
+ assert.equal(provider[0].candidate_count, 1);
+ assert.deepEqual(provider[0].finish_reasons, ['STOP']);
+ assert.equal(terminal.status, 'ok');
+ assert.equal(fs.readFileSync(path.join(out, 'settlement-verdicts.jsonl'), 'utf8').trim().split('\n').length, 1);
+ }
+ results.push({ mode, provider_attempts: provider.length, terminal: terminal.status });
+}
+
+console.log(JSON.stringify({ verdict: 'PASS', cases: results }, null, 2));
diff --git a/verification/TK-10933-e2e-proof.json b/verification/TK-10933-e2e-proof.json
index 5b84742..e6485d0 100644
--- a/verification/TK-10933-e2e-proof.json
+++ b/verification/TK-10933-e2e-proof.json
@@ -2,9 +2,9 @@
"ticket": "TK-10933",
"intent": "Bound each Stroheim settlement item, durably record attempt and cost state, and verify the guard with a strictly one-item real-network canary without Shopify writes.",
"risk_tier": "R4 approved one-item external integration canary",
- "environment": "local Node.js process; one real vendor-image fetch and one bounded Gemini item; no Shopify path",
+ "environment": "local Node.js process; one prior real bounded Gemini item plus deterministic local diagnostics mocks; no Shopify path",
"baseline_commit": "6c4a2daac58648d7a7d53471f611502f7b374bbf",
- "timestamp": "2026-08-30T15:07:23Z",
+ "timestamp": "2026-08-30T15:11:23Z",
"precondition": {
"canonical_payloads": 771,
"canonical_verdicts": 300,
@@ -74,6 +74,19 @@
"created.jsonl remained absent"
],
"verdict": "PASS"
+ },
+ {
+ "name": "redacted provider observability",
+ "command": "node scripts/stroheim-onboard/test-settlement-diagnostics.mjs",
+ "assertions": [
+ "malformed JSON records six HTTP-200 attempts, candidate/finish metadata, response length, SHA-256 prefix, and SyntaxError without raw response text",
+ "empty candidates record six HTTP-200 empty_candidate attempts with candidate_count 0 and prompt block reason OTHER",
+ "provider error records one HTTP-400 provider_error:INVALID_ARGUMENT attempt and numeric provider error code without provider message",
+ "successful strict JSON records one HTTP-200 strict_json attempt and writes one OK verdict",
+ "all terminal events record retry_count and provider_billing_unverified",
+ "attempt ledgers contain no prompt, image/base64, secret marker, API key, or raw provider response"
+ ],
+ "verdict": "PASS"
}
],
"negative_error_retry": "The mock never resolves until AbortController cancellation. Five repeated runs prove bounded cancellation and terminal event durability.",
← bb130ed record one-item Stroheim network canary
·
back to Designerwallcoverings
·
auto-data-snapshot: 2026-08-30T08:29:04 (4 data files) — dat c71677c →