← back to Wallco Ai

max-debug: add missing Gemini fetch timeout (45s — body-stall no longer hangs child→fail-closed→unpublish); raise ollama 120→180s + spawn 150→240s + batch 90→240s (queue-wait under concurrency); guard SIGTERM-check + stdout-only VERDICT match

bf0a5f32126f76c346f8bad2c9b70133b8939825 · 2026-06-10 07:13:33 -0700 · Steve

Files touched

• M scripts/settlement-gate-batch.js
• M scripts/settlement_postgen_vision_check.js
• M scripts/settlement_tick_guard.js

Diff

commit bf0a5f32126f76c346f8bad2c9b70133b8939825
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jun 10 07:13:33 2026 -0700

    max-debug: add missing Gemini fetch timeout (45s — body-stall no longer hangs child→fail-closed→unpublish); raise ollama 120→180s + spawn 150→240s + batch 90→240s (queue-wait under concurrency); guard SIGTERM-check + stdout-only VERDICT match
---
 scripts/settlement-gate-batch.js           |  8 +++++---
 scripts/settlement_postgen_vision_check.js | 11 +++++++++--
 scripts/settlement_tick_guard.js           | 19 +++++++++++++------
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/scripts/settlement-gate-batch.js b/scripts/settlement-gate-batch.js
index 6de67b1..4d979c9 100644
--- a/scripts/settlement-gate-batch.js
+++ b/scripts/settlement-gate-batch.js
@@ -29,9 +29,11 @@ const psql = sql => execSync(`psql -d dw_unified -tAc "${sql.replace(/"/g, '\\"'
 
 function gateOne(id, localPath) {
   const r = spawnSync('node', [path.join(__dirname, 'settlement_postgen_vision_check.js'), localPath, CATEGORY],
-    { cwd: ROOT, encoding: 'utf8', timeout: 90_000, env: process.env });
-  const out = `${r.stdout || ''}${r.stderr || ''}`;
-  const m = out.match(/VERDICT:\s*(BLOCK|NEEDS REVIEW|OK)/);
+    { cwd: ROOT, encoding: 'utf8', timeout: 240_000, env: process.env });  // was 90s — UNDER the child's own 180s ollama timeout; concurrency made it SIGTERM mid-check → false fail-closed (max-debug 2026-06-10)
+  // SIGTERM (r.signal) / non-zero leaves no stdout VERDICT → falls to the
+  // REVIEW-VISION-ERROR else-branch below = correct fail-closed. Match stdout
+  // ONLY so a stray 'VERDICT:'-shaped string in stderr/reasoning can't mis-match.
+  const m = (r.stdout || '').match(/VERDICT:\s*(BLOCK|NEEDS REVIEW|OK)/);
   let verdict;
   if (r.status === 2 || (m && m[1] === 'BLOCK')) verdict = 'BLOCK';
   else if (m) verdict = m[1] === 'NEEDS REVIEW' ? 'NEEDS_REVIEW' : 'OK';
diff --git a/scripts/settlement_postgen_vision_check.js b/scripts/settlement_postgen_vision_check.js
index bff24d0..22a990f 100644
--- a/scripts/settlement_postgen_vision_check.js
+++ b/scripts/settlement_postgen_vision_check.js
@@ -96,7 +96,8 @@ function parseJsonLoose(text) {
 //   on a swap-heavy box can't overrun and spuriously fail-closed (→ the sweep
 //   would unpublish). Caller (settlement_tick_guard spawnSync) is 150s to match.
 const VISION_KEEP_ALIVE = process.env.OLLAMA_KEEP_ALIVE || '2m';
-const VISION_TIMEOUT_MS = 120_000;
+const VISION_TIMEOUT_MS = 180_000;   // ollama: cold-load + a queued wait behind another serialized call (2026-06-10 max-debug)
+const GEMINI_TIMEOUT_MS = 45_000;    // Gemini: a stalled body (not just 429) must NOT hang the child to the spawn-timeout → fail-closed → unpublish
 
 async function ollamaGen(host, model, b64, prompt) {
   const ac = new AbortController();
@@ -158,11 +159,17 @@ async function ollamaVision(b64, prompt) {
   const t0 = Date.now();
   let result = null, source = 'gemini';
   // 1) PRIMARY — Gemini (most accurate). Falls through to local on any failure.
+  // AbortController: without it a stalled Gemini body hangs here until the parent
+  // spawnSync SIGTERMs the whole child → no VERDICT → fail-closed → the sweep
+  // unpublishes a CLEAN design (max-debug 2026-06-10, the missed critical bug).
+  const gac = new AbortController();
+  const gto = setTimeout(() => gac.abort(), GEMINI_TIMEOUT_MS);
   try {
     const r = await fetch(url, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body),
+      signal: gac.signal,
     });
     if (r.ok) {
       const j = await r.json();
@@ -185,7 +192,7 @@ async function ollamaVision(b64, prompt) {
     }
   } catch (e) {
     console.error(`Gemini error: ${e.message} — falling back to local ollama vision`);
-  }
+  } finally { clearTimeout(gto); }
 
   // 2) FALLBACK — free local ollama vision (no Gemini-cap dependency).
   if (!result) {
diff --git a/scripts/settlement_tick_guard.js b/scripts/settlement_tick_guard.js
index 755e519..1834f99 100644
--- a/scripts/settlement_tick_guard.js
+++ b/scripts/settlement_tick_guard.js
@@ -47,13 +47,20 @@ function postgenVision(localPath, title) {
   const r = spawnSync('node', [
     path.join(__dirname, 'settlement_postgen_vision_check.js'),
     localPath, title || '',
-    // 150s: the vision check's local-ollama fallback may COLD-load gemma3:12b
-    // (~8GB) on a memory-pressured box; its internal AbortController is 120s, so
-    // this must exceed it or the parent would SIGTERM mid-check → spurious
-    // fail-closed → the sweep unpublishes a clean design (2026-06-10 review).
-  ], { cwd: ROOT, encoding: 'utf8', timeout: 150_000, env: process.env });
+    // 240s: must exceed the child's worst case = Gemini 45s + ollama 180s (cold
+    // load + a queued wait behind another serialized ollama call). Too low and the
+    // parent SIGTERMs mid-check → no VERDICT → fail-closed → sweep unpublishes a
+    // clean design (max-debug 2026-06-10).
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 240_000, env: process.env });
+  // If the child was killed by a signal (spawnSync timeout SIGTERM), it never got
+  // to print VERDICT — treat as a clean ERROR, don't scan a truncated buffer.
+  if (r.signal) {
+    return { verdict: 'ERROR', prohibited: false, error: true, raw: `vision child killed by ${r.signal} (timeout)` };
+  }
   const out = `${r.stdout || ''}${r.stderr || ''}`;
-  const m = out.match(/VERDICT:\s*(BLOCK|NEEDS REVIEW|OK)/);
+  // Match the verdict on STDOUT ONLY — stderr carries fallback log lines; never
+  // let a stray 'VERDICT:'-shaped string in a log or the reasoning field match.
+  const m = (r.stdout || '').match(/VERDICT:\s*(BLOCK|NEEDS REVIEW|OK)/);
   if (r.status === 2 || (m && m[1] === 'BLOCK')) {
     return { verdict: 'BLOCK', prohibited: true, error: false, raw: out.slice(-500) };
   }

← f6b54ad settlement vision (2-agent review): keep_alive=2m frees ~8GB  ·  back to Wallco Ai  ·  max-debug security: sanitize vision-check TITLE (callers pas 88181c9 →