[object Object]

← back to Wallco Ai

settlement vision (2-agent review): keep_alive=2m frees ~8GB idle; timeouts 75→120s/90→150s prevent cold-load fail-closed; in-code model fallback (gemma3→llava) kills the missing-model foot-gun on any box

f6b54ad8e9a35b0bd5c5896f59c479e1390b3ec7 · 2026-06-10 07:00:29 -0700 · Steve

Files touched

Diff

commit f6b54ad8e9a35b0bd5c5896f59c479e1390b3ec7
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jun 10 07:00:29 2026 -0700

    settlement vision (2-agent review): keep_alive=2m frees ~8GB idle; timeouts 75→120s/90→150s prevent cold-load fail-closed; in-code model fallback (gemma3→llava) kills the missing-model foot-gun on any box
---
 scripts/settlement_postgen_vision_check.js | 42 +++++++++++++++++++++++++-----
 scripts/settlement_tick_guard.js           |  6 ++++-
 2 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/scripts/settlement_postgen_vision_check.js b/scripts/settlement_postgen_vision_check.js
index b026ea7..bff24d0 100644
--- a/scripts/settlement_postgen_vision_check.js
+++ b/scripts/settlement_postgen_vision_check.js
@@ -87,11 +87,20 @@ function parseJsonLoose(text) {
 // settlement prompt + image, local + unmetered — so a Gemini outage no longer
 // fail-closes the gate into mass-unpublishing clean designs (the 2026-06-09
 // cascade: cap → "vision unavailable" → guard unpublished 153 clean designs).
-async function ollamaVision(b64, prompt) {
-  const model = process.env.OLLAMA_VISION_MODEL || 'gemma3:12b';
-  const host  = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
+//
+// VISION_KEEP_ALIVE: unload the model after this idle window so the ~8GB
+//   gemma3:12b doesn't linger resident on a memory-pressured box (2026-06-10
+//   review: yesterday's panics were memory exhaustion). Stays warm across a
+//   burst (sweep) since each check resets the timer; unloads in the long idle
+//   between sweeps. VISION_TIMEOUT_MS raised 75s→120s so a deliberate COLD load
+//   on a swap-heavy box can't overrun and spuriously fail-closed (→ the sweep
+//   would unpublish). Caller (settlement_tick_guard spawnSync) is 150s to match.
+const VISION_KEEP_ALIVE = process.env.OLLAMA_KEEP_ALIVE || '2m';
+const VISION_TIMEOUT_MS = 120_000;
+
+async function ollamaGen(host, model, b64, prompt) {
   const ac = new AbortController();
-  const to = setTimeout(() => ac.abort(), 75_000);
+  const to = setTimeout(() => ac.abort(), VISION_TIMEOUT_MS);
   try {
     const r = await fetch(`${host}/api/generate`, {
       method: 'POST',
@@ -101,17 +110,36 @@ async function ollamaVision(b64, prompt) {
         prompt: prompt + '\n\nReturn ONLY the JSON object, no other text.',
         images: [b64],
         stream: false,
+        keep_alive: VISION_KEEP_ALIVE,
         options: { temperature: 0 },
       }),
       signal: ac.signal,
     });
-    if (!r.ok) return null;
+    if (r.status === 404) return { notFound: true };   // model not pulled on THIS box
+    if (!r.ok) return null;                             // other HTTP error
     const j = await r.json();
-    return parseJsonLoose(j.response || '');
-  } catch { return null; }
+    return { result: parseJsonLoose(j.response || '') };
+  } catch { return null; }                              // timeout/abort/network
   finally { clearTimeout(to); }
 }
 
+async function ollamaVision(b64, prompt) {
+  const host = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
+  const primary = process.env.OLLAMA_VISION_MODEL || 'gemma3:12b';
+  // Model fallback: if the configured model isn't present on this box (e.g. prod
+  // has llava but not gemma3), try the next instead of fail-closing. Only an
+  // INSTANT 404 advances to the next model — a real timeout/error returns null
+  // immediately so we never chain two slow attempts past the 150s spawn budget.
+  const models = [...new Set([primary, 'llava:latest'])];
+  for (const model of models) {
+    const out = await ollamaGen(host, model, b64, prompt);
+    if (out === null) return null;        // timeout/error — don't burn budget on another slow model
+    if (out.notFound) continue;           // model absent (instant) — try the next
+    return out.result;                    // got a response (parse may be null; handled upstream)
+  }
+  return null;
+}
+
 (async () => {
   const body = {
     contents: [{
diff --git a/scripts/settlement_tick_guard.js b/scripts/settlement_tick_guard.js
index c19813c..755e519 100644
--- a/scripts/settlement_tick_guard.js
+++ b/scripts/settlement_tick_guard.js
@@ -47,7 +47,11 @@ function postgenVision(localPath, title) {
   const r = spawnSync('node', [
     path.join(__dirname, 'settlement_postgen_vision_check.js'),
     localPath, title || '',
-  ], { cwd: ROOT, encoding: 'utf8', timeout: 90_000, env: process.env });
+    // 150s: the vision check's local-ollama fallback may COLD-load gemma3:12b
+    // (~8GB) on a memory-pressured box; its internal AbortController is 120s, so
+    // this must exceed it or the parent would SIGTERM mid-check → spurious
+    // fail-closed → the sweep unpublishes a clean design (2026-06-10 review).
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 150_000, env: process.env });
   const out = `${r.stdout || ''}${r.stderr || ''}`;
   const m = out.match(/VERDICT:\s*(BLOCK|NEEDS REVIEW|OK)/);
   if (r.status === 2 || (m && m[1] === 'BLOCK')) {

← 52ea65f security: strip DB DSN passwords from audit/digest docs (use  ·  back to Wallco Ai  ·  max-debug: add missing Gemini fetch timeout (45s — body-stal bf0a5f3 →