← back to Enrich Local Hybrid
enrich-local: fast-fail exo (15s) + consecutive-failure circuit breaker so a loaded/cold ring can't stall a 1500-image cron (TK-12090)
5193103c3139d54fe9349cf2c3549b2a28f9f656 · 2026-09-23 17:47:04 -0700 · Steve Abrams
Files touched
Diff
commit 5193103c3139d54fe9349cf2c3549b2a28f9f656
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 23 17:47:04 2026 -0700
enrich-local: fast-fail exo (15s) + consecutive-failure circuit breaker so a loaded/cold ring can't stall a 1500-image cron (TK-12090)
---
enrich-local.js | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
diff --git a/enrich-local.js b/enrich-local.js
index 5c2da30..66d6424 100644
--- a/enrich-local.js
+++ b/enrich-local.js
@@ -20,6 +20,11 @@
// as a legacy alias.
// VISION_FALLBACK gemini (default) | none
// ENRICH_PY python3 path (default 'python3')
+// EXO_TIMEOUT_MS exo-only attempt budget (default 15000). Short on purpose: a loaded
+// ring must fail fast to the fallback, not stall the batch.
+// VISION_TIMEOUT_MS fallback (Gemini) budget (default 60000).
+// EXO_BREAKER_N consecutive exo failures before the ring is skipped for the rest of
+// the run (default 5).
const { spawnSync } = require('child_process');
const path = require('path');
const fs = require('fs');
@@ -32,6 +37,17 @@ const PY = process.env.ENRICH_PY || 'python3';
const PALETTE_PY = path.join(__dirname, 'enrich-palette.py');
const MAX_COLORS = 6;
+// TK-12090: batch throughput guards. The exo attempt gets its OWN short timeout so a
+// saturated/cold ring fails fast to the fallback instead of burning the shared 120s per
+// image (at --limit 1500 that was ~50h/cron, i.e. never finishing, and the tiers stack).
+// After EXO_BREAKER_N consecutive exo failures the ring is skipped for the rest of the
+// process, bounding total wasted wait to N * EXO_TIMEOUT_MS rather than 1500 * timeout.
+const VISION_TIMEOUT_MS = Number(process.env.VISION_TIMEOUT_MS) || 60000; // fallback (Gemini) budget
+const EXO_TIMEOUT_MS = Number(process.env.EXO_TIMEOUT_MS) || 15000; // exo-only budget
+const EXO_BREAKER_N = Number(process.env.EXO_BREAKER_N) || 5;
+let _exoFailStreak = 0;
+let _exoBreakerTripped = false;
+
let _libPromise = null;
function exoLib() {
if (!_libPromise) {
@@ -75,9 +91,21 @@ async function visionAnalyze(imageB64, palette) {
prompt,
image: { b64: imageB64, mime: 'image/jpeg' },
model: process.env.VISION_MODEL || process.env.ENRICH_VL_MODEL || undefined,
- timeoutMs: 120000,
+ timeoutMs: VISION_TIMEOUT_MS,
+ exoTimeoutMs: EXO_TIMEOUT_MS,
+ skipExo: _exoBreakerTripped,
maxTokens: 800,
});
+ if (r.ok && r.provider === 'exo') {
+ _exoFailStreak = 0; // a success clears the streak
+ } else if (!_exoBreakerTripped) {
+ // Either exo failed (fell back / not measured) or it was skipped. Count only real attempts.
+ if (++_exoFailStreak >= EXO_BREAKER_N) {
+ _exoBreakerTripped = true;
+ console.log(` → exo circuit breaker OPEN after ${_exoFailStreak} consecutive failures ` +
+ `— remaining images this run go straight to the fallback (saves ${EXO_TIMEOUT_MS}ms/image)`);
+ }
+ }
if (!r.ok) throw new Error(r.error || 'vision call not measured (exo ring + Gemini fallback both unavailable)');
return extractJson(r.text);
}
← 63ea825 Route local vision leg through shared exo-vision lib, drop d
·
back to Enrich Local Hybrid
·
enrich-local: EXO_TIMEOUT_MS 15s->45s (warm real-image visio e7e778b →