build-mural-master: retry per-tile ESRGAN calls (4x backoff) — one transient Replicate 400 was crashing a whole mural build [21079 fix]

bd8def21ea0aae64d8d947244e219dd9a14f2e75 · 2026-06-03 14:11:02 -0700 · Steve Abrams

Files touched

M scripts/build-mural-master.py

Diff

commit bd8def21ea0aae64d8d947244e219dd9a14f2e75
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jun 3 14:11:02 2026 -0700

    build-mural-master: retry per-tile ESRGAN calls (4x backoff) — one transient Replicate 400 was crashing a whole mural build [21079 fix]
---
 scripts/build-mural-master.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/scripts/build-mural-master.py b/scripts/build-mural-master.py
index 604f539..53129ce 100644
--- a/scripts/build-mural-master.py
+++ b/scripts/build-mural-master.py
@@ -138,6 +138,22 @@ def assemble_master(focal, bleed_px):
 _ESRGAN_MAX_IN = 1448   # Replicate real-esrgan GPU cap (~2.09M px input)
 
 
+def _esrgan_x4_retry(img, token, tries=4):
+    """real-esrgan x4 with retries — a single transient Replicate error (400/
+    429/5xx/timeout) on any one of a mural's ~17 tile calls otherwise crashes
+    the whole build. Retry with backoff; re-raise only if all attempts fail."""
+    last = None
+    for i in range(tries):
+        try:
+            return _bt._esrgan_x4(img, token)
+        except Exception as e:
+            last = e
+            print(f"      esrgan tile attempt {i+1}/{tries} failed ({str(e)[:70]}); retrying",
+                  file=sys.stderr)
+            time.sleep(3 + i * 4)
+    raise last
+
+
 def _replicate_run_latest(model, inp, token, poll_max=600):
     """Run a Replicate model's latest version via the version-pinned
     /v1/predictions path (the model-predictions endpoint 404s for some models;
@@ -211,7 +227,7 @@ def esrgan_tiled(img, token, target_long):
     levels = 0
     while max(img.size) * 2 < target_long and levels < 4:
         if max(img.size) <= _ESRGAN_MAX_IN:
-            img = _bt._esrgan_x4(img, token); levels += 1; continue
+            img = _esrgan_x4_retry(img, token); levels += 1; continue
         img = _esrgan_x4_tiled(img, token); levels += 1
     cw, ch = img.size
     s = target_long / max(cw, ch)
@@ -234,7 +250,7 @@ def _esrgan_x4_tiled(img, token, tile=1200, overlap=128):
             x0, y0 = tx, ty
             x1, y1 = min(tx + tile, W), min(ty + tile, H)
             piece = img.crop((x0, y0, x1, y1))
-            up = _bt._esrgan_x4(piece, token)  # piece <= 1200 < 1448 cap
+            up = _esrgan_x4_retry(piece, token)  # piece <= 1200 < 1448 cap
             pw, ph = up.size
             # feather mask: ramp in over the overlap on each interior edge
             m = Image.new("L", (pw, ph), 255); d = ImageDraw.Draw(m)

← 333f44d designs grid: add 'Digital for Sale' bulk action (unpublish · back to Wallco Ai · PDP themes: route v1-compact/v2-bento/v3-tabs through themeV f9e64ac →