← back to Designer Wallcoverings
enrich-full.py: retry-with-backoff on transient ollama outages instead of writing fail-stubs (a stub marked the SKU done and burned it forever; an ollama flap corrupted 2066 SKUs this way)
db33be403c4e6fb246222df375f5945374bb7fef · 2026-07-01 13:47:59 -0700 · Steve
Files touched
M onboarding/sangetsu-lilycolor/scripts/enrich-full.py
Diff
commit db33be403c4e6fb246222df375f5945374bb7fef
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jul 1 13:47:59 2026 -0700
enrich-full.py: retry-with-backoff on transient ollama outages instead of writing fail-stubs (a stub marked the SKU done and burned it forever; an ollama flap corrupted 2066 SKUs this way)
---
.../sangetsu-lilycolor/scripts/enrich-full.py | 62 ++++++++++++++++------
1 file changed, 45 insertions(+), 17 deletions(-)
diff --git a/onboarding/sangetsu-lilycolor/scripts/enrich-full.py b/onboarding/sangetsu-lilycolor/scripts/enrich-full.py
index 08460e90..785b5886 100644
--- a/onboarding/sangetsu-lilycolor/scripts/enrich-full.py
+++ b/onboarding/sangetsu-lilycolor/scripts/enrich-full.py
@@ -20,7 +20,7 @@ imageType, description, _provider, _cost, wall_s.
Usage: enrich-full.py [out_jsonl]
out_jsonl defaults to staging/enrichment-full-063026A.jsonl
"""
-import sys, os, json, time, subprocess, urllib.request, fcntl
+import sys, os, json, time, subprocess, urllib.request, urllib.error, fcntl
HERE = os.path.dirname(os.path.abspath(__file__))
PROJ = os.path.dirname(HERE)
@@ -39,6 +39,10 @@ PALETTE_PY = os.path.expanduser("~/Projects/enrich-local-hybrid/enrich-palette.p
LEXICON_PY = os.path.join(HERE, "lexicon-colorway.py")
DEFAULT_OUT = os.path.join(PROJ, "staging", "enrichment-full-063026A.jsonl")
IMG_EXTS = (".jpg", ".jpeg", ".png")
+# How many times to wait-and-retry a single SKU through a transient ollama outage
+# before deferring it (writing nothing, so a later run reclaims it). Backoff is
+# capped at 60s/attempt, so 30 ≈ up to ~25 min of tolerated downtime per SKU.
+MAX_TRANSIENT_RETRIES = int(os.environ.get("ENRICH_MAX_RETRIES", "30"))
def palette(img_path, k=6):
@@ -238,24 +242,47 @@ def main():
print("Nothing to do — all discovered SKUs already enriched.", flush=True)
return
- ok = fail = 0
+ ok = fail = deferred = 0
t_start = time.time()
# Append mode — never clobber prior work (resumable).
with open(out_path, "a") as fout:
for i, (sku, img) in enumerate(todo, 1):
- try:
- row = enrich_one(sku, img)
- if row.get("error") or row.get("usable") is False:
+ attempt = 0
+ while True:
+ try:
+ row = enrich_one(sku, img)
+ if row.get("error") or row.get("usable") is False:
+ fail += 1
+ else:
+ ok += 1
+ fout.write(json.dumps(row, ensure_ascii=False) + "\n")
+ fout.flush()
+ break
+ except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
+ # TRANSIENT ollama outage (Errno 61 Connection refused / timeout).
+ # NEVER write a stub: a stub row carries a "sku" key, which load_done
+ # would treat as done and skip forever. Wait for ollama to return and
+ # retry the SAME SKU with capped backoff; if it stays down past the
+ # retry budget, defer (write nothing) so a later run reclaims it.
+ attempt += 1
+ if attempt > MAX_TRANSIENT_RETRIES:
+ deferred += 1
+ print(f"[{i}/{len(todo)}] {sku} DEFER after {attempt-1} retries "
+ f"(ollama still down) — left for a later sweep", flush=True)
+ break
+ wait = min(60, 5 * attempt)
+ print(f"[{i}/{len(todo)}] {sku} ollama down ({str(e)[:60]}) — "
+ f"retry {attempt}/{MAX_TRANSIENT_RETRIES} in {wait}s", flush=True)
+ time.sleep(wait)
+ continue
+ except Exception as e:
+ # PERMANENT per-SKU error (bad/missing/corrupt image). Stub it so it
+ # counts as done and the run doesn't retry a genuinely broken image.
fail += 1
- else:
- ok += 1
- fout.write(json.dumps(row, ensure_ascii=False) + "\n")
- fout.flush()
- except Exception as e:
- fail += 1
- fout.write(json.dumps({"sku": sku, "error": str(e)[:200]}) + "\n")
- fout.flush()
- print(f"[{i}/{len(todo)}] {sku} ERROR {str(e)[:120]}", flush=True)
+ fout.write(json.dumps({"sku": sku, "error": str(e)[:200]}) + "\n")
+ fout.flush()
+ print(f"[{i}/{len(todo)}] {sku} ERROR {str(e)[:120]}", flush=True)
+ break
if i % 25 == 0 or i == len(todo):
elapsed = time.time() - t_start
@@ -264,11 +291,12 @@ def main():
eta = remaining / rate if rate > 0 else 0
total_done = len(done) + i
print(f"[{i}/{len(todo)}] progress {total_done}/{total} "
- f"ok={ok} fail={fail} rate={rate*60:.1f}/min "
+ f"ok={ok} fail={fail} deferred={deferred} rate={rate*60:.1f}/min "
f"eta={fmt_eta(eta)}", flush=True)
- print(f"\nDONE this pass: ok={ok} fail={fail} processed={len(todo)} "
- f"grand_total_now={len(done)+len(todo)}/{total}", flush=True)
+ print(f"\nDONE this pass: ok={ok} fail={fail} deferred={deferred} "
+ f"processed={len(todo)} grand_total_now={len(done)+len(todo)-deferred}/{total}"
+ f"{' (deferred SKUs reclaimed on next run)' if deferred else ''}", flush=True)
if __name__ == "__main__":
← ec40684a auto-save: 2026-07-01T13:44:23 (5 files) — pending-approval/
·
back to Designer Wallcoverings
·
enrich-full.py: treat mid-run Henry unmount as transient (Vo dcd19e57 →