[object Object]

← back to Dw Photo Capture

fix threading race in _get_model(): add _MODEL_LOCK + double-checked locking pattern

c11515c72b15eb3dc13c63282e2d3b3165aa4053 · 2026-09-20 04:02:42 -0700 · steve@designerwallcoverings.com

Two concurrent first-/search requests both saw _MODEL["model"] is None and
both loaded the 605MB CLIP model, causing a ~1.2GB transient memory spike
(observed in prod logs 2026-09-19). Fix: add _MODEL_LOCK = threading.Lock()
and wrap the load body with with _MODEL_LOCK / inner None-recheck so only
one thread ever loads the model.

Redeploy to Kamatera is GATED (TK-11931).

Files touched

Diff

commit c11515c72b15eb3dc13c63282e2d3b3165aa4053
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date:   Sun Sep 20 04:02:42 2026 -0700

    fix threading race in _get_model(): add _MODEL_LOCK + double-checked locking pattern
    
    Two concurrent first-/search requests both saw _MODEL["model"] is None and
    both loaded the 605MB CLIP model, causing a ~1.2GB transient memory spike
    (observed in prod logs 2026-09-19). Fix: add _MODEL_LOCK = threading.Lock()
    and wrap the load body with with _MODEL_LOCK / inner None-recheck so only
    one thread ever loads the model.
    
    Redeploy to Kamatera is GATED (TK-11931).
---
 visual-search/search_service.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/visual-search/search_service.py b/visual-search/search_service.py
index 7e1493c..9812abc 100644
--- a/visual-search/search_service.py
+++ b/visual-search/search_service.py
@@ -20,19 +20,22 @@ PORT = int(os.environ.get("VS_PORT", "9914"))
 HOST = os.environ.get("VS_HOST", "127.0.0.1")
 
 _MODEL = {"model": None, "preprocess": None}
+_MODEL_LOCK = threading.Lock()
 def _get_model():
     """Load CLIP on first /search only. Uses the SAME weights (ViT-B-32/laion2b + optional
     dw_clip_ft.pt) the catalog was embedded with, so re-embedded photos share the catalog's space."""
     if _MODEL["model"] is None:
-        import torch, open_clip
-        print("loading CLIP…", flush=True)
-        model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="laion2b_s34b_b79k")
-        _ft = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dw_clip_ft.pt")
-        if os.path.exists(_ft):
-            model.load_state_dict(torch.load(_ft, map_location="cpu"), strict=False)
-            print("loaded fine-tuned DW CLIP weights:", _ft, flush=True)
-        model.eval(); torch.set_num_threads(max(1, os.cpu_count() - 2))
-        _MODEL["model"], _MODEL["preprocess"] = model, preprocess
+        with _MODEL_LOCK:
+            if _MODEL["model"] is None:  # re-check inside lock (double-checked locking pattern)
+                import torch, open_clip
+                print("loading CLIP…", flush=True)
+                model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="laion2b_s34b_b79k")
+                _ft = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dw_clip_ft.pt")
+                if os.path.exists(_ft):
+                    model.load_state_dict(torch.load(_ft, map_location="cpu"), strict=False)
+                    print("loaded fine-tuned DW CLIP weights:", _ft, flush=True)
+                model.eval(); torch.set_num_threads(max(1, os.cpu_count() - 2))
+                _MODEL["model"], _MODEL["preprocess"] = model, preprocess
     return _MODEL["model"], _MODEL["preprocess"]
 
 LOCK = threading.Lock()

← 618d031 auto-data-snapshot: 2026-09-20T03:53:01 (1 data files) — dat  ·  back to Dw Photo Capture  ·  auto-data-snapshot: 2026-09-20T04:26:57 (1 data files) — dat 3b53e0e →