← back to Dw Photo Capture
Overnight DW CLIP fine-tune pipeline (MPS): build_manifest + resumable 256px downloader + open_clip contrastive fine-tune + overnight orchestrator
5126f02536fa229e927c3ff8e74638478f4f33b8 · 2026-07-06 21:07:10 -0700 · Steve Abrams
Files touched
A visual-search/train/.gitignoreA visual-search/train/build_manifest.shA visual-search/train/download.pyA visual-search/train/finetune.pyA visual-search/train/run_overnight.sh
Diff
commit 5126f02536fa229e927c3ff8e74638478f4f33b8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 6 21:07:10 2026 -0700
Overnight DW CLIP fine-tune pipeline (MPS): build_manifest + resumable 256px downloader + open_clip contrastive fine-tune + overnight orchestrator
---
visual-search/train/.gitignore | 5 ++
visual-search/train/build_manifest.sh | 26 +++++++++
visual-search/train/download.py | 64 ++++++++++++++++++++++
visual-search/train/finetune.py | 100 ++++++++++++++++++++++++++++++++++
visual-search/train/run_overnight.sh | 23 ++++++++
5 files changed, 218 insertions(+)
diff --git a/visual-search/train/.gitignore b/visual-search/train/.gitignore
new file mode 100644
index 0000000..9160e94
--- /dev/null
+++ b/visual-search/train/.gitignore
@@ -0,0 +1,5 @@
+venv/
+cache/
+ckpt/
+logs/
+manifest.tsv
diff --git a/visual-search/train/build_manifest.sh b/visual-search/train/build_manifest.sh
new file mode 100755
index 0000000..7fcebb5
--- /dev/null
+++ b/visual-search/train/build_manifest.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Build the training manifest from the LOCAL dw_unified mirror → manifest.tsv
+# Columns: id \t image_url \t pattern_name \t color \t vendor \t product_type
+# One catalog image + its text = one CLIP contrastive pair. Only rows with a real
+# pattern name and an http image are usable. Color is cleaned of the JSON-blob form.
+set -e
+cd "$(dirname "$0")"
+DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
+LIMIT="${1:-100000}" # 100k diverse pairs = a strong overnight fine-tune (many epochs) without a 245k download
+psql "$DB" -F$'\t' -tA -c "
+ select id,
+ image_url,
+ regexp_replace(pattern_name, E'[\t\n\r]', ' ', 'g'),
+ -- color_name may be plain text OR a JSON blob {\"Name\":\"Beige\",...}; pull the Name
+ coalesce(
+ nullif(regexp_replace(coalesce(color_name,''), E'.*\"Name\":\\s*\"([^\"]+)\".*', E'\\1'), color_name),
+ color_primary, ''),
+ coalesce(original_vendor_name, vendor_code, ''),
+ coalesce(product_type, 'Wallcovering')
+ from vendor_catalog
+ where image_url like 'http%'
+ and pattern_name is not null and pattern_name <> ''
+ order by random()
+ limit $LIMIT
+" > manifest.tsv
+echo "manifest rows: $(wc -l < manifest.tsv)"
diff --git a/visual-search/train/download.py b/visual-search/train/download.py
new file mode 100644
index 0000000..0409a76
--- /dev/null
+++ b/visual-search/train/download.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Resumable, threaded image downloader for the DW CLIP fine-tune.
+# Reads manifest.tsv, fetches each catalog image at SMALL size (Shopify CDN width=512 so we pull
+# ~20KB not ~1MB), downscales to 256px, writes cache/<id>.jpg. Skips already-cached ids so it
+# resumes cleanly. Bandwidth ≈ rows × ~25KB (not × 1MB) thanks to the CDN width rewrite.
+import os, io, sys, time, threading, queue, urllib.request, urllib.parse
+from PIL import Image
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+CACHE = os.path.join(HERE, 'cache'); os.makedirs(CACHE, exist_ok=True)
+SIZE = 256; WORKERS = 16
+Image.MAX_IMAGE_PIXELS = None
+
+def small_url(u):
+ # Shopify CDN honors ?width= — ask for 512 so a 256 thumbnail is crisp but the download stays tiny.
+ if 'shopify' in u or 'cdn.shopify' in u:
+ pr = urllib.parse.urlparse(u); q = urllib.parse.parse_qs(pr.query); q['width'] = ['512']
+ return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q, doseq=True)))
+ return u
+
+def fetch(idv, url):
+ out = os.path.join(CACHE, idv + '.jpg')
+ if os.path.exists(out): return 'skip'
+ try:
+ req = urllib.request.Request(small_url(url), headers={'User-Agent': 'dwtrain/1'})
+ data = urllib.request.urlopen(req, timeout=25).read()
+ im = Image.open(io.BytesIO(data)).convert('RGB')
+ im.thumbnail((SIZE, SIZE))
+ im.save(out, 'JPEG', quality=88)
+ return 'ok'
+ except Exception:
+ return 'err'
+
+def main():
+ rows = []
+ with open(os.path.join(HERE, 'manifest.tsv')) as f:
+ for line in f:
+ parts = line.rstrip('\n').split('\t')
+ if len(parts) >= 2 and parts[0] and parts[1].startswith('http'):
+ rows.append((parts[0], parts[1]))
+ q = queue.Queue()
+ for r in rows: q.put(r)
+ total = len(rows)
+ counts = {'ok': 0, 'skip': 0, 'err': 0}
+ lock = threading.Lock(); t0 = time.time()
+ def worker():
+ while True:
+ try: idv, url = q.get_nowait()
+ except queue.Empty: return
+ r = fetch(idv, url)
+ with lock:
+ counts[r] += 1
+ done = counts['ok'] + counts['skip'] + counts['err']
+ if done % 500 == 0 or done == total:
+ rate = done / max(1e-6, time.time() - t0)
+ print(f"{done}/{total} ok={counts['ok']} skip={counts['skip']} err={counts['err']} {rate:.0f}/s", flush=True)
+ q.task_done()
+ ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
+ for t in ts: t.start()
+ for t in ts: t.join()
+ print(f"DONE cached={counts['ok']+counts['skip']} err={counts['err']}", flush=True)
+
+if __name__ == '__main__':
+ main()
diff --git a/visual-search/train/finetune.py b/visual-search/train/finetune.py
new file mode 100644
index 0000000..ae3ce31
--- /dev/null
+++ b/visual-search/train/finetune.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# Fine-tune open_clip ViT-B-32 (laion2b) on the DW catalog so image embeddings specialize to
+# wallcovering/fabric patterns → sharper "really ID the pattern" retrieval than off-the-shelf CLIP.
+# Same architecture + pretrained tag as the live search service, so the fine-tuned visual weights
+# are a DROP-IN replacement. Standard symmetric CLIP contrastive (InfoNCE) loss. Runs on Apple MPS.
+#
+# PYTORCH_ENABLE_MPS_FALLBACK=1 ./venv/bin/python finetune.py --epochs 4 --batch 96
+import os, sys, json, time, argparse, random
+os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
+import torch, torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import open_clip
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+CACHE = os.path.join(HERE, 'cache'); CKPT = os.path.join(HERE, 'ckpt'); os.makedirs(CKPT, exist_ok=True)
+Image.MAX_IMAGE_PIXELS = None
+
+def clean(s): return ' '.join((s or '').replace('|', ' ').split())[:120]
+
+def caption(pattern, color, vendor, ptype):
+ # natural DW-domain caption: "<pattern> in <color>, <vendor> <type>"
+ bits = [clean(pattern)]
+ if color and color.strip(): bits.append('in ' + clean(color))
+ tail = ' '.join(x for x in [clean(vendor), clean(ptype).lower()] if x)
+ txt = ', '.join([b for b in bits if b])
+ if tail: txt += ', ' + tail
+ return txt.lower().strip(', ')
+
+class DWSet(Dataset):
+ def __init__(self, rows, preprocess, tokenizer):
+ self.rows = rows; self.pp = preprocess; self.tok = tokenizer
+ def __len__(self): return len(self.rows)
+ def __getitem__(self, i):
+ idv, pat, col, ven, typ = self.rows[i]
+ img = self.pp(Image.open(os.path.join(CACHE, idv + '.jpg')).convert('RGB'))
+ txt = self.tok([caption(pat, col, ven, typ)])[0]
+ return img, txt
+
+def load_rows():
+ rows = []
+ with open(os.path.join(HERE, 'manifest.tsv')) as f:
+ for line in f:
+ p = line.rstrip('\n').split('\t')
+ if len(p) < 6: continue
+ idv = p[0]
+ if os.path.exists(os.path.join(CACHE, idv + '.jpg')):
+ rows.append((idv, p[2], p[3], p[4], p[5])) # id, pattern, color, vendor, type
+ return rows
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--epochs', type=int, default=4)
+ ap.add_argument('--batch', type=int, default=96)
+ ap.add_argument('--lr', type=float, default=3e-6) # gentle — avoid catastrophic forgetting
+ ap.add_argument('--max_hours', type=float, default=7.5)
+ a = ap.parse_args()
+
+ dev = 'mps' if torch.backends.mps.is_available() else 'cpu'
+ print(f"device={dev}", flush=True)
+ model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
+ tokenizer = open_clip.get_tokenizer('ViT-B-32')
+ model = model.to(dev).float(); model.train()
+
+ rows = load_rows()
+ if len(rows) < 64:
+ print(f"only {len(rows)} cached images — need more; is download.py done? exiting", flush=True); return
+ random.seed(0); random.shuffle(rows)
+ print(f"training pairs (cached): {len(rows)}", flush=True)
+ ds = DWSet(rows, preprocess, tokenizer)
+ dl = DataLoader(ds, batch_size=a.batch, shuffle=True, num_workers=4, drop_last=True, persistent_workers=True)
+
+ opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.1)
+ t0 = time.time(); step = 0
+ for ep in range(a.epochs):
+ for imgs, txts in dl:
+ imgs = imgs.to(dev); txts = txts.to(dev)
+ imf = F.normalize(model.encode_image(imgs), dim=-1)
+ txf = F.normalize(model.encode_text(txts), dim=-1)
+ scale = model.logit_scale.exp().clamp(max=100)
+ logits = scale * imf @ txf.t()
+ labels = torch.arange(logits.size(0), device=dev)
+ loss = (F.cross_entropy(logits, labels) + F.cross_entropy(logits.t(), labels)) / 2
+ opt.zero_grad(); loss.backward(); opt.step()
+ with torch.no_grad(): model.logit_scale.clamp_(0, 4.6052)
+ step += 1
+ if step % 20 == 0:
+ el = (time.time() - t0) / 3600
+ print(f"ep{ep} step{step} loss={loss.item():.4f} elapsed={el:.2f}h", flush=True)
+ if (time.time() - t0) / 3600 > a.max_hours:
+ print("max_hours reached — stopping", flush=True)
+ torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft.pt')); return
+ ckpt = os.path.join(CKPT, f'dw_clip_ft_ep{ep}.pt')
+ torch.save(model.state_dict(), ckpt)
+ print(f"saved {ckpt}", flush=True)
+ torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft.pt'))
+ print(f"DONE — final weights → ckpt/dw_clip_ft.pt ({(time.time()-t0)/3600:.2f}h)", flush=True)
+
+if __name__ == '__main__':
+ main()
diff --git a/visual-search/train/run_overnight.sh b/visual-search/train/run_overnight.sh
new file mode 100755
index 0000000..efebdb4
--- /dev/null
+++ b/visual-search/train/run_overnight.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Overnight DW CLIP fine-tune on this Mac (MPS). Chains: manifest → download (256px cache) → train.
+# Resumable — re-running skips cached images and continues. Logs everything to logs/overnight.log.
+# nohup ./run_overnight.sh > logs/overnight.log 2>&1 &
+set -e
+cd "$(dirname "$0")"
+export DW_UNIFIED_DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
+export PYTORCH_ENABLE_MPS_FALLBACK=1
+PY=./venv/bin/python
+stamp(){ echo "[$(date '+%F %T')] $*"; }
+
+stamp "STEP 1/3 — build manifest"
+[ -f manifest.tsv ] || bash build_manifest.sh
+stamp "manifest rows: $(wc -l < manifest.tsv)"
+
+stamp "STEP 2/3 — download 256px image cache (resumable)"
+$PY download.py
+stamp "cache images: $(ls cache | wc -l)"
+
+stamp "STEP 3/3 — fine-tune CLIP on MPS"
+$PY finetune.py --epochs 5 --batch 96 --max_hours 7.0
+
+stamp "OVERNIGHT RUN COMPLETE — weights at ckpt/dw_clip_ft.pt"
← 942ce16 5x: 2 consecutive clean sweeps (0 app defects) — six-way + f
·
back to Dw Photo Capture
·
Visual pattern-ID in capture flow (front photo → identify-mu b87d402 →