← back to Dw Photo Capture
snapshot before restart: preserve in-flight work (auto-saved by /restart pre-reboot)
7575b3827f139619e21ca3ae99dc97accdb5df23 · 2026-08-17 06:52:32 -0700 · Steve
Files touched
D visual-search/train/build_manifest.shD visual-search/train/download.pyD visual-search/train/eval_aug.pyD visual-search/train/finetune.pyD visual-search/train/finetune_aug.pyD visual-search/train/mac_deploy_embed.pyD visual-search/train/robustness_fusion.pyD visual-search/train/robustness_test.pyD visual-search/train/run_overnight.shD visual-search/train/sanity_check.py
Diff
commit 7575b3827f139619e21ca3ae99dc97accdb5df23
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Aug 17 06:52:32 2026 -0700
snapshot before restart: preserve in-flight work (auto-saved by /restart pre-reboot)
---
visual-search/train/build_manifest.sh | 26 --------
visual-search/train/download.py | 64 -------------------
visual-search/train/eval_aug.py | 58 ------------------
visual-search/train/finetune.py | 100 ------------------------------
visual-search/train/finetune_aug.py | 94 ----------------------------
visual-search/train/mac_deploy_embed.py | 102 -------------------------------
visual-search/train/robustness_fusion.py | 53 ----------------
visual-search/train/robustness_test.py | 73 ----------------------
visual-search/train/run_overnight.sh | 23 -------
visual-search/train/sanity_check.py | 61 ------------------
10 files changed, 654 deletions(-)
diff --git a/visual-search/train/build_manifest.sh b/visual-search/train/build_manifest.sh
deleted file mode 100755
index 7fcebb5..0000000
--- a/visual-search/train/build_manifest.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-# Build the training manifest from the LOCAL dw_unified mirror → manifest.tsv
-# Columns: id \t image_url \t pattern_name \t color \t vendor \t product_type
-# One catalog image + its text = one CLIP contrastive pair. Only rows with a real
-# pattern name and an http image are usable. Color is cleaned of the JSON-blob form.
-set -e
-cd "$(dirname "$0")"
-DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
-LIMIT="${1:-100000}" # 100k diverse pairs = a strong overnight fine-tune (many epochs) without a 245k download
-psql "$DB" -F$'\t' -tA -c "
- select id,
- image_url,
- regexp_replace(pattern_name, E'[\t\n\r]', ' ', 'g'),
- -- color_name may be plain text OR a JSON blob {\"Name\":\"Beige\",...}; pull the Name
- coalesce(
- nullif(regexp_replace(coalesce(color_name,''), E'.*\"Name\":\\s*\"([^\"]+)\".*', E'\\1'), color_name),
- color_primary, ''),
- coalesce(original_vendor_name, vendor_code, ''),
- coalesce(product_type, 'Wallcovering')
- from vendor_catalog
- where image_url like 'http%'
- and pattern_name is not null and pattern_name <> ''
- order by random()
- limit $LIMIT
-" > manifest.tsv
-echo "manifest rows: $(wc -l < manifest.tsv)"
diff --git a/visual-search/train/download.py b/visual-search/train/download.py
deleted file mode 100644
index 0409a76..0000000
--- a/visual-search/train/download.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-# Resumable, threaded image downloader for the DW CLIP fine-tune.
-# Reads manifest.tsv, fetches each catalog image at SMALL size (Shopify CDN width=512 so we pull
-# ~20KB not ~1MB), downscales to 256px, writes cache/<id>.jpg. Skips already-cached ids so it
-# resumes cleanly. Bandwidth ≈ rows × ~25KB (not × 1MB) thanks to the CDN width rewrite.
-import os, io, sys, time, threading, queue, urllib.request, urllib.parse
-from PIL import Image
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-CACHE = os.path.join(HERE, 'cache'); os.makedirs(CACHE, exist_ok=True)
-SIZE = 256; WORKERS = 16
-Image.MAX_IMAGE_PIXELS = None
-
-def small_url(u):
- # Shopify CDN honors ?width= — ask for 512 so a 256 thumbnail is crisp but the download stays tiny.
- if 'shopify' in u or 'cdn.shopify' in u:
- pr = urllib.parse.urlparse(u); q = urllib.parse.parse_qs(pr.query); q['width'] = ['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q, doseq=True)))
- return u
-
-def fetch(idv, url):
- out = os.path.join(CACHE, idv + '.jpg')
- if os.path.exists(out): return 'skip'
- try:
- req = urllib.request.Request(small_url(url), headers={'User-Agent': 'dwtrain/1'})
- data = urllib.request.urlopen(req, timeout=25).read()
- im = Image.open(io.BytesIO(data)).convert('RGB')
- im.thumbnail((SIZE, SIZE))
- im.save(out, 'JPEG', quality=88)
- return 'ok'
- except Exception:
- return 'err'
-
-def main():
- rows = []
- with open(os.path.join(HERE, 'manifest.tsv')) as f:
- for line in f:
- parts = line.rstrip('\n').split('\t')
- if len(parts) >= 2 and parts[0] and parts[1].startswith('http'):
- rows.append((parts[0], parts[1]))
- q = queue.Queue()
- for r in rows: q.put(r)
- total = len(rows)
- counts = {'ok': 0, 'skip': 0, 'err': 0}
- lock = threading.Lock(); t0 = time.time()
- def worker():
- while True:
- try: idv, url = q.get_nowait()
- except queue.Empty: return
- r = fetch(idv, url)
- with lock:
- counts[r] += 1
- done = counts['ok'] + counts['skip'] + counts['err']
- if done % 500 == 0 or done == total:
- rate = done / max(1e-6, time.time() - t0)
- print(f"{done}/{total} ok={counts['ok']} skip={counts['skip']} err={counts['err']} {rate:.0f}/s", flush=True)
- q.task_done()
- ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
- for t in ts: t.start()
- for t in ts: t.join()
- print(f"DONE cached={counts['ok']+counts['skip']} err={counts['err']}", flush=True)
-
-if __name__ == '__main__':
- main()
diff --git a/visual-search/train/eval_aug.py b/visual-search/train/eval_aug.py
deleted file mode 100644
index 6091917..0000000
--- a/visual-search/train/eval_aug.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-# A/B the augmentation re-fine-tune on the REAL task: degraded-query → clean-gallery image retrieval.
-# Held-out catalog images (NOT in the training cache). Gallery = clean embeddings; queries = the SAME
-# images degraded (handheld-photo sim). top-1 = does the degraded query retrieve its own clean image.
-# Compares base CLIP vs dw_clip_ft.pt (current) vs dw_clip_ft_aug.pt (new) → recommends the winner.
-import os, io, glob, urllib.request, urllib.parse, subprocess, random
-os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
-import torch, torch.nn.functional as F, open_clip
-from PIL import Image
-import finetune_aug as A # reuse augment()
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-DB = os.environ["DW_UNIFIED_DB"]; N = 300
-trained = set(os.path.splitext(f)[0] for f in os.listdir(os.path.join(HERE, 'cache')))
-
-def small(u):
- if 'shopify' in u:
- pr=urllib.parse.urlparse(u);q=urllib.parse.parse_qs(pr.query);q['width']=['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q,doseq=True)))
- return u
-
-sql = ("select id,image_url from vendor_catalog where left(image_url,4)='http' "
- "and pattern_name is not null and pattern_name<>'' order by random() limit 1500")
-rows = [l.split('\t') for l in subprocess.check_output(['psql',DB,'-F','\t','-tA','-c',sql]).decode().splitlines() if '\t' in l]
-heldout = [r for r in rows if r[0] not in trained][:N]
-
-random.seed(0)
-clean, deglabel = [], []
-for idv,url in heldout:
- try:
- data=urllib.request.urlopen(urllib.request.Request(small(url),headers={'User-Agent':'e/1'}),timeout=20).read()
- im=Image.open(io.BytesIO(data)).convert('RGB'); clean.append(im); deglabel.append(A.augment(im.copy()))
- except Exception: pass
-print(f"held-out pairs: {len(clean)}", flush=True)
-
-dev='mps' if torch.backends.mps.is_available() else 'cpu'
-model,_,pp=open_clip.create_model_and_transforms('ViT-B-32',pretrained='laion2b_s34b_b79k'); model=model.to(dev).float().eval()
-CL=torch.stack([pp(i) for i in clean]).to(dev); DG=torch.stack([pp(i) for i in deglabel]).to(dev)
-
-def top1(sd):
- if sd is None: model.load_state_dict(open_clip.create_model('ViT-B-32',pretrained='laion2b_s34b_b79k').state_dict(),strict=False)
- else: model.load_state_dict(sd,strict=False)
- with torch.no_grad():
- g=F.normalize(model.encode_image(CL),dim=-1); q=F.normalize(model.encode_image(DG),dim=-1)
- sim=q@g.t(); top=sim.argmax(dim=1); acc=(top==torch.arange(len(clean),device=dev)).float().mean().item()
- # also top-5
- t5=sim.topk(5,dim=1).indices; r5=sum(1 for i in range(len(clean)) if i in t5[i]).__truediv__(len(clean))
- return acc*100, r5*100
-
-print(f"\n{'model':<26} deg→clean top-1 top-5")
-a1,a5=top1(None); print(f"{'base laion2b':<26} {a1:5.1f}% {a5:5.1f}%")
-b1,b5=top1(torch.load(os.path.join(HERE,'ckpt','dw_clip_ft.pt'),map_location='cpu')); print(f"{'dw_clip_ft (current)':<26} {b1:5.1f}% {b5:5.1f}%")
-aug=os.path.join(HERE,'ckpt','dw_clip_ft_aug.pt')
-if os.path.exists(aug):
- c1,c5=top1(torch.load(aug,map_location='cpu')); print(f"{'dw_clip_ft_aug (NEW)':<26} {c1:5.1f}% {c5:5.1f}%")
- print(f"\n★ AUG vs current: top-1 {c1:.1f}% vs {b1:.1f}% → {'DEPLOY (better)' if c1>b1+1 else 'KEEP CURRENT (not better)'}")
-else:
- print("\n(dw_clip_ft_aug.pt not present yet — run after training finishes)")
diff --git a/visual-search/train/finetune.py b/visual-search/train/finetune.py
deleted file mode 100644
index ae3ce31..0000000
--- a/visual-search/train/finetune.py
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env python3
-# Fine-tune open_clip ViT-B-32 (laion2b) on the DW catalog so image embeddings specialize to
-# wallcovering/fabric patterns → sharper "really ID the pattern" retrieval than off-the-shelf CLIP.
-# Same architecture + pretrained tag as the live search service, so the fine-tuned visual weights
-# are a DROP-IN replacement. Standard symmetric CLIP contrastive (InfoNCE) loss. Runs on Apple MPS.
-#
-# PYTORCH_ENABLE_MPS_FALLBACK=1 ./venv/bin/python finetune.py --epochs 4 --batch 96
-import os, sys, json, time, argparse, random
-os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
-import torch, torch.nn.functional as F
-from torch.utils.data import Dataset, DataLoader
-from PIL import Image
-import open_clip
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-CACHE = os.path.join(HERE, 'cache'); CKPT = os.path.join(HERE, 'ckpt'); os.makedirs(CKPT, exist_ok=True)
-Image.MAX_IMAGE_PIXELS = None
-
-def clean(s): return ' '.join((s or '').replace('|', ' ').split())[:120]
-
-def caption(pattern, color, vendor, ptype):
- # natural DW-domain caption: "<pattern> in <color>, <vendor> <type>"
- bits = [clean(pattern)]
- if color and color.strip(): bits.append('in ' + clean(color))
- tail = ' '.join(x for x in [clean(vendor), clean(ptype).lower()] if x)
- txt = ', '.join([b for b in bits if b])
- if tail: txt += ', ' + tail
- return txt.lower().strip(', ')
-
-class DWSet(Dataset):
- def __init__(self, rows, preprocess, tokenizer):
- self.rows = rows; self.pp = preprocess; self.tok = tokenizer
- def __len__(self): return len(self.rows)
- def __getitem__(self, i):
- idv, pat, col, ven, typ = self.rows[i]
- img = self.pp(Image.open(os.path.join(CACHE, idv + '.jpg')).convert('RGB'))
- txt = self.tok([caption(pat, col, ven, typ)])[0]
- return img, txt
-
-def load_rows():
- rows = []
- with open(os.path.join(HERE, 'manifest.tsv')) as f:
- for line in f:
- p = line.rstrip('\n').split('\t')
- if len(p) < 6: continue
- idv = p[0]
- if os.path.exists(os.path.join(CACHE, idv + '.jpg')):
- rows.append((idv, p[2], p[3], p[4], p[5])) # id, pattern, color, vendor, type
- return rows
-
-def main():
- ap = argparse.ArgumentParser()
- ap.add_argument('--epochs', type=int, default=4)
- ap.add_argument('--batch', type=int, default=96)
- ap.add_argument('--lr', type=float, default=3e-6) # gentle — avoid catastrophic forgetting
- ap.add_argument('--max_hours', type=float, default=7.5)
- a = ap.parse_args()
-
- dev = 'mps' if torch.backends.mps.is_available() else 'cpu'
- print(f"device={dev}", flush=True)
- model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
- tokenizer = open_clip.get_tokenizer('ViT-B-32')
- model = model.to(dev).float(); model.train()
-
- rows = load_rows()
- if len(rows) < 64:
- print(f"only {len(rows)} cached images — need more; is download.py done? exiting", flush=True); return
- random.seed(0); random.shuffle(rows)
- print(f"training pairs (cached): {len(rows)}", flush=True)
- ds = DWSet(rows, preprocess, tokenizer)
- dl = DataLoader(ds, batch_size=a.batch, shuffle=True, num_workers=4, drop_last=True, persistent_workers=True)
-
- opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.1)
- t0 = time.time(); step = 0
- for ep in range(a.epochs):
- for imgs, txts in dl:
- imgs = imgs.to(dev); txts = txts.to(dev)
- imf = F.normalize(model.encode_image(imgs), dim=-1)
- txf = F.normalize(model.encode_text(txts), dim=-1)
- scale = model.logit_scale.exp().clamp(max=100)
- logits = scale * imf @ txf.t()
- labels = torch.arange(logits.size(0), device=dev)
- loss = (F.cross_entropy(logits, labels) + F.cross_entropy(logits.t(), labels)) / 2
- opt.zero_grad(); loss.backward(); opt.step()
- with torch.no_grad(): model.logit_scale.clamp_(0, 4.6052)
- step += 1
- if step % 20 == 0:
- el = (time.time() - t0) / 3600
- print(f"ep{ep} step{step} loss={loss.item():.4f} elapsed={el:.2f}h", flush=True)
- if (time.time() - t0) / 3600 > a.max_hours:
- print("max_hours reached — stopping", flush=True)
- torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft.pt')); return
- ckpt = os.path.join(CKPT, f'dw_clip_ft_ep{ep}.pt')
- torch.save(model.state_dict(), ckpt)
- print(f"saved {ckpt}", flush=True)
- torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft.pt'))
- print(f"DONE — final weights → ckpt/dw_clip_ft.pt ({(time.time()-t0)/3600:.2f}h)", flush=True)
-
-if __name__ == '__main__':
- main()
diff --git a/visual-search/train/finetune_aug.py b/visual-search/train/finetune_aug.py
deleted file mode 100644
index 845b37f..0000000
--- a/visual-search/train/finetune_aug.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-# AUGMENTATION re-fine-tune: warm-start from dw_clip_ft.pt and continue training with heavy
-# handheld-photo augmentation (rotate/crop/blur/glare/JPEG) on the IMAGE side, so the model learns
-# that a degraded phone shot maps to the same catalog identity — the gap robustness_test.py exposed
-# (28% top-1 on degraded photos). Caption stays clean; only the image is degraded during training.
-# PYTORCH_ENABLE_MPS_FALLBACK=1 ./venv/bin/python finetune_aug.py --epochs 3 --batch 96
-import os, sys, io, time, random, argparse
-os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
-import torch, torch.nn.functional as F
-from torch.utils.data import Dataset, DataLoader
-from PIL import Image, ImageFilter, ImageEnhance
-import open_clip
-import finetune as T # reuse caption() + load_rows()
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-CACHE = os.path.join(HERE, 'cache'); CKPT = os.path.join(HERE, 'ckpt')
-Image.MAX_IMAGE_PIXELS = None
-
-def augment(im):
- # simulate a handheld iPhone shot of a physical sample — varied strength, each applied ~probabilistically
- if im.width < 48 or im.height < 48: return im # too small to degrade safely (corrupt/tiny cache)
- w, h = im.size
- if random.random() < 0.7: im = im.rotate(random.uniform(-8, 8), expand=False, fillcolor=(238,238,238))
- if random.random() < 0.7:
- cx, cy = random.uniform(0.02, 0.10), random.uniform(0.02, 0.10)
- l,t,r,bt = int(w*cx), int(h*cy), int(w*(1-cx)), int(h*(1-cy))
- if r-l >= 32 and bt-t >= 32: im = im.crop((l,t,r,bt)) # never crop to an empty/tiny box
- if random.random() < 0.6: im = im.filter(ImageFilter.GaussianBlur(random.uniform(0.4, 1.6)))
- if random.random() < 0.7: im = ImageEnhance.Brightness(im).enhance(random.uniform(0.85, 1.30)) # glare / shadow
- if random.random() < 0.5: im = ImageEnhance.Contrast(im).enhance(random.uniform(0.80, 1.15))
- if random.random() < 0.6 and im.width >= 32 and im.height >= 32:
- try:
- b = io.BytesIO(); im.save(b, 'JPEG', quality=random.randint(45, 80)); b.seek(0); im = Image.open(b).convert('RGB')
- except Exception: pass # a bad recompress must never crash training
- return im
-
-class AugSet(Dataset):
- def __init__(self, rows, preprocess, tokenizer):
- self.rows = rows; self.pp = preprocess; self.tok = tokenizer
- def __len__(self): return len(self.rows)
- def __getitem__(self, i):
- idv, pat, col, ven, typ = self.rows[i]
- try:
- im = Image.open(os.path.join(CACHE, idv + '.jpg')).convert('RGB')
- img = self.pp(augment(im)) # <-- degrade before CLIP preprocess
- except Exception:
- return self.__getitem__((i + 1) % len(self.rows)) # corrupt cache image → use a neighbor, never crash
- txt = self.tok([T.caption(pat, col, ven, typ)])[0]
- return img, txt
-
-def main():
- ap = argparse.ArgumentParser()
- ap.add_argument('--epochs', type=int, default=3)
- ap.add_argument('--batch', type=int, default=96)
- ap.add_argument('--lr', type=float, default=2e-6) # gentle — warm-starting an already-good model
- ap.add_argument('--max_hours', type=float, default=3.0)
- a = ap.parse_args()
-
- dev = 'mps' if torch.backends.mps.is_available() else 'cpu'
- model, _, preprocess = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
- tokenizer = open_clip.get_tokenizer('ViT-B-32')
- warm = os.path.join(CKPT, 'dw_clip_ft.pt') # WARM-START from the current fine-tuned model
- model.load_state_dict(torch.load(warm, map_location='cpu'), strict=False)
- print('warm-started from dw_clip_ft.pt · device', dev, flush=True)
- model = model.to(dev).float(); model.train()
-
- rows = T.load_rows()
- print(f'training pairs (cached): {len(rows)}', flush=True)
- dl = DataLoader(AugSet(rows, preprocess, tokenizer), batch_size=a.batch, shuffle=True,
- num_workers=5, drop_last=True, persistent_workers=True)
- opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.1)
- t0 = time.time(); step = 0
- for ep in range(a.epochs):
- for imgs, txts in dl:
- imgs = imgs.to(dev); txts = txts.to(dev)
- imf = F.normalize(model.encode_image(imgs), dim=-1)
- txf = F.normalize(model.encode_text(txts), dim=-1)
- scale = model.logit_scale.exp().clamp(max=100)
- logits = scale * imf @ txf.t()
- labels = torch.arange(logits.size(0), device=dev)
- loss = (F.cross_entropy(logits, labels) + F.cross_entropy(logits.t(), labels)) / 2
- opt.zero_grad(); loss.backward(); opt.step()
- with torch.no_grad(): model.logit_scale.clamp_(0, 4.6052)
- step += 1
- if step % 20 == 0: print(f'ep{ep} step{step} loss={loss.item():.4f} elapsed={(time.time()-t0)/3600:.2f}h', flush=True)
- if (time.time()-t0)/3600 > a.max_hours:
- torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft_aug.pt')); print('max_hours — saved', flush=True); return
- torch.save(model.state_dict(), os.path.join(CKPT, f'dw_clip_ft_aug_ep{ep}.pt'))
- print(f'saved ep{ep}', flush=True)
- torch.save(model.state_dict(), os.path.join(CKPT, 'dw_clip_ft_aug.pt'))
- print(f'DONE — aug weights → ckpt/dw_clip_ft_aug.pt ({(time.time()-t0)/3600:.2f}h)', flush=True)
-
-if __name__ == '__main__':
- main()
diff --git a/visual-search/train/mac_deploy_embed.py b/visual-search/train/mac_deploy_embed.py
deleted file mode 100644
index d179ff8..0000000
--- a/visual-search/train/mac_deploy_embed.py
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-# LOCAL deploy embedder: embed the whole catalog with the AUGMENTED model (dw_clip_ft_aug.pt) on the
-# Mac's MPS, writing a Postgres COPY file (text format) for Kamatera to ingest LOCALLY. No prod-DB
-# connection from the Mac — it only produces a data file. Reuses the 90k training image cache; dead
-# URLs get a NULL-embedding marker (so the index self-terminates, same as the daemon). Resumable.
-import os, io, sys, csv, time, threading, queue, urllib.request, urllib.parse
-os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
-import torch, torch.nn.functional as F, open_clip
-from PIL import Image
-csv.field_size_limit(10_000_000); Image.MAX_IMAGE_PIXELS = None
-
-HERE = os.path.dirname(os.path.abspath(__file__)); CACHE = os.path.join(HERE, 'cache')
-MANI = os.path.join(HERE, 'deploy_manifest.tsv'); OUT = os.path.join(HERE, 'deploy_embeddings.copy')
-DONE = os.path.join(HERE, 'deploy_done.txt')
-UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
-WORKERS = 20; EMB_BATCH = 128; CHUNK = 1500
-
-model, _, pp = open_clip.create_model_and_transforms("ViT-B-32", pretrained="laion2b_s34b_b79k")
-model.load_state_dict(torch.load(os.path.join(HERE, 'ckpt', 'dw_clip_ft_aug.pt'), map_location='cpu'), strict=False)
-dev = 'mps' if torch.backends.mps.is_available() else 'cpu'
-model = model.to(dev).float().eval(); DIM = model.visual.output_dim
-print(f"AUG model loaded · device {dev} · dim {DIM}", flush=True)
-
-def small(u):
- if 'shopify' in u:
- pr = urllib.parse.urlparse(u); q = urllib.parse.parse_qs(pr.query); q['width'] = ['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q, doseq=True)))
- return u
-def esc(s): return (s or '').replace('\\', '\\\\').replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')
-
-def load_img(row):
- # returns (row, tensor)=ok · (row,'DEAD')=genuine 404/410 → NULL marker · (row,None)=transient → retry later
- idv, dw, mfr, vc, pat, url = row
- cp = os.path.join(CACHE, idv + '.jpg')
- if os.path.exists(cp) and os.path.getsize(cp) > 200:
- try: return (row, pp(Image.open(cp).convert('RGB')))
- except Exception: pass
- dead = False
- for attempt in range(3):
- try:
- data = urllib.request.urlopen(urllib.request.Request(small(url), headers={'User-Agent': UA}), timeout=25).read()
- im = Image.open(io.BytesIO(data)).convert('RGB')
- try: im.copy().save(cp, 'JPEG', quality=88) # populate cache for resumes
- except Exception: pass
- return (row, pp(im))
- except urllib.error.HTTPError as e:
- if e.code in (404, 410, 403 if False else 0): dead = True; break # genuinely gone
- if e.code in (429, 500, 502, 503): # rate-limit / transient → back off + retry
- time.sleep(1.5 * (attempt + 1)); continue
- dead = True; break
- except Exception:
- time.sleep(1.0 * (attempt + 1)); continue # network hiccup → retry
- return (row, 'DEAD' if dead else None)
-
-def main():
- done = set()
- if os.path.exists(DONE):
- with open(DONE) as f: done = set(l.strip() for l in f if l.strip())
- rows = []
- with open(MANI) as f:
- for r in csv.reader(f, delimiter='\t'):
- if len(r) >= 6 and r[0] and r[0] not in done and r[5].startswith('http'): rows.append(r)
- print(f"to embed: {len(rows)} (already done {len(done)})", flush=True)
- fout = open(OUT, 'a'); fdone = open(DONE, 'a')
- t0 = time.time(); total = 0
- for i in range(0, len(rows), CHUNK):
- batch = rows[i:i+CHUNK]
- got = [None]*len(batch); q = queue.Queue()
- for j, r in enumerate(batch): q.put((j, r))
- lock = threading.Lock()
- def worker():
- while True:
- try: j, r = q.get_nowait()
- except queue.Empty: return
- got[j] = load_img(r); q.task_done()
- ts = [threading.Thread(target=worker, daemon=True) for _ in range(WORKERS)]
- for t in ts: t.start()
- for t in ts: t.join()
- ok = [g for g in got if g and not isinstance(g[1], str) and g[1] is not None]
- bad = [g for g in got if g and g[1] == 'DEAD'] # genuine 404 → NULL marker
- # g[1] is None → transient failure: NOT written, NOT marked done → retried on the next resume pass
- # embed the good ones in sub-batches
- for k in range(0, len(ok), EMB_BATCH):
- sub = ok[k:k+EMB_BATCH]
- with torch.no_grad():
- v = model.encode_image(torch.stack([g[1] for g in sub]).to(dev))
- v = (v / v.norm(dim=-1, keepdim=True)).cpu().numpy().astype('float32')
- for g, emb in zip(sub, v):
- r = g[0]; hx = emb.tobytes().hex()
- fout.write(f"{r[0]}\t{esc(r[1])}\t{esc(r[2])}\t{esc(r[3])}\t{esc(r[4])}\t{esc(r[5])}\t{DIM}\t\\\\x{hx}\n")
- fdone.write(r[0] + "\n")
- for g in bad:
- r = g[0]
- fout.write(f"{r[0]}\t{esc(r[1])}\t{esc(r[2])}\t{esc(r[3])}\t{esc(r[4])}\t{esc(r[5])}\t\\N\t\\N\n")
- fdone.write(r[0] + "\n")
- fout.flush(); fdone.flush(); total += len(batch)
- print(f"{total}/{len(rows)} embedded {total/max(1e-6,time.time()-t0):.0f}/s", flush=True)
- fout.close(); fdone.close()
- print(f"DONE — {total} rows → {OUT}", flush=True)
-
-if __name__ == '__main__':
- main()
diff --git a/visual-search/train/robustness_fusion.py b/visual-search/train/robustness_fusion.py
deleted file mode 100644
index ef6e0cc..0000000
--- a/visual-search/train/robustness_fusion.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-# Quantify the multi-photo FUSION win: for each held-out item, query the LIVE identify-multi with
-# 1 degraded view vs 3 independent degraded views (fusion), measure top-1 correct on each.
-import os, io, sys, json, base64, subprocess, urllib.request, urllib.parse, ssl, random
-from PIL import Image, ImageFilter, ImageEnhance
-
-APP = "https://photo.designerwallcoverings.com/api/identify-multi"
-AUTH = base64.b64encode(b"admin:DW2024!").decode()
-N = 40; ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
-random.seed(1)
-
-cmd = ('DB="$(grep ^DW_UNIFIED_DB= /root/public-projects/dwphoto/.env|cut -d= -f2-)"; '
- 'psql "$DB" -F "|" -tA -c "select e.dw_sku, e.image_url from image_embeddings e where e.embedding is not null '
- "and left(e.image_url,4)='http' order by random() limit " + str(N) + '"')
-rows = [l.split('|',1) for l in subprocess.check_output(['ssh','root@45.61.58.125',cmd]).decode().splitlines() if '|' in l]
-
-def small(u):
- if 'shopify' in u:
- pr=urllib.parse.urlparse(u);q=urllib.parse.parse_qs(pr.query);q['width']=['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q,doseq=True)))
- return u
-def degrade(im):
- w,h=im.size
- im=im.rotate(random.uniform(-8,8),fillcolor=(238,238,238))
- cx,cy=random.uniform(0.02,0.10),random.uniform(0.02,0.10); im=im.crop((int(w*cx),int(h*cy),int(w*(1-cx)),int(h*(1-cy))))
- im=im.filter(ImageFilter.GaussianBlur(random.uniform(0.5,1.6)))
- im=ImageEnhance.Brightness(im).enhance(random.uniform(0.85,1.28))
- b=io.BytesIO(); im.save(b,'JPEG',quality=random.randint(45,75)); return Image.open(io.BytesIO(b.getvalue())).convert('RGB')
-def durl(im): b=io.BytesIO();im.save(b,'JPEG',quality=80);return 'data:image/jpeg;base64,'+base64.b64encode(b.getvalue()).decode()
-def query(payload):
- req=urllib.request.Request(APP,data=json.dumps(payload).encode(),headers={'Content-Type':'application/json','Authorization':'Basic '+AUTH})
- r=json.load(urllib.request.urlopen(req,timeout=30,context=ctx)); v=r.get('visual') or []
- return (str(v[0].get('dw_sku')) if v else None)
-
-s1=s3=n=0
-for dw,url in rows:
- try:
- data=urllib.request.urlopen(urllib.request.Request(small(url),headers={'User-Agent':'f/1'}),timeout=20).read()
- im=Image.open(io.BytesIO(data)).convert('RGB')
- v1=durl(degrade(im.copy())); views=[durl(degrade(im.copy())) for _ in range(3)]
- n+=1
- if query({'front':v1})==dw: s1+=1
- if query({'fronts':views})==dw: s3+=1
- except Exception as e: print('skip',e,flush=True)
-print(f"\n=== single-view vs 3-view FUSION (live index), n={n} ===")
-print(f"1 view top-1: {s1}/{n} ({100*s1/max(1,n):.0f}%)")
-print(f"3 views top-1: {s3}/{n} ({100*s3/max(1,n):.0f}%) → {'+' if s3>=s1 else ''}{100*(s3-s1)/max(1,n):.0f} pts")
-
-# NOTE: this A/B is unreliable in practice — the vendor/Shopify CDNs rate-limit (HTTP 403) the
-# harness's image downloads after volume, and clean-vs-degraded dw_sku comparison is sensitive to
-# CDN availability. The FUSION FEATURE itself is verified directly (3 views of one swatch → the
-# corroborated product 'Atlas' ranks #1 with views=3 over single-view noise). Treat this script's
-# numbers as CDN-permitting only; re-run when the rate limit has reset.
diff --git a/visual-search/train/robustness_test.py b/visual-search/train/robustness_test.py
deleted file mode 100644
index a5b66d2..0000000
--- a/visual-search/train/robustness_test.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-# Real-world-ish validation of the fine-tuned visual-ID: does a DEGRADED "phone-like" photo of a
-# catalog item still retrieve the CORRECT product from the live FT index — and do the confidence
-# thresholds (>=0.92 high, >=0.85 medium) mean anything on that query distribution?
-# We can't shoot real samples autonomously, so we simulate handheld capture: downscale, JPEG-recompress,
-# blur, brightness/glare, small rotate+crop — then query the LIVE /api/identify-multi endpoint.
-import os, io, sys, json, base64, subprocess, urllib.request, urllib.parse, ssl
-from PIL import Image, ImageFilter, ImageEnhance
-
-APP = "https://photo.designerwallcoverings.com/api/identify-multi"
-AUTH = base64.b64encode(b"admin:DW2024!").decode()
-N = 40
-ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
-
-# pull N random rows that ARE embedded on KAMATERA (the live FT index) — so the correct answer is in it
-remote_sql = ("select e.dw_sku, e.image_url from image_embeddings e where e.embedding is not null "
- "and left(e.image_url,4)='http' order by random() limit " + str(N))
-cmd = ('DB="$(grep ^DW_UNIFIED_DB= /root/public-projects/dwphoto/.env|cut -d= -f2-)"; '
- 'psql "$DB" -F "|" -tA -c "' + remote_sql + '"')
-out = subprocess.check_output(['ssh','root@45.61.58.125', cmd]).decode()
-rows = [l.split('|',1) for l in out.splitlines() if '|' in l]
-print(f"pulled {len(rows)} live-indexed rows", flush=True)
-
-def small(u):
- if 'shopify' in u:
- pr=urllib.parse.urlparse(u);q=urllib.parse.parse_qs(pr.query);q['width']=['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q,doseq=True)))
- return u
-
-def degrade(im):
- # mimic a handheld iPhone shot of a physical sample
- w,h = im.size
- im = im.rotate(4, expand=False, fillcolor=(240,240,240)) # slight tilt
- im = im.crop((int(w*0.06), int(h*0.06), int(w*0.94), int(h*0.94))) # off-center crop
- im = im.resize((max(64,im.size[0]//2), max(64,im.size[1]//2))) # lower res (distance)
- im = im.filter(ImageFilter.GaussianBlur(1.2)) # hand shake / focus
- im = ImageEnhance.Brightness(im).enhance(1.18) # glare/overexposure
- im = ImageEnhance.Contrast(im).enhance(0.92)
- b=io.BytesIO(); im.save(b,'JPEG',quality=55); b.seek(0) # phone JPEG compression
- return Image.open(b).convert('RGB')
-
-def query(im):
- bio=io.BytesIO(); im.save(bio,'JPEG',quality=80)
- b64='data:image/jpeg;base64,'+base64.b64encode(bio.getvalue()).decode()
- req=urllib.request.Request(APP, data=json.dumps({'front':b64}).encode(),
- headers={'Content-Type':'application/json','Authorization':'Basic '+AUTH,'User-Agent':'rt/1'})
- return json.load(urllib.request.urlopen(req, timeout=30, context=ctx))
-
-top1=0; top5=0; conf={'high':0,'medium':0,'low':0}; scored=[]; n=0
-for dw_sku,url in rows:
- try:
- data=urllib.request.urlopen(urllib.request.Request(small(url),headers={'User-Agent':'rt/1'}),timeout=20).read()
- q=degrade(Image.open(io.BytesIO(data)).convert('RGB'))
- r=query(q); vis=r.get('visual') or []
- if not vis: continue
- n+=1
- skus=[str(v.get('dw_sku')) for v in vis]
- if skus and skus[0]==dw_sku: top1+=1
- if dw_sku in skus[:5]: top5+=1
- conf[r.get('confidence','low')] = conf.get(r.get('confidence','low'),0)+1
- scored.append((dw_sku, round(vis[0].get('score',0),3), skus[0]==dw_sku, r.get('confidence')))
- except Exception as e:
- print('skip',e, flush=True)
-
-print(f"\n=== DEGRADED (phone-like) image→image retrieval on the LIVE FT index, n={n} ===")
-print(f"top-1 correct: {top1}/{n} ({100*top1/max(1,n):.0f}%) top-5: {top5}/{n} ({100*top5/max(1,n):.0f}%)")
-print(f"confidence buckets: {conf}")
-correct_scores=[s for _,s,c,_ in scored if c]; wrong_scores=[s for _,s,c,_ in scored if not c]
-if correct_scores: print(f"top score when CORRECT: min={min(correct_scores):.3f} avg={sum(correct_scores)/len(correct_scores):.3f}")
-if wrong_scores: print(f"top score when WRONG: max={max(wrong_scores):.3f} avg={sum(wrong_scores)/len(wrong_scores):.3f}")
-# threshold sanity: of matches labeled high (>=0.92), how many were actually correct?
-high=[c for _,s,c,cf in scored if cf=='high'];
-if high: print(f"of 'high'-confidence results, {sum(high)}/{len(high)} were the CORRECT product ({100*sum(high)/len(high):.0f}%)")
diff --git a/visual-search/train/run_overnight.sh b/visual-search/train/run_overnight.sh
deleted file mode 100755
index efebdb4..0000000
--- a/visual-search/train/run_overnight.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Overnight DW CLIP fine-tune on this Mac (MPS). Chains: manifest → download (256px cache) → train.
-# Resumable — re-running skips cached images and continues. Logs everything to logs/overnight.log.
-# nohup ./run_overnight.sh > logs/overnight.log 2>&1 &
-set -e
-cd "$(dirname "$0")"
-export DW_UNIFIED_DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
-export PYTORCH_ENABLE_MPS_FALLBACK=1
-PY=./venv/bin/python
-stamp(){ echo "[$(date '+%F %T')] $*"; }
-
-stamp "STEP 1/3 — build manifest"
-[ -f manifest.tsv ] || bash build_manifest.sh
-stamp "manifest rows: $(wc -l < manifest.tsv)"
-
-stamp "STEP 2/3 — download 256px image cache (resumable)"
-$PY download.py
-stamp "cache images: $(ls cache | wc -l)"
-
-stamp "STEP 3/3 — fine-tune CLIP on MPS"
-$PY finetune.py --epochs 5 --batch 96 --max_hours 7.0
-
-stamp "OVERNIGHT RUN COMPLETE — weights at ckpt/dw_clip_ft.pt"
diff --git a/visual-search/train/sanity_check.py b/visual-search/train/sanity_check.py
deleted file mode 100644
index a736c2b..0000000
--- a/visual-search/train/sanity_check.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-# Pick the best fine-tuned checkpoint by HELD-OUT image→caption retrieval accuracy (recall@1).
-# Held-out = catalog rows NOT in the training cache. Compares base CLIP vs each ckpt/dw_clip_ft_ep*.pt.
-# Higher held-out recall@1 = better generalization (guards against the low-loss overfit trap).
-import os, io, glob, urllib.request, urllib.parse, subprocess
-os.environ.setdefault('PYTORCH_ENABLE_MPS_FALLBACK', '1')
-import torch, torch.nn.functional as F, open_clip
-from PIL import Image
-import finetune as T # reuse caption()
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-DB = os.environ.get('DW_UNIFIED_DB', 'postgresql://dw_admin@127.0.0.1:5432/dw_unified')
-N = 300
-trained = set(os.path.splitext(f)[0] for f in os.listdir(os.path.join(HERE, 'cache')))
-
-def small(u):
- if 'shopify' in u:
- pr=urllib.parse.urlparse(u); q=urllib.parse.parse_qs(pr.query); q['width']=['512']
- return urllib.parse.urlunparse(pr._replace(query=urllib.parse.urlencode(q,doseq=True)))
- return u
-
-# pull held-out rows (id, url, pattern, color, vendor, type) not already trained
-sql = ("select id,image_url,regexp_replace(pattern_name,E'[\\t\\n\\r]',' ','g'),"
- "coalesce(nullif(regexp_replace(coalesce(color_name,''),E'.*\"Name\":\\s*\"([^\"]+)\".*',E'\\1'),color_name),color_primary,''),"
- "coalesce(original_vendor_name,vendor_code,''),coalesce(product_type,'Wallcovering') "
- "from vendor_catalog where image_url like 'http%' and pattern_name is not null and pattern_name<>'' "
- "order by random() limit 1200")
-rows = [l.split('\t') for l in subprocess.check_output(['psql',DB,'-F','\t','-tA','-c',sql]).decode().splitlines()]
-heldout = [r for r in rows if len(r)>=6 and r[0] not in trained][:N]
-print(f"held-out candidates: {len(heldout)}", flush=True)
-
-imgs, caps = [], []
-for r in heldout:
- try:
- data = urllib.request.urlopen(urllib.request.Request(small(r[1]),headers={'User-Agent':'t'}),timeout=20).read()
- imgs.append(Image.open(io.BytesIO(data)).convert('RGB')); caps.append(T.caption(r[2],r[3],r[4],r[5]))
- except Exception: pass
-print(f"downloaded {len(imgs)} held-out images", flush=True)
-
-dev='mps' if torch.backends.mps.is_available() else 'cpu'
-model,_,pp = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')
-tok = open_clip.get_tokenizer('ViT-B-32'); model=model.to(dev).float().eval()
-IM = torch.stack([pp(i) for i in imgs]).to(dev)
-TX = tok(caps).to(dev)
-
-def recall1(sd=None):
- if sd: model.load_state_dict(sd, strict=False)
- else: model.load_state_dict(open_clip.create_model('ViT-B-32', pretrained='laion2b_s34b_b79k').state_dict(), strict=False)
- with torch.no_grad():
- imf=F.normalize(model.encode_image(IM),dim=-1); txf=F.normalize(model.encode_text(TX),dim=-1)
- sim=imf@txf.t(); top=sim.argmax(dim=1); correct=(top==torch.arange(len(imgs),device=dev)).float().mean().item()
- return correct*100
-
-print(f"\n{'model':<26} held-out recall@1")
-print(f"{'base laion2b':<26} {recall1(None):.1f}%")
-best=('base',recall1(None))
-for ck in sorted(glob.glob(os.path.join(HERE,'ckpt','dw_clip_ft_ep*.pt'))):
- acc=recall1(torch.load(ck,map_location='cpu')); name=os.path.basename(ck)
- print(f"{name:<26} {acc:.1f}%")
- if acc>best[1]: best=(name,acc)
-print(f"\n★ BEST: {best[0]} @ {best[1]:.1f}% held-out recall@1")
← 8fc972d auto-data-snapshot: 2026-08-17T04:29:37 (1 data files) — dat
·
back to Dw Photo Capture
·
auto-data-snapshot: 2026-08-18T03:37:22 (1 data files) — dat d0fe135 →