`visual-search/train/run_overnight.sh`

24 lines
#!/bin/bash
# Overnight DW CLIP fine-tune on this Mac (MPS). Chains: manifest → download (256px cache) → train.
# Resumable — re-running skips cached images and continues. Logs everything to logs/overnight.log.
#   nohup ./run_overnight.sh > logs/overnight.log 2>&1 &
set -e
cd "$(dirname "$0")"
export DW_UNIFIED_DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
export PYTORCH_ENABLE_MPS_FALLBACK=1
PY=./venv/bin/python
stamp(){ echo "[$(date '+%F %T')] $*"; }

stamp "STEP 1/3 — build manifest"
[ -f manifest.tsv ] || bash build_manifest.sh
stamp "manifest rows: $(wc -l < manifest.tsv)"

stamp "STEP 2/3 — download 256px image cache (resumable)"
$PY download.py
stamp "cache images: $(ls cache | wc -l)"

stamp "STEP 3/3 — fine-tune CLIP on MPS"
$PY finetune.py --epochs 5 --batch 96 --max_hours 7.0

stamp "OVERNIGHT RUN COMPLETE — weights at ckpt/dw_clip_ft.pt"