`visual-search/train/build_manifest.sh`

27 lines
#!/bin/bash
# Build the training manifest from the LOCAL dw_unified mirror → manifest.tsv
# Columns: id \t image_url \t pattern_name \t color \t vendor \t product_type
# One catalog image + its text = one CLIP contrastive pair. Only rows with a real
# pattern name and an http image are usable. Color is cleaned of the JSON-blob form.
set -e
cd "$(dirname "$0")"
DB="${DW_UNIFIED_DB:-postgresql://dw_admin@127.0.0.1:5432/dw_unified}"
LIMIT="${1:-100000}"   # 100k diverse pairs = a strong overnight fine-tune (many epochs) without a 245k download
psql "$DB" -F$'\t' -tA -c "
  select id,
         image_url,
         regexp_replace(pattern_name, E'[\t\n\r]', ' ', 'g'),
         -- color_name may be plain text OR a JSON blob {\"Name\":\"Beige\",...}; pull the Name
         coalesce(
           nullif(regexp_replace(coalesce(color_name,''), E'.*\"Name\":\\s*\"([^\"]+)\".*', E'\\1'), color_name),
           color_primary, ''),
         coalesce(original_vendor_name, vendor_code, ''),
         coalesce(product_type, 'Wallcovering')
  from vendor_catalog
  where image_url like 'http%'
    and pattern_name is not null and pattern_name <> ''
  order by random()
  limit $LIMIT
" > manifest.tsv
echo "manifest rows: $(wc -l < manifest.tsv)"