`visual-search/embed-daemon.sh`

19 lines
#!/bin/bash
# pm2-managed embed daemon — resumable, survives crash/reboot (pm2 resurrect), auto-refreshes the
# visual-search index. Embeds in 20k chunks; when the catalog is fully embedded it idles and
# re-checks every 30 min (so newly-added products get embedded too). Replaces the bare nohup run.
cd /root/public-projects/dwphoto/visual-search || exit 1
export DW_UNIFIED_DB="$(grep ^DW_UNIFIED_DB= ../.env | cut -d= -f2-)"
while true; do
  REMAIN="$(psql "$DW_UNIFIED_DB" -tAc "select count(*) from vendor_catalog v left join image_embeddings e on e.vc_id=v.id where v.image_url is not null and v.image_url<>'' and e.vc_id is null" 2>/dev/null)"
  REMAIN="${REMAIN:-0}"
  if [ "$REMAIN" -gt 0 ]; then
    echo "$(date -u +%FT%TZ) daemon: $REMAIN remaining — embedding a 20k chunk" >> embed.log
    nice -n 15 python3 embed_catalog.py --workers 24 --limit 20000 >> embed.log 2>&1
    curl -s http://127.0.0.1:9914/reload >/dev/null 2>&1   # refresh the in-RAM search index
  else
    echo "$(date -u +%FT%TZ) daemon: catalog fully embedded — idle, re-check in 30m" >> embed.log
    sleep 1800
  fi
done