`scripts/loop.sh`

39 lines
#!/usr/bin/env bash
# Long-running ingest loop. Discover once, then perpetually ingest PDPs +
# download images until the queue is empty. Designed for unattended runs.
#
# Usage:
#   nohup bash scripts/loop.sh > logs/loop.log 2>&1 &
#   tail -f logs/loop.log
set -uo pipefail
cd "$(dirname "$0")/.."
mkdir -p logs

# 1) Auto-discover (only if queue is mostly empty)
PENDING=$(psql -d patternbank_archive -tA -c "SELECT COUNT(*) FROM crawl_queue WHERE kind='pdp' AND done=FALSE")
if [ "${PENDING:-0}" -lt 200 ]; then
  echo "[$(date -u +%FT%TZ)] discover --auto --max-pages=3"
  node scripts/discover.js --auto --max-pages=3 || true
fi

# 2) PDP ingest + image download loop
while true; do
  PENDING=$(psql -d patternbank_archive -tA -c "SELECT COUNT(*) FROM crawl_queue WHERE kind='pdp' AND done=FALSE")
  IMG_PENDING=$(psql -d patternbank_archive -tA -c "SELECT COUNT(*) FROM pattern_images WHERE local_path IS NULL")
  echo "[$(date -u +%FT%TZ)] queue.pending=${PENDING} img.pending=${IMG_PENDING}"
  if [ "${PENDING:-0}" -le 0 ] && [ "${IMG_PENDING:-0}" -le 0 ]; then
    echo "[$(date -u +%FT%TZ)] all done; sleeping 600s before re-checking"
    sleep 600
    # Refresh discover with one more page each time we drain
    node scripts/discover.js --auto --max-pages=1 || true
    continue
  fi
  if [ "${PENDING:-0}" -gt 0 ]; then
    node scripts/ingest-pdp.js --limit=20 --concurrency=2 || true
  fi
  if [ "${IMG_PENDING:-0}" -gt 0 ]; then
    node scripts/download-images.js --limit=40 || true
  fi
  sleep 5
done