`overnight_yolo.sh`

27 lines
#!/usr/bin/env bash
# Overnight autonomous vendor-corporate harvest + local color enrichment loop.
# $0 — pure HTTP crawl + local Pillow color (NO paid API, NO tokens). Resumable each cycle.
# Runs unattended (keep-awake pinned). Launch: nohup bash overnight_yolo.sh &
cd "$(dirname "$0")"
LOG="staging/overnight-yolo.log"
LOCK="staging/.overnight.lock"
exec >> "$LOG" 2>&1
if [ -f "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then echo "already running"; exit 0; fi
echo $$ > "$LOCK"; trap 'rm -f "$LOCK"' EXIT
echo "=== [$(date)] overnight_yolo START (pid $$) ==="
for cycle in $(seq 1 24); do
  echo "--- cycle $cycle  [$(date '+%F %H:%M')] ---"
  # 1) harvest full corporate catalogs (resumable — completes remaining vendors)
  python3 crawl_vendor_corporate.py
  # 2) local color-ID enrichment on everything harvested ($0 Pillow)
  python3 enrich_corporate_colors.py
  # 3) summary
  V=$(ls staging/corporate/*.jsonl 2>/dev/null | wc -l | tr -d ' ')
  P=$(cat staging/corporate/*.jsonl 2>/dev/null | wc -l | tr -d ' ')
  C=$(wc -l < staging/corporate-colors.jsonl 2>/dev/null | tr -d ' ')
  echo "cycle $cycle done: $V vendors / $P products harvested / $C color-enriched"
  # if a full harvest + enrich is complete and stable, longer sleep; else keep cycling
  sleep 2400   # 40 min between cycles (retry failed vendors, catch site changes)
done
echo "=== [$(date)] overnight_yolo END ==="