`scripts/run-sweep.sh`

40 lines
#!/usr/bin/env bash
# run-sweep.sh — full chunked multi-firm LA County sweep. Runs refresh-all.js across all 14 firms in
# several Browserbase sessions (chunked by hub-group to keep each session a sane length). Each chunk
# appends+dedups into data/listings.json, so partial progress survives interruption. Per-firm capture
# counts are logged each chunk — even a 0 tells us which firm needs a precise extractor.
#
# COST: ~1 Browserbase session per chunk (~$0.05-0.10 for a ~15-20min session). 6 chunks ≈ $0.30-0.60.
# Printed per chunk + a running note. Run from the project root:  bash scripts/run-sweep.sh
set -uo pipefail
cd "$(dirname "$0")/.."
export NODE_PATH="$HOME/.claude/skills/browserbase/node_modules"
export CC_FIRMS="crexi,marcusmillichap,lee-associates,colliers,cbre,jll,matthews,kidder,naicapital,cushwake,newmark,berkadia,stepp,loopnet"

# 6 hub groups covering the LA County submarket hubs.
CHUNKS=(
  "Van Nuys,Reseda,North Hollywood,Sherman Oaks,Encino,Woodland Hills,Northridge"
  "Canoga Park,Studio City,Burbank,Glendale,Pasadena,Altadena,Los Angeles"
  "Silver Lake,Echo Park,Highland Park,Hollywood,West Hollywood,Koreatown,Culver City"
  "Santa Monica,Venice,Mar Vista,Beverly Hills,Brentwood,Pacific Palisades,Malibu"
  "Inglewood,Hawthorne,Gardena,Torrance,Long Beach,San Pedro,Downey"
  "Whittier,El Monte,Pomona,Lancaster,Palmdale,Santa Clarita"
)

before=$(node -e "console.log(require('./data/listings.json').listings.length)")
echo "=== LA County firm sweep — ${#CHUNKS[@]} chunks, 14 firms. Starting listings: $before ==="
i=0
for hubs in "${CHUNKS[@]}"; do
  i=$((i+1))
  echo ""
  echo "──── CHUNK $i/${#CHUNKS[@]} · hubs: $hubs ────"
  CC_NAV="$hubs" node scripts/refresh-all.js 2>&1 | sed 's/^/  /'
  now=$(node -e "console.log(require('./data/listings.json').listings.length)" 2>/dev/null || echo "?")
  echo "  [chunk $i done] total listings now: $now"
done
after=$(node -e "console.log(require('./data/listings.json').listings.length)")
echo ""
echo "=== SWEEP COMPLETE. listings $before → $after (+$((after-before))). Cost: ~\$0.30-0.60 (${#CHUNKS[@]} Browserbase sessions). ==="
echo "Per-firm breakdown of the LAST chunk is in data/last-refresh.json."
echo "Next: npm run analyze  (re-rank into ranked.json so the viewer shows new deals)."