`scripts/post_crawl_pipeline.sh`

28 lines
#!/usr/bin/env bash
# Wait for the crawler bg pid to exit, then run score → enrich:classify → log final stats.
# Usage: nohup ./scripts/post_crawl_pipeline.sh <crawl_pid> > logs/post-crawl.log 2>&1 &
set -e
CRAWL_PID=${1:?usage: post_crawl_pipeline.sh <crawl_pid>}
cd "$(dirname "$0")/.."

echo "[post-crawl] watching pid $CRAWL_PID …"
while kill -0 "$CRAWL_PID" 2>/dev/null; do sleep 30; done
echo "[post-crawl] pid $CRAWL_PID exited at $(date -Iseconds)"

echo "[post-crawl] step 1/3 · score"
npm run score 2>&1 | tail -8

echo "[post-crawl] step 2/3 · enrich:classify"
npm run enrich:classify 2>&1 | tail -8

echo "[post-crawl] step 3/3 · final stats"
psql ventura_corridor -c "
SELECT
  (SELECT COUNT(*) FROM businesses WHERE on_corridor) AS total,
  (SELECT COUNT(*) FROM businesses WHERE on_corridor AND website IS NOT NULL) AS with_site,
  (SELECT COUNT(DISTINCT business_id) FROM front_page_audits WHERE error_message IS NULL OR error_message LIKE '[fallback:%') AS audited,
  (SELECT COUNT(DISTINCT business_id) FROM seo_scores) AS scored,
  (SELECT COUNT(*) FROM business_enrichment) AS enriched;
"
echo "[post-crawl] done at $(date -Iseconds)"