`scripts/refresh.sh`

18 lines
#!/bin/bash
set -euo pipefail
cd "$(dirname "$0")/.."
export PATH="/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"
ts=$(date +%Y-%m-%d)
{
  echo "=== CSLB refresh $ts ==="
  bash scripts/download.sh
  python3 scripts/build.py
  python3 scripts/enrich_links.py && mv out/cslb_call_list_CA.enriched.csv out/cslb_call_list_CA.csv
  python3 scripts/split.py
  mkdir -p out/history
  cp out/cslb_call_list_CA.csv "out/history/cslb_call_list_CA_${ts}.csv"
  # keep only last 8 weekly snapshots
  ls -1t out/history/cslb_call_list_CA_*.csv 2>/dev/null | tail -n +9 | xargs -r rm -f
  echo "=== done $ts: $(wc -l < out/cslb_call_list_CA.csv) rows ==="
} >> data/refresh.log 2>&1