`scripts/run-refresh.sh`

36 lines
#!/bin/bash
# run-refresh.sh — daily LA Socrata/ArcGIS incremental refresh with a self-rotating log.
# Keeps tmp/ingest.log bounded on a disk-tight box WITHOUT a separate scheduled job:
# each run rotates the log if it exceeds MAXBYTES (one .1 backup kept), then appends.
# Passes through any extra args to cli.js (so `run-refresh.sh --max=1` = a bounded smoke).
# $0 local — free public APIs.
set -euo pipefail
cd "$(dirname "$0")/.."            # project root

NODE=/opt/homebrew/bin/node
LOG="tmp/ingest.log"
MAXBYTES=$((5 * 1024 * 1024))     # 5 MB → rotate; max on-disk footprint ≈ 10 MB (log + .1)
mkdir -p tmp

# rotate BEFORE opening the append fd (so the size check is honest)
if [ -f "$LOG" ] && [ "$(stat -f%z "$LOG")" -gt "$MAXBYTES" ]; then
  mv -f "$LOG" "$LOG.1"
fi

# disk preflight — NEVER ingest into Postgres on a near-full disk. An incremental `all`
# writes to PG; at ENOSPC, PG crashes (the kamatera-disk-growth precedent). Fail LOUD +
# clean (skip + CNCP note) so the canary/watchdog see a healthy skip, not a PG crash.
MIN_FREE_GB="${MIN_FREE_GB:-10}"
AVAIL_GB=$(( $(df -k . | awk 'NR==2{print $4}') / 1024 / 1024 ))
if [ "$AVAIL_GB" -lt "$MIN_FREE_GB" ]; then
  MSG="[LA REFRESH SKIPPED $(date +%F)] disk ${AVAIL_GB}G free < ${MIN_FREE_GB}G floor — skipped ingest to avoid an ENOSPC Postgres crash. Free space, then the next 04:30 run resumes."
  echo "$MSG" >> "$LOG"
  curl -sS --max-time 8 "${CNCP_URL:-http://localhost:3333}/api/parking-lot" -H 'Content-Type: application/json' \
    -d "$(jq -n --arg u "file://$PWD" --arg note "$MSG" '{url:$u, note:$note}')" >/dev/null 2>&1 || true
  exit 0
fi

echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] refresh cycle start (disk ${AVAIL_GB}G free) ===" >> "$LOG"
"$NODE" src/cli.js all "$@" >> "$LOG" 2>&1
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] refresh cycle done (\$0) ===" >> "$LOG"