`disk-forecaster/forecast-disk.sh`

89 lines
#!/usr/bin/env bash
# Kamatera Disk Runway Forecaster  (Council idea #1, 2026-08-09)
# -------------------------------------------------------------
# Forecasting, not thresholding: samples du per top directory, appends to a
# rolling log, fits a linear growth rate over the trailing window, and
# extrapolates DAYS-UNTIL-FULL. Fires a George email + CNCP card when any
# tracked path's runway drops below THRESHOLD_DAYS.
#
# Born from TK-10236: prod Postgres ENOSPC-crashed on a 98%-full disk. A
# threshold canary only screams once you're already near the wall; this
# predicts the wall ~2 weeks out so the prune happens on a weekday, calmly.
#
# Designed to run on Kamatera via cron/launchd (weekly). Safe to dry-run
# anywhere: pass a ROOT to scan and it just prints the forecast (no email).
#
# Usage:
#   ROOT=/root TOP=20 THRESHOLD_DAYS=14 ./forecast-disk.sh          # real
#   ROOT="$HOME/Projects" DRY=1 ./forecast-disk.sh                  # dry-run
set -euo pipefail

ROOT="${ROOT:-/root}"
TOP="${TOP:-20}"
THRESHOLD_DAYS="${THRESHOLD_DAYS:-14}"
LOG="${LOG:-$HOME/disk-growth-log.tsv}"
DRY="${DRY:-0}"
NOW="$(date +%s)"

# --- capacity of the filesystem holding ROOT (KB) ---
avail_kb="$(df -Pk "$ROOT" | awk 'NR==2{print $4}')"
size_kb="$(df -Pk "$ROOT" | awk 'NR==2{print $2}')"

# --- sample: du -sk per top-N child dir, append to the rolling log ---
touch "$LOG"
du -sk "$ROOT"/*/ 2>/dev/null | sort -rn | head -n "$TOP" | while IFS=$'\t' read -r kb path; do
  printf '%s\t%s\t%s\n' "$NOW" "$kb" "$path" >> "$LOG"
done

# --- forecast per path from the trailing samples in the log ---
# linear rate = (latest - earliest)/(t_latest - t_earliest) KB/sec over >=2 pts
# days_until_full = free_kb / (rate_kb_per_sec * 86400), attributed to fastest grower
alerts="$(awk -v now="$NOW" -v avail="$avail_kb" -v thr="$THRESHOLD_DAYS" '
  { t=$1; kb=$2; p=$3;
    if (!(p in firstT) || t<firstT[p]) { firstT[p]=t; firstKB[p]=kb }
    if (!(p in lastT)  || t>lastT[p])  { lastT[p]=t;  lastKB[p]=kb }
  }
  END {
    for (p in lastT) {
      dt = lastT[p]-firstT[p];
      if (dt <= 0) continue;                       # need >=2 samples apart
      rate = (lastKB[p]-firstKB[p])/dt;            # KB/sec
      if (rate <= 0) continue;                     # flat/shrinking = no runway risk
      days = avail / (rate*86400);
      if (days < thr) printf "%s\t%.1f\t%.2f\n", p, days, rate*86400/1024/1024;
    }
  }' "$LOG" | sort -t$'\t' -k2 -n)"

pct_used="$(df -Pk "$ROOT" | awk 'NR==2{gsub("%","",$5); print $5}')"
echo "[$(date -u +%FT%TZ)] ROOT=$ROOT used=${pct_used}% avail=$((avail_kb/1024/1024))G threshold=${THRESHOLD_DAYS}d"

if [ -z "$alerts" ]; then
  echo "OK: no tracked path forecast to fill within ${THRESHOLD_DAYS} days (or <2 samples yet)."
  exit 0
fi

echo "RUNWAY ALERT — paths forecast to fill within ${THRESHOLD_DAYS} days:"
echo "$alerts" | awk -F'\t' '{printf "  %-40s  %5.1f days  (+%.2f GB/day)\n",$1,$2,$3}'

if [ "$DRY" = "1" ]; then
  echo "(dry-run: email/CNCP suppressed)"
  exit 0
fi

# --- fire George email + CNCP card (only on real run) ---
body="Kamatera disk runway forecast — paths projected to fill within ${THRESHOLD_DAYS} days:
$(echo "$alerts" | awk -F'\t' '{printf "  %s : %.1f days (+%.2f GB/day)\n",$1,$2,$3}')

Filesystem holding ${ROOT}: ${pct_used}% used, $((avail_kb/1024/1024))G free.
Prune before the wall — TK-10236 was an ENOSPC crash. Suggested: homesonspec/var/snapshots cache."
curl -s -X POST http://127.0.0.1:9850/api/send \
  -H 'Content-Type: application/json' \
  -u "admin:${GEORGE_BASIC_AUTH_PASS:-}" \
  -d "$(python3 -c 'import json,os,sys; print(json.dumps({"to":"steve@designerwallcoverings.com","subject":"[disk-forecaster] Kamatera runway < '"$THRESHOLD_DAYS"'d","text":sys.stdin.read()}))' <<<"$body")" \
  >/dev/null 2>&1 || echo "(warn: George send failed)"
curl -s -X POST http://127.0.0.1:3333/api/parking-lot \
  -H 'Content-Type: application/json' \
  -d "{\"project\":\"kamatera-ops\",\"title\":\"Disk runway < ${THRESHOLD_DAYS}d\",\"detail\":$(python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))' <<<"$body")}" \
  >/dev/null 2>&1 || true
echo "alerted."