`scripts/status-snapshot.sh`

126 lines
#!/usr/bin/env bash
# Status-snapshot cron — runs every 2 minutes on prod, captures health
# of butlr + wallco-ai (the two pm2 processes Steve cares about most),
# appends JSONL to ~/status/butlr-wallco.jsonl, and rotates a "latest.json"
# sidecar that surfaces at /admin/status (see routes/admin.js).
#
# Triggers an alert (writes to ~/status/ALERT-<ts>.txt) on:
#   - HTTP healthcheck returns non-2xx
#   - pm2 process is not 'online'
#   - restart_time jumped by >5 since the last snapshot (flap detector)
#
# Wired via launchd: ~/Library/LaunchAgents/com.steve.butlr-status-cron.plist
# (StartInterval 120). Idempotent — safe to run by hand.

set -euo pipefail

STATUS_DIR="${HOME}/status"
mkdir -p "$STATUS_DIR"

LOG="$STATUS_DIR/butlr-wallco.jsonl"
LATEST="$STATUS_DIR/latest.json"

ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ")

# Healthchecks (curl --max-time 8)
butlr_code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 8 https://butlr.agentabrams.com/healthz || echo "000")
wallco_code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 8 https://wallco.ai/design/70n -L || echo "000")

# pm2 stats — JSON via ssh; falls back to '?' on ssh failure
pm2_blob=$(ssh -o ConnectTimeout=5 root@45.61.58.125 'pm2 jlist 2>/dev/null' 2>/dev/null || echo '[]')

# Extract per-process stats in python (jq isn't always present)
stats=$(echo "$pm2_blob" | python3 -c "
import sys, json, time, os
try:
  d = json.load(sys.stdin)
except Exception:
  print(json.dumps({'butlr': None, 'wallco-ai': None}))
  sys.exit(0)
out = {}
for n in ('butlr', 'wallco-ai'):
  proc = next((p for p in d if p['name'] == n), None)
  if not proc:
    out[n] = None
    continue
  e = proc.get('pm2_env', {})
  age_s = int(time.time() * 1000 - e.get('pm_uptime', 0)) // 1000
  out[n] = {
    'pid': proc.get('pid'),
    'uptime_s': age_s,
    'restarts': e.get('restart_time', 0),
    'status': e.get('status'),
    'memory_mb': round(proc.get('monit', {}).get('memory', 0) / 1024 / 1024, 1),
    'cpu_pct': proc.get('monit', {}).get('cpu', 0),
  }
print(json.dumps(out))
")

# Build snapshot JSON
snapshot=$(python3 -c "
import json, sys
stats = json.loads('$stats')
snap = {
  'ts': '$ts',
  'butlr': {
    'healthz_code': '$butlr_code',
    'pm2': stats.get('butlr'),
  },
  'wallco_ai': {
    'healthz_code': '$wallco_code',
    'pm2': stats.get('wallco-ai'),
  },
}
print(json.dumps(snap))
")

# Append to log + update latest
echo "$snapshot" >> "$LOG"
echo "$snapshot" > "$LATEST"

# Alert detection
alerts=()
if [[ ! "$butlr_code" =~ ^2 ]]; then alerts+=("butlr healthz=$butlr_code"); fi
if [[ ! "$wallco_code" =~ ^2 ]]; then alerts+=("wallco healthz=$wallco_code"); fi

# Flap detection: compare restart_time to prior snapshot
prior=$(tail -2 "$LOG" 2>/dev/null | head -1 || echo "")
if [[ -n "$prior" ]] && [[ "$prior" != "$snapshot" ]]; then
  flap_check=$(python3 -c "
import json
prev = json.loads('''$prior''')
curr = json.loads('''$snapshot''')
out = []
for k, label in [('butlr', 'butlr'), ('wallco_ai', 'wallco-ai')]:
  pp = (prev.get(k, {}) or {}).get('pm2') or {}
  cp = (curr.get(k, {}) or {}).get('pm2') or {}
  pr = pp.get('restarts', 0) or 0
  cr = cp.get('restarts', 0) or 0
  if cr - pr > 5:
    out.append(f'{label} flap: {pr}→{cr} (+{cr-pr})')
print('\n'.join(out))
" 2>/dev/null || echo "")
  if [[ -n "$flap_check" ]]; then alerts+=("$flap_check"); fi
fi

if (( ${#alerts[@]} > 0 )); then
  alert_file="$STATUS_DIR/ALERT-$(date -u +"%Y%m%dT%H%M%SZ").txt"
  {
    echo "Time: $ts"
    echo ""
    echo "Issues:"
    for a in "${alerts[@]}"; do echo "  - $a"; done
    echo ""
    echo "Snapshot:"
    echo "$snapshot" | python3 -m json.tool
  } > "$alert_file"
fi

# Trim log to last 10000 entries (keep ~14 days at 2-min cadence)
if [[ -f "$LOG" ]]; then
  lines=$(wc -l < "$LOG")
  if (( lines > 10000 )); then
    tail -10000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
  fi
fi