← back to Watches

monitoring/prometheus/alerts.yml

163 lines

groups:
  # ============================================================================
  # Application Health Alerts
  # ============================================================================
  - name: application_health
    interval: 30s
    rules:
      - alert: ApplicationDown
        expr: up{job="omega-watches"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Omega Watches application is down"
          description: "Application has been down for more than 2 minutes on {{ $labels.instance }}"

      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value }} requests per second on {{ $labels.instance }}"

      - alert: SlowResponseTime
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Slow response times detected"
          description: "95th percentile response time is {{ $value }} seconds"

  # ============================================================================
  # System Resource Alerts
  # ============================================================================
  - name: system_resources
    interval: 30s
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage detected"
          description: "CPU usage is {{ $value }}% on {{ $labels.instance }}"

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage detected"
          description: "Memory usage is {{ $value }}% on {{ $labels.instance }}"

      - alert: DiskSpaceLow
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low disk space"
          description: "Only {{ $value }}% disk space remaining on {{ $labels.instance }}"

      - alert: HighDiskIO
        expr: rate(node_disk_io_time_seconds_total[5m]) > 0.8
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High disk I/O"
          description: "Disk I/O usage is {{ $value }} on {{ $labels.instance }}"

  # ============================================================================
  # Node.js Process Alerts
  # ============================================================================
  - name: nodejs_process
    interval: 30s
    rules:
      - alert: NodeJSMemoryLeak
        expr: nodejs_heap_size_used_bytes / nodejs_heap_size_total_bytes > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Possible Node.js memory leak"
          description: "Heap usage is {{ $value }}% on {{ $labels.instance }}"

      - alert: EventLoopLag
        expr: nodejs_eventloop_lag_seconds > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High event loop lag"
          description: "Event loop lag is {{ $value }} seconds on {{ $labels.instance }}"

  # ============================================================================
  # Redis Alerts
  # ============================================================================
  - name: redis_alerts
    interval: 30s
    rules:
      - alert: RedisDown
        expr: up{job="redis"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Redis is down"
          description: "Redis has been down for more than 2 minutes"

      - alert: RedisMemoryHigh
        expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Redis memory usage high"
          description: "Redis memory usage is {{ $value }}%"

      - alert: RedisRejectedConnections
        expr: rate(redis_rejected_connections_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Redis rejecting connections"
          description: "Redis is rejecting connections at {{ $value }} per second"

  # ============================================================================
  # Backup Alerts
  # ============================================================================
  - name: backup_alerts
    interval: 1h
    rules:
      - alert: BackupNotCompleted
        expr: time() - backup_last_success_timestamp > 86400
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "Backup has not completed in 24 hours"
          description: "Last successful backup was {{ $value }} seconds ago"

  # ============================================================================
  # SSL Certificate Alerts
  # ============================================================================
  - name: ssl_alerts
    interval: 1d
    rules:
      - alert: SSLCertificateExpiringSoon
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "SSL certificate expiring soon"
          description: "SSL certificate expires in {{ $value }} days"