linux-formula/linux/meta/prometheus.yml

{%- from "linux/map.jinja" import monitoring with context %}
server:
  alert:
    SystemCpuIdleTooLow:
      {%- set cpu_idle_threshold = monitoring.cpu_idle_percentage.warn|float %}
      if: avg_over_time(cpu_usage_idle{cpu="cpu-total"}[5m]) < {{ cpu_idle_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Idle CPU usage too low on {{ $labels.host }}'
        description: 'The average idle CPU usage is too low on node {{ $labels.host }} (current value={{ $value }}, threshold={% endraw %}{{ cpu_idle_threshold}})'
    SystemDiskSpaceTooLow:
      if: 'predict_linear(disk_free[1h], 8*3600) < 0'
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Free space for {{ $labels.path }} too low on {{ $labels.host }}'
        description: 'The disk partition ({{ $labels.path }}) will be full in less than 8 hours on {{ $labels.host }}'
      {% endraw %}
    SystemDiskInodesTooLow:
      if: 'predict_linear(disk_inodes_free[1h], 8*3600) < 0'
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Free inodes for {{ $labels.path }} too low on {{ $labels.host }}'
        description: 'The disk inodes ({{ $labels.path }}) will be full in less than 8 hours on {{ $labels.host }}'
      {% endraw %}
    SystemMemoryAvailableTooLow:
      {%- set mem_avail_threshold = monitoring.free_memory_percentage.warn|float %}
      if: avg_over_time(mem_available_percent[5m]) < {{ mem_avail_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Free memory too low on {{ $labels.host }}'
        description: 'The percentage of free memory is too low on node {{ $labels.host }} (current value={{ $value }}, threshold={% endraw %}{{ mem_avail_threshold }})'
    SystemLoad5TooHigh:
      if: system_load5 / system_n_cpus > {{ monitoring.load_5.warn }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'High system load (5m) on {{ $labels.host }}'
        description: 'High system load (5m) on node {{ $labels.host }}'
      {% endraw %}
    SystemRxPacketsDroppedTooHigh:
      {%- set net_rx_dropped_threshold = monitoring.rx_packets_dropped_rate.warn %}
      if: avg_over_time(net_drop_in[1m]) > {{ net_rx_dropped_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Too many received packets dropped on {{ $labels.host }} for interface {{ $labels.interface }}'
        description: 'The average number of received packets which are dropped is too high on node {{ $labels.host }} for interface {{ $label.interface }} (current value={{ $value }}, threshold={% endraw %}{{ net_rx_dropped_threshold }})'
    SystemTxPacketsDroppedTooHigh:
      {%- set net_tx_dropped_threshold = monitoring.tx_packets_dropped_rate.warn %}
      if: avg_over_time(net_drop_out[1m]) > {{ net_tx_dropped_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Too many transmitted packets dropped on {{ $labels.host }} for interface {{ $labels.interface }}'
        description: 'The average number of transmitted packets which are dropped is too high on node {{ $labels.host }} for interface {{ $label.interface }} (current value={{ $value }}, threshold={% endraw %}{{ net_tx_dropped_threshold }})'
    SystemSwapUsed:
      {%- set swap_used_threshold = monitoring.swap.warn.strip('%')|float %}
      if: avg_over_time(swap_used_percent[1m]) > {{ swap_used_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Swap usage too high on {{ $labels.host }}'
        description: 'The average percentage of used swap is too high on node {{ $labels.host }} (current value={{ $value }}%, threshold={% endraw %}{{ swap_used_threshold }})'
    SystemSwapIn:
      {%- set swap_in_threshold = monitoring.swap_in_rate.warn %}
      if: rate(swap_in[2m]) > {{ swap_in_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Swap input throughput too high on {{ $labels.host }}'
        description: 'The rate of swap input bytes is too high on node {{ $labels.host }} (current value={{ $value }}b/s, threshold={% endraw %}{{ swap_in_threshold }})'
    SystemSwapOut:
      {%- set swap_out_threshold = monitoring.swap_out_rate.warn %}
      if: rate(swap_out[2m]) > {{ swap_out_threshold }}
      {% raw %}
      labels:
        severity: warning
        service: system
      annotations:
        summary: 'Swap output throughput too high on {{ $labels.host }}'
        description: 'The rate of swap output bytes is too high on node {{ $labels.host }} (current value={{ $value }}b/s, threshold={% endraw %}{{ swap_out_threshold }})'