groups:
- name: node-exporter
  rules:
  - alert: NodeFilesystemSpaceFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        space left and is filling up.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
      summary: Filesystem is predicted to run out of space within the next 24 hours.
    expr: |-
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15
      and
        predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: NodeFilesystemSpaceFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        space left and is filling up fast.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
      summary: Filesystem is predicted to run out of space within the next 4 hours.
    expr: |-
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10
      and
        predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: NodeFilesystemAlmostOutOfSpace
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        space left.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
      summary: Filesystem has less than 5% space left.
    expr: |-
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 30m
    labels:
      severity: warning
  - alert: NodeFilesystemAlmostOutOfSpace
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        space left.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
      summary: Filesystem has less than 3% space left.
    expr: |-
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 30m
    labels:
      severity: critical
  - alert: NodeFilesystemFilesFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        inodes left and is filling up.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 24 hours.
    expr: |-
      (
        node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: NodeFilesystemFilesFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        inodes left and is filling up fast.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 4 hours.
    expr: |-
      (
        node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: NodeFilesystemAlmostOutOfFiles
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        inodes left.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 5% inodes left.
    expr: |-
      (
        node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: warning
  - alert: NodeFilesystemAlmostOutOfFiles
    annotations:
      description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
        }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
        inodes left.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 3% inodes left.
    expr: |-
      (
        node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
      )
    for: 1h
    labels:
      severity: critical
  - alert: NodeNetworkReceiveErrs
    annotations:
      description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
        {{ printf "%.0f" $value }} receive errors in the last two minutes.'
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
      summary: Network interface is reporting many receive errors.
    expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m])
      > 0.01
    for: 1h
    labels:
      severity: warning
  - alert: NodeNetworkTransmitErrs
    annotations:
      description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
        {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
      summary: Network interface is reporting many transmit errors.
    expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m])
      > 0.01
    for: 1h
    labels:
      severity: warning
  - alert: NodeHighNumberConntrackEntriesUsed
    annotations:
      description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
      summary: Number of conntrack are getting close to the limit.
    expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit)
      > 0.75
    labels:
      severity: warning
  - alert: NodeTextFileCollectorScrapeError
    annotations:
      description: Node Exporter text file collector on {{ $labels.instance }} failed
        to scrape.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
      summary: Node Exporter text file collector failed to scrape.
    expr: node_textfile_scrape_error{job="node-exporter"} == 1
    labels:
      severity: warning
  - alert: NodeClockSkewDetected
    annotations:
      description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s.
        Ensure NTP is configured correctly on this host.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
      summary: Clock skew detected.
    expr: |-
      (
        node_timex_offset_seconds{job="node-exporter"} > 0.05
      and
        deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
      )
      or
      (
        node_timex_offset_seconds{job="node-exporter"} < -0.05
      and
        deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
      )
    for: 10m
    labels:
      severity: warning
  - alert: NodeClockNotSynchronising
    annotations:
      description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP
        is configured on this host.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
      summary: Clock not synchronising.
    expr: |-
      min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
      and
      node_timex_maxerror_seconds{job="node-exporter"} >= 16
    for: 10m
    labels:
      severity: warning
  - alert: NodeRAIDDegraded
    annotations:
      description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is
        in degraded state due to one or more disks failures. Number of spare drives
        is insufficient to fix issue automatically.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
      summary: RAID Array is degraded.
    expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}
      - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"})
      > 0
    for: 15m
    labels:
      severity: critical
  - alert: NodeRAIDDiskFailure
    annotations:
      description: At least one device in RAID array at {{ $labels.instance }} failed.
        Array '{{ $labels.device }}' needs attention and possibly a disk swap.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
      summary: Failed device in RAID array.
    expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}
      > 0
    labels:
      severity: warning
  - alert: NodeFileDescriptorLimit
    annotations:
      description: File descriptors limit at {{ $labels.instance }} is currently at
        {{ printf "%.2f" $value }}%.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
      summary: Kernel is predicted to exhaust file descriptors limit soon.
    expr: |-
      (
        node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
      )
    for: 15m
    labels:
      severity: warning
  - alert: NodeFileDescriptorLimit
    annotations:
      description: File descriptors limit at {{ $labels.instance }} is currently at
        {{ printf "%.2f" $value }}%.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
      summary: Kernel is predicted to exhaust file descriptors limit soon.
    expr: |-
      (
        node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
      )
    for: 15m
    labels:
      severity: critical
  - alert: NodeCPUHighUsage
    annotations:
      description: |
        CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
      summary: High CPU usage.
    expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter",
      mode!="idle"}[2m]))) * 100 > 90
    for: 15m
    labels:
      severity: info
  - alert: NodeSystemSaturation
    annotations:
      description: |
        System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
        This might indicate this instance resources saturation and can cause it becoming unresponsive.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
      summary: System saturated, load per core is very high.
    expr: |-
      node_load1{job="node-exporter"}
      / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
    for: 15m
    labels:
      severity: warning
  - alert: NodeMemoryMajorPagesFaults
    annotations:
      description: |
        Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
        Please check that there is enough memory available at this instance.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
      summary: Memory major page faults are occurring at very high rate.
    expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
    for: 15m
    labels:
      severity: warning
  - alert: NodeMemoryHighUtilization
    annotations:
      description: |
        Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
      summary: Host is running out of memory.
    expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"}
      * 100) > 90
    for: 15m
    labels:
      severity: warning
  - alert: NodeDiskIOSaturation
    annotations:
      description: |
        Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
        This symptom might indicate disk saturation.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
      summary: Disk IO queue is high.
    expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      > 10
    for: 30m
    labels:
      severity: warning
  - alert: NodeSystemdServiceFailed
    annotations:
      description: Systemd service {{ $labels.name }} has entered failed state at
        {{ $labels.instance }}
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
      summary: Systemd service has entered failed state.
    expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
    for: 5m
    labels:
      severity: warning
  - alert: NodeBondingDegraded
    annotations:
      description: Bonding interface {{ $labels.master }} on {{ $labels.instance }}
        is in degraded state due to one or more slave failures.
      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
      summary: Bonding interface is degraded
    expr: (node_bonding_slaves - node_bonding_active) != 0
    for: 5m
    labels:
      severity: warning
