groups:
  - name: kubernetes-storage
    rules:
      - alert: KubePersistentVolumeFillingUp
        annotations:
          description:
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
            {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
          summary: PersistentVolume is filling up.
        expr: |-
          (
            kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
              /
            kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
          ) < 0.03
          and
          kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1m
        labels:
          severity: critical
      - alert: KubePersistentVolumeFillingUp
        annotations:
          description:
            Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
            {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value
            | humanizePercentage }} is available.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
          summary: PersistentVolume is filling up.
        expr: |-
          (
            kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
              /
            kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
          ) < 0.15
          and
          kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
          and
          predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1h
        labels:
          severity: warning
      - alert: KubePersistentVolumeInodesFillingUp
        annotations:
          description:
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
            {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
          summary: PersistentVolumeInodes are filling up.
        expr: |-
          (
            kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
              /
            kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
          ) < 0.03
          and
          kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1m
        labels:
          severity: critical
      - alert: KubePersistentVolumeInodesFillingUp
        annotations:
          description:
            Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster
            {{ . }} {{- end }} is expected to run out of inodes within four days. Currently
            {{ $value | humanizePercentage }} of its inodes are free.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
          summary: PersistentVolumeInodes are filling up.
        expr: |-
          (
            kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
              /
            kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
          ) < 0.15
          and
          kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
          and
          predict_linear(kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on (cluster, namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1h
        labels:
          severity: warning
      - alert: KubePersistentVolumeErrors
        annotations:
          description:
            The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster
            -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
          summary: PersistentVolume is having issues with provisioning.
        expr:
          kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"}
          > 0
        for: 5m
        labels:
          severity: critical
