groups:
  - name: kubernetes-apps
    rules:
      - alert: KubePodCrashLooping
        annotations:
          description:
            'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is in waiting state (reason: "CrashLoopBackOff").'
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping
          summary: Pod is crash looping.
        expr:
          max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff",
          job="kube-state-metrics", namespace=~".*"}[5m]) >= 1
        for: 15m
        labels:
          severity: warning
      - alert: KubePodNotReady
        annotations:
          description:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready
            state for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
          summary: Pod has been in a non-ready state for more than 15 minutes.
        expr: |-
          sum by (namespace, pod, cluster) (
            max by (namespace, pod, cluster) (
              kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"}
            ) * on (namespace, pod, cluster) group_left(owner_kind) topk by (namespace, pod, cluster) (
              1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
            )
          ) > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeDeploymentGenerationMismatch
        annotations:
          description:
            Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment
            }} does not match, this indicates that the Deployment has failed but has not
            been rolled back.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch
          summary: Deployment generation mismatch due to possible roll-back
        expr: |-
          kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
            !=
          kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"}
        for: 15m
        labels:
          severity: warning
      - alert: KubeDeploymentReplicasMismatch
        annotations:
          description:
            Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has
            not matched the expected number of replicas for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
          summary: Deployment has not matched the expected number of replicas.
        expr: |-
          (
            kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"}
              >
            kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"}
          ) and (
            changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeDeploymentRolloutStuck
        annotations:
          description:
            Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment
            }} is not progressing for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck
          summary: Deployment rollout is not progressing.
        expr: |-
          kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics", namespace=~".*"}
          != 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetReplicasMismatch
        annotations:
          description:
            StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has
            not matched the expected number of replicas for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch
          summary: StatefulSet has not matched the expected number of replicas.
        expr: |-
          (
            kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"}
              !=
            kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~".*"}
          ) and (
            changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetGenerationMismatch
        annotations:
          description:
            StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset
            }} does not match, this indicates that the StatefulSet has failed but has
            not been rolled back.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch
          summary: StatefulSet generation mismatch due to possible roll-back
        expr: |-
          kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
            !=
          kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"}
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetUpdateNotRolledOut
        annotations:
          description:
            StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update
            has not been rolled out.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
          summary: StatefulSet update has not been rolled out.
        expr: |-
          (
            max by (namespace, statefulset) (
              kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
                unless
              kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
            )
              *
            (
              kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"}
                !=
              kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}
            )
          )  and (
            changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeDaemonSetRolloutStuck
        annotations:
          description:
            DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not
            finished or progressed for at least 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck
          summary: DaemonSet rollout is stuck.
        expr: |-
          (
            (
              kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"}
               !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
            ) or (
              kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
               !=
              0
            ) or (
              kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}
               !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
            ) or (
              kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"}
               !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
            )
          ) and (
            changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeContainerWaiting
        annotations:
          description:
            pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container
            {{ $labels.container}} has been in waiting state for longer than 1 hour.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting
          summary: Pod container waiting longer than 1 hour
        expr:
          sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",
          namespace=~".*"}) > 0
        for: 1h
        labels:
          severity: warning
      - alert: KubeDaemonSetNotScheduled
        annotations:
          description:
            "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
            }} are not scheduled."
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled
          summary: DaemonSet pods are not scheduled.
        expr: |-
          kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
            -
          kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0
        for: 10m
        labels:
          severity: warning
      - alert: KubeDaemonSetMisScheduled
        annotations:
          description:
            "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
            }} are running where they are not supposed to run."
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled
          summary: DaemonSet pods are misscheduled.
        expr:
          kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
          > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeJobNotCompleted
        annotations:
          description:
            Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more
            than {{ "43200" | humanizeDuration }} to complete.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted
          summary: Job did not complete in time
        expr: |-
          time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"}
            and
          kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200
        labels:
          severity: warning
      - alert: KubeJobFailed
        annotations:
          description:
            Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
            Removing failed job after investigation should clear this alert.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
          summary: Job failed to complete.
        expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"}  > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeHpaReplicasMismatch
        annotations:
          description:
            HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }}
            has not matched the desired number of replicas for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch
          summary: HPA has not matched desired number of replicas.
        expr: |-
          (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"}
            !=
          kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"})
            and
          (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
            >
          kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"})
            and
          (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
            <
          kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"})
            and
          changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeHpaMaxedOut
        annotations:
          description:
            HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }}
            has been running at max replicas for longer than 15 minutes.
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout
          summary: HPA is running at max replicas
        expr: |-
          kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
            ==
          kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}
        for: 15m
        labels:
          severity: warning
