282 lines
13 KiB
YAML
282 lines
13 KiB
YAML
groups:
|
|
- name: kubernetes-apps
|
|
rules:
|
|
- alert: KubePodCrashLooping
|
|
annotations:
|
|
description:
|
|
'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
|
|
}}) is in waiting state (reason: "CrashLoopBackOff").'
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping
|
|
summary: Pod is crash looping.
|
|
expr:
|
|
max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff",
|
|
job="kube-state-metrics", namespace=~".*"}[5m]) >= 1
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubePodNotReady
|
|
annotations:
|
|
description:
|
|
Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready
|
|
state for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
|
|
summary: Pod has been in a non-ready state for more than 15 minutes.
|
|
expr: |-
|
|
sum by (namespace, pod, cluster) (
|
|
max by (namespace, pod, cluster) (
|
|
kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"}
|
|
) * on (namespace, pod, cluster) group_left(owner_kind) topk by (namespace, pod, cluster) (
|
|
1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
|
|
)
|
|
) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDeploymentGenerationMismatch
|
|
annotations:
|
|
description:
|
|
Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment
|
|
}} does not match, this indicates that the Deployment has failed but has not
|
|
been rolled back.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch
|
|
summary: Deployment generation mismatch due to possible roll-back
|
|
expr: |-
|
|
kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"}
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDeploymentReplicasMismatch
|
|
annotations:
|
|
description:
|
|
Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has
|
|
not matched the expected number of replicas for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
|
|
summary: Deployment has not matched the expected number of replicas.
|
|
expr: |-
|
|
(
|
|
kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
>
|
|
kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"}
|
|
) and (
|
|
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
|
|
==
|
|
0
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDeploymentRolloutStuck
|
|
annotations:
|
|
description:
|
|
Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment
|
|
}} is not progressing for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck
|
|
summary: Deployment rollout is not progressing.
|
|
expr: |-
|
|
kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics", namespace=~".*"}
|
|
!= 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeStatefulSetReplicasMismatch
|
|
annotations:
|
|
description:
|
|
StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has
|
|
not matched the expected number of replicas for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch
|
|
summary: StatefulSet has not matched the expected number of replicas.
|
|
expr: |-
|
|
(
|
|
kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
) and (
|
|
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
|
|
==
|
|
0
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeStatefulSetGenerationMismatch
|
|
annotations:
|
|
description:
|
|
StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset
|
|
}} does not match, this indicates that the StatefulSet has failed but has
|
|
not been rolled back.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch
|
|
summary: StatefulSet generation mismatch due to possible roll-back
|
|
expr: |-
|
|
kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"}
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeStatefulSetUpdateNotRolledOut
|
|
annotations:
|
|
description:
|
|
StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update
|
|
has not been rolled out.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
|
|
summary: StatefulSet update has not been rolled out.
|
|
expr: |-
|
|
(
|
|
max by (namespace, statefulset) (
|
|
kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
|
|
unless
|
|
kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
|
|
)
|
|
*
|
|
(
|
|
kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}
|
|
)
|
|
) and (
|
|
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m])
|
|
==
|
|
0
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDaemonSetRolloutStuck
|
|
annotations:
|
|
description:
|
|
DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not
|
|
finished or progressed for at least 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck
|
|
summary: DaemonSet rollout is stuck.
|
|
expr: |-
|
|
(
|
|
(
|
|
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
) or (
|
|
kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
0
|
|
) or (
|
|
kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
) or (
|
|
kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
)
|
|
) and (
|
|
changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m])
|
|
==
|
|
0
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeContainerWaiting
|
|
annotations:
|
|
description:
|
|
pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container
|
|
{{ $labels.container}} has been in waiting state for longer than 1 hour.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting
|
|
summary: Pod container waiting longer than 1 hour
|
|
expr:
|
|
sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",
|
|
namespace=~".*"}) > 0
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDaemonSetNotScheduled
|
|
annotations:
|
|
description:
|
|
"{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
|
|
}} are not scheduled."
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled
|
|
summary: DaemonSet pods are not scheduled.
|
|
expr: |-
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
|
|
-
|
|
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeDaemonSetMisScheduled
|
|
annotations:
|
|
description:
|
|
"{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
|
|
}} are running where they are not supposed to run."
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled
|
|
summary: DaemonSet pods are misscheduled.
|
|
expr:
|
|
kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
|
|
> 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeJobNotCompleted
|
|
annotations:
|
|
description:
|
|
Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more
|
|
than {{ "43200" | humanizeDuration }} to complete.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted
|
|
summary: Job did not complete in time
|
|
expr: |-
|
|
time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"}
|
|
and
|
|
kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeJobFailed
|
|
annotations:
|
|
description:
|
|
Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
|
|
Removing failed job after investigation should clear this alert.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
|
|
summary: Job failed to complete.
|
|
expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"} > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeHpaReplicasMismatch
|
|
annotations:
|
|
description:
|
|
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}
|
|
has not matched the desired number of replicas for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch
|
|
summary: HPA has not matched desired number of replicas.
|
|
expr: |-
|
|
(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
!=
|
|
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"})
|
|
and
|
|
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
>
|
|
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"})
|
|
and
|
|
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
<
|
|
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"})
|
|
and
|
|
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: KubeHpaMaxedOut
|
|
annotations:
|
|
description:
|
|
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}
|
|
has been running at max replicas for longer than 15 minutes.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout
|
|
summary: HPA is running at max replicas
|
|
expr: |-
|
|
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
==
|
|
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|