fix: Add rules and remove CPU limit for x509

2025-06-20 16:40:59 +02:00
parent cd5ff63c8a
commit 73ca5987fe
10 changed files with 1189 additions and 2 deletions
@@ -0,0 +1,115 @@
+groups:
+- name: kubernetes-resources
+  rules:
+  - alert: KubeCPUOvercommit
+    annotations:
+      description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests
+        for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
+      summary: Cluster has overcommitted CPU resource requests.
+    expr: |-
+      sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
+      and
+      (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
+    for: 10m
+    labels:
+      severity: warning
+  - alert: KubeMemoryOvercommit
+    annotations:
+      description: Cluster {{ $labels.cluster }} has overcommitted memory resource
+        requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node
+        failure.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
+      summary: Cluster has overcommitted memory resource requests.
+    expr: |-
+      sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
+      and
+      (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
+    for: 10m
+    labels:
+      severity: warning
+  - alert: KubeCPUQuotaOvercommit
+    annotations:
+      description: Cluster {{ $labels.cluster }}  has overcommitted CPU resource requests
+        for Namespaces.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
+      summary: Cluster has overcommitted CPU resource requests.
+    expr: |-
+      sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
+        /
+      sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
+        > 1.5
+    for: 5m
+    labels:
+      severity: warning
+  - alert: KubeMemoryQuotaOvercommit
+    annotations:
+      description: Cluster {{ $labels.cluster }}  has overcommitted memory resource
+        requests for Namespaces.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
+      summary: Cluster has overcommitted memory resource requests.
+    expr: |-
+      sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
+        /
+      sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
+        > 1.5
+    for: 5m
+    labels:
+      severity: warning
+  - alert: KubeQuotaAlmostFull
+    annotations:
+      description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
+        }} of its {{ $labels.resource }} quota.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull
+      summary: Namespace quota is going to be full.
+    expr: |-
+      kube_resourcequota{job="kube-state-metrics", type="used"}
+        / ignoring(instance, job, type)
+      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+        > 0.9 < 1
+    for: 15m
+    labels:
+      severity: info
+  - alert: KubeQuotaFullyUsed
+    annotations:
+      description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
+        }} of its {{ $labels.resource }} quota.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused
+      summary: Namespace quota is fully used.
+    expr: |-
+      kube_resourcequota{job="kube-state-metrics", type="used"}
+        / ignoring(instance, job, type)
+      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+        == 1
+    for: 15m
+    labels:
+      severity: info
+  - alert: KubeQuotaExceeded
+    annotations:
+      description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
+        }} of its {{ $labels.resource }} quota.
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded
+      summary: Namespace quota has exceeded the limits.
+    expr: |-
+      kube_resourcequota{job="kube-state-metrics", type="used"}
+        / ignoring(instance, job, type)
+      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+        > 1
+    for: 15m
+    labels:
+      severity: warning
+  - alert: CPUThrottlingHigh
+    annotations:
+      description: '{{ $value | humanizePercentage }} throttling of CPU in namespace
+        {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod
+        }}.'
+      runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh
+      summary: Processes experience elevated CPU throttling.
+    expr: |-
+      sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace)
+        /
+      sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace)
+        > ( 25 / 100 )
+    for: 15m
+    labels:
+      severity: info