From 0b73242b65f331d1f476e8bccbe518a423f098d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20J=C3=B6rg?= Date: Wed, 6 May 2026 10:37:24 +0200 Subject: [PATCH] major(ingress): Migrate hel1 to ha-proxy controller Most ingresses annotations should be work with small changes. --- charts/docs/values.yaml | 2 +- charts/makai/values.yaml | 2 +- helmfile.d/ingress-haproxy.yaml.gotmpl | 44 +++++++++ values/attic/manifests/ingress.yaml | 9 +- values/docs/values/values.yaml | 8 +- values/gatus/manifests/ingress.yaml | 12 +-- values/gitea/values/values.yaml | 11 +-- values/ingress-haproxy/env-hel1.yaml.gotmpl | 15 +++ values/ingress-haproxy/env.yaml.gotmpl | 10 ++ .../manifests/ingress-haproxy.yaml | 40 ++++++++ ...iliumNetworkPolicy-allow-host-traffic.yaml | 16 +++ ...iumNetworkPolicy-allow-hubble-traffic.yaml | 16 +++ ...etworkPolicy-allow-prometheus-metrics.yaml | 19 ++++ ...Policy-allow-world-to-ingress-haproxy.yaml | 21 ++++ .../values/ingress-haproxy.yaml.gotmpl | 98 +++++++++++++++++++ values/ingress-nginx/env-hel1.yaml.gotmpl | 2 +- values/makai/values/values.yaml | 8 +- values/system/hel1/hubble-ui-ingress.yaml | 8 +- .../kyverno/whitelist-internal-ingresses.yaml | 18 ++-- 19 files changed, 310 insertions(+), 49 deletions(-) create mode 100644 helmfile.d/ingress-haproxy.yaml.gotmpl create mode 100644 values/ingress-haproxy/env-hel1.yaml.gotmpl create mode 100644 values/ingress-haproxy/env.yaml.gotmpl create mode 100644 values/ingress-haproxy/manifests/ingress-haproxy.yaml create mode 100644 values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-host-traffic.yaml create mode 100644 values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-hubble-traffic.yaml create mode 100644 values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-prometheus-metrics.yaml create mode 100644 values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-world-to-ingress-haproxy.yaml create mode 100644 values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl diff --git a/charts/docs/values.yaml b/charts/docs/values.yaml index 4767328d..95be97b4 100644 --- a/charts/docs/values.yaml +++ b/charts/docs/values.yaml @@ -47,7 +47,7 @@ service: port: 8080 ingress: enabled: true - className: nginx + className: haproxy persistence: enabled: false size: 1G diff --git a/charts/makai/values.yaml b/charts/makai/values.yaml index 898b8e2a..177571c0 100644 --- a/charts/makai/values.yaml +++ b/charts/makai/values.yaml @@ -47,7 +47,7 @@ service: port: 8080 ingress: enabled: true - className: nginx + className: haproxy persistence: enabled: false size: 1G diff --git a/helmfile.d/ingress-haproxy.yaml.gotmpl b/helmfile.d/ingress-haproxy.yaml.gotmpl new file mode 100644 index 00000000..785d70f4 --- /dev/null +++ b/helmfile.d/ingress-haproxy.yaml.gotmpl @@ -0,0 +1,44 @@ +bases: + - ../envs/environments.yaml.gotmpl + +repositories: +- name: haproxytech + oci: true + url: 'ghcr.io/haproxytech/helm-charts' + +commonLabels: + tier: system + +releases: +- name: ingress-haproxy + namespace: ingress-haproxy + chart: haproxytech/kubernetes-ingress + version: 1.42.0 + condition: haproxy.enabled + values: + - ../values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl + - ../values/ingress-haproxy/values/ingress-haproxy-{{ .Environment.Name }}.yaml.gotmpl + postRenderer: ../bin/kustomizer + postRendererArgs: + - ../values/ingress-haproxy/kustomize/{{ .Environment.Name }} + missingFileHandler: Info +- name: manifests + namespace: ingress-haproxy + chart: manifests + condition: haproxy.enabled + missingFileHandler: Info + values: + - ../values/env.yaml + - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml + - ../values/ingress-haproxy/env.yaml.gotmpl + - ../values/ingress-haproxy/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl + hooks: + - events: [ prepare, cleanup ] + showlogs: true + command: ../bin/helmify + args: + - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}' + - '{{`{{ .Release.Chart }}`}}' + - '{{`{{ .Environment.Name }}`}}' + - ../values/ingress-haproxy/manifests + - manifests diff --git a/values/attic/manifests/ingress.yaml b/values/attic/manifests/ingress.yaml index 55b7b2b6..d949aa85 100644 --- a/values/attic/manifests/ingress.yaml +++ b/values/attic/manifests/ingress.yaml @@ -3,17 +3,14 @@ kind: Ingress metadata: annotations: cert-manager.io/cluster-issuer: letsencrypt-production - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/proxy-body-size: "0" - nginx.ingress.kubernetes.io/proxy-read-timeout: "600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "600" - nginx.ingress.kubernetes.io/ssl-redirect: "true" + haproxy.org/backend-protocol: h1 + haproxy.org/timeout-server: 600s labels: app.kubernetes.io/component: attic name: attic namespace: attic spec: - ingressClassName: nginx + ingressClassName: haproxy rules: - host: attic.srv.oceanbox.io http: diff --git a/values/docs/values/values.yaml b/values/docs/values/values.yaml index 28a57101..23c63da8 100644 --- a/values/docs/values/values.yaml +++ b/values/docs/values/values.yaml @@ -8,13 +8,11 @@ env: value: "1" ingress: enabled: true - className: "nginx" + className: "haproxy" annotations: cert-manager.io/cluster-issuer: letsencrypt-production - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/proxy-buffer-size: 128k - nginx.ingress.kubernetes.io/ssl-redirect: "true" + haproxy.org/backend-protocol: h1 + haproxy.org/cors-enable: "true" oceanbox.io/expose: internal hosts: - host: docs.oceanbox.io diff --git a/values/gatus/manifests/ingress.yaml b/values/gatus/manifests/ingress.yaml index 22f734ec..fc5af4ef 100644 --- a/values/gatus/manifests/ingress.yaml +++ b/values/gatus/manifests/ingress.yaml @@ -3,17 +3,15 @@ kind: Ingress metadata: annotations: cert-manager.io/cluster-issuer: ca-issuer - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/cors-allow-headers: Content-Type, x-gatus-cache - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/proxy-buffer-size: 128k - nginx.ingress.kubernetes.io/ssl-redirect: "true" + haproxy.org/backend-protocol: h1 + haproxy.org/cors-allow-headers: Content-Type, x-gatus-cache + haproxy.org/cors-enable: "true" labels: app.kubernetes.io/name: gatus name: gatus namespace: uptime spec: - ingressClassName: nginx + ingressClassName: haproxy rules: - host: uptime.adm.hel1.obx http: @@ -24,7 +22,7 @@ spec: port: number: 80 path: / - pathType: ImplementationSpecific + pathType: Prefix tls: - hosts: - uptime.adm.hel1.obx diff --git a/values/gitea/values/values.yaml b/values/gitea/values/values.yaml index 2c62d3d2..110df50c 100644 --- a/values/gitea/values/values.yaml +++ b/values/gitea/values/values.yaml @@ -135,15 +135,12 @@ gitea: ingress: enabled: true - className: nginx + className: haproxy annotations: cert-manager.io/cluster-issuer: letsencrypt-production - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/proxy-body-size: "0" - nginx.ingress.kubernetes.io/proxy-read-timeout: "600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "600" - nginx.ingress.kubernetes.io/whitelist-source-range: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,172.19.255.0/24,100.64.0.0/12,185.125.160.4/32,37.27.203.38/32 + haproxy.org/backend-protocol: h1 + haproxy.org/timeout-server: 600s + haproxy.org/allow-list: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,172.19.255.0/24,100.64.0.0/12,185.125.160.4/32,37.27.203.38/32 hosts: - host: git.oceanbox.io paths: diff --git a/values/ingress-haproxy/env-hel1.yaml.gotmpl b/values/ingress-haproxy/env-hel1.yaml.gotmpl new file mode 100644 index 00000000..099aabf5 --- /dev/null +++ b/values/ingress-haproxy/env-hel1.yaml.gotmpl @@ -0,0 +1,15 @@ +haproxy: + enabled: true + autosync: true + pdb: + minAvailable: 1 + resources: + controller: + cpu: "100m" + memory: "100Mi" + annotations: + load-balancer.hetzner.cloud/http-redirect-http: "true" + load-balancer.hetzner.cloud/location: hel1 + load-balancer.hetzner.cloud/name: load-balancer-1 + load-balancer.hetzner.cloud/type: lb11 + load-balancer.hetzner.cloud/use-private-ip: "true" diff --git a/values/ingress-haproxy/env.yaml.gotmpl b/values/ingress-haproxy/env.yaml.gotmpl new file mode 100644 index 00000000..2a51695f --- /dev/null +++ b/values/ingress-haproxy/env.yaml.gotmpl @@ -0,0 +1,10 @@ +haproxy: + enabled: false + autosync: true + pdb: + minAvailable: 1 + resources: + controller: + cpu: "100m" + memory: "100Mi" + annotations: [] diff --git a/values/ingress-haproxy/manifests/ingress-haproxy.yaml b/values/ingress-haproxy/manifests/ingress-haproxy.yaml new file mode 100644 index 00000000..2e73f06a --- /dev/null +++ b/values/ingress-haproxy/manifests/ingress-haproxy.yaml @@ -0,0 +1,40 @@ +{{- if .Values.clusterConfig.argo.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ingress-haproxy + namespace: argocd + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + destination: + namespace: ingress-haproxy + server: 'https://kubernetes.default.svc' + sources: + - repoURL: {{ .Values.clusterConfig.manifests }} + targetRevision: HEAD + path: helmfile.d + plugin: + name: helmfile-cmp + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterConfig.cluster }} + - name: HELMFILE_ENVIRONMENT + value: default + - name: HELMFILE_FILE_PATH + value: ingress-haproxy.yaml.gotmpl + project: sys + syncPolicy: + managedNamespaceMetadata: + labels: + component: sys + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - ServerSideApply=true + {{- if .Values.haproxy.autosync }} + automated: + prune: true + # selfHeal: false + {{- end }} +{{- end }} diff --git a/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-host-traffic.yaml b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-host-traffic.yaml new file mode 100644 index 00000000..a7063e9c --- /dev/null +++ b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-host-traffic.yaml @@ -0,0 +1,16 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-host-traffic + namespace: ingress-haproxy +spec: + egress: + - toEntities: + - kube-apiserver + - host + endpointSelector: + matchLabels: + app.kubernetes.io/name: kubernetes-ingress + app.kubernetes.io/instance: ingress-haproxy +{{- end }} diff --git a/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-hubble-traffic.yaml b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-hubble-traffic.yaml new file mode 100644 index 00000000..d4c8a7ce --- /dev/null +++ b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-hubble-traffic.yaml @@ -0,0 +1,16 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-hubble-traffic + namespace: ingress-haproxy +spec: + egress: + - toFQDNs: + - matchPattern: hubble.*.*.* + - matchPattern: hubble.*.*.*.* + endpointSelector: + matchLabels: + app.kubernetes.io/name: kubernetes-ingress + app.kubernetes.io/instance: ingress-haproxy +{{- end }} diff --git a/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-prometheus-metrics.yaml b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-prometheus-metrics.yaml new file mode 100644 index 00000000..864e02cb --- /dev/null +++ b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-prometheus-metrics.yaml @@ -0,0 +1,19 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-prometheus-metrics + namespace: ingress-haproxy +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/instance: ingress-haproxy + ingress: + - fromEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: prometheus + - toPorts: + - ports: + - port: "1024" + protocol: TCP +{{- end }} diff --git a/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-world-to-ingress-haproxy.yaml b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-world-to-ingress-haproxy.yaml new file mode 100644 index 00000000..8d2bc1e7 --- /dev/null +++ b/values/ingress-haproxy/manifests/policies/CiliumNetworkPolicy-allow-world-to-ingress-haproxy.yaml @@ -0,0 +1,21 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-world-to-ingress-haproxy + namespace: ingress-haproxy +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: kubernetes-ingress + app.kubernetes.io/instance: ingress-haproxy + ingress: + - fromEntities: + - world + - toPorts: + - ports: + - port: "80" + protocol: TCP + - port: "443" + protocol: TCP +{{- end }} diff --git a/values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl b/values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl new file mode 100644 index 00000000..653b03a6 --- /dev/null +++ b/values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl @@ -0,0 +1,98 @@ +## HAProxy Kubernetes Ingress Controller configuration +## Ref: https://www.haproxy.com/documentation/kubernetes-ingress/ +## +controller: + resources: + requests: + cpu: {{ .Values.haproxy.resources.controller.cpu }} + memory: {{ .Values.haproxy.resources.controller.memory }} + + ingressClass: haproxy + + ingressClassResource: + name: haproxy + default: true + + config: + body-size: "0" + tune.bufsize: "131072" + + tolerations: + - key: unschedulable + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: {{ .Values.clusterConfig.ingress_nodes }} + + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - ingress-haproxy + - key: app.kubernetes.io/name + operator: In + values: + - kubernetes-ingress + topologyKey: "kubernetes.io/hostname" + + replicaCount: {{ .Values.clusterConfig.ingress_replica_count }} + + PodDisruptionBudget: + enable: true + minAvailable: {{ .Values.haproxy.pdb.minAvailable }} + + service: + {{- if .Values.clusterConfig.ingress_loadbalancer }} + type: LoadBalancer + {{- if .Values.clusterConfig.ingress_nodeport }} + nodePorts: + http: 30080 + https: 30443 + {{- end }} + {{- else if .Values.clusterConfig.ingress_nodeport }} + type: NodePort + externalTrafficPolicy: Local + nodePorts: + http: 30080 + https: 30443 + {{- else }} + type: ClusterIP + {{- end }} + annotations: + {{- with .Values.haproxy.annotations }} + {{ toYaml . | nindent 8 }} + {{- end }} + + hostNetwork: {{ .Values.clusterConfig.ingress_hostnetwork }} + + hostPorts: + enable: {{ .Values.clusterConfig.ingress_hostport }} + http: 80 + https: 443 + + stats: + enabled: true + + prometheus: + enabled: true + port: 1024 + service: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "1024" + + serviceMonitor: + enabled: true diff --git a/values/ingress-nginx/env-hel1.yaml.gotmpl b/values/ingress-nginx/env-hel1.yaml.gotmpl index 7c994ae8..70184663 100644 --- a/values/ingress-nginx/env-hel1.yaml.gotmpl +++ b/values/ingress-nginx/env-hel1.yaml.gotmpl @@ -1,5 +1,5 @@ nginx: - enabled: true + enabled: false autosync: true pdb: minAvailable: 1 diff --git a/values/makai/values/values.yaml b/values/makai/values/values.yaml index e38ef9f5..b2bae275 100644 --- a/values/makai/values/values.yaml +++ b/values/makai/values/values.yaml @@ -8,13 +8,11 @@ env: value: "1" ingress: enabled: true - className: "nginx" + className: "haproxy" annotations: cert-manager.io/cluster-issuer: letsencrypt-production - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/proxy-buffer-size: 128k - nginx.ingress.kubernetes.io/ssl-redirect: "true" + haproxy.org/backend-protocol: h1 + haproxy.org/cors-enable: "true" oceanbox.io/expose: internal hosts: - host: makai.oceanbox.io diff --git a/values/system/hel1/hubble-ui-ingress.yaml b/values/system/hel1/hubble-ui-ingress.yaml index b14da61f..6753994d 100644 --- a/values/system/hel1/hubble-ui-ingress.yaml +++ b/values/system/hel1/hubble-ui-ingress.yaml @@ -2,13 +2,11 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - nginx.ingress.kubernetes.io/auth-signin: https://$host/oauth2/start?rd=$escaped_request_uri - nginx.ingress.kubernetes.io/auth-url: https://$host/oauth2/auth oceanbox.io/expose: internal name: hubble-ui namespace: kube-system spec: - ingressClassName: nginx + ingressClassName: haproxy rules: - host: hubble.hel1.oceanbox.io http: @@ -26,13 +24,11 @@ kind: Ingress metadata: annotations: cert-manager.io/cluster-issuer: letsencrypt-production - nginx.ingress.kubernetes.io/proxy-buffer-size: 8k - nginx.ingress.kubernetes.io/proxy-busy-buffers-size: 16k oceanbox.io/expose: internal name: hubble-ui-oauth2-proxy namespace: kube-system spec: - ingressClassName: nginx + ingressClassName: haproxy rules: - host: hubble.hel1.oceanbox.io http: diff --git a/values/system/hel1/kyverno/whitelist-internal-ingresses.yaml b/values/system/hel1/kyverno/whitelist-internal-ingresses.yaml index b3197e69..d6a1255f 100644 --- a/values/system/hel1/kyverno/whitelist-internal-ingresses.yaml +++ b/values/system/hel1/kyverno/whitelist-internal-ingresses.yaml @@ -14,9 +14,8 @@ metadata: whitelist to the already existing ones spec: mutateExistingOnPolicyUpdate: false - #precondition: has whitelist annotation or rules: - - name: ensure-nginx-whitelist-exists + - name: ensure-haproxy-allowlist-exists skipBackgroundRequests: true match: resources: @@ -28,8 +27,8 @@ spec: patchStrategicMerge: metadata: annotations: - +(nginx.ingress.kubernetes.io/whitelist-source-range): "" - - name: append-existing-whitelist + +(haproxy.org/allow-list): "" + - name: append-existing-haproxy-allowlist skipBackgroundRequests: true match: resources: @@ -39,7 +38,7 @@ spec: oceanbox.io/expose: internal preconditions: any: - - key: "{{`{{request.object.metadata.annotations.\"nginx.ingress.kubernetes.io/whitelist-source-range\"}}`}}" + - key: "{{`{{request.object.metadata.annotations.\"haproxy.org/allow-list\"}}`}}" operator: NotEquals value: "" mutate: @@ -47,9 +46,9 @@ spec: metadata: annotations: {{- with .Values.clusterConfig.ingress_whitelist }} - nginx.ingress.kubernetes.io/whitelist-source-range: "{{`{{ @ }}`}},{{ join "," . }}" + haproxy.org/allow-list: "{{`{{ @ }}`}},{{ join "," . }}" {{- end }} - - name: add-nginx-whitelist + - name: add-haproxy-allowlist skipBackgroundRequests: true match: resources: @@ -59,7 +58,7 @@ spec: oceanbox.io/expose: internal preconditions: any: - - key: "{{`{{request.object.metadata.annotations.\"nginx.ingress.kubernetes.io/whitelist-source-range\"}}`}}" + - key: "{{`{{request.object.metadata.annotations.\"haproxy.org/allow-list\"}}`}}" operator: Equals value: "" mutate: @@ -67,7 +66,6 @@ spec: metadata: annotations: {{- with .Values.clusterConfig.ingress_whitelist }} - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ join "," . }}" + haproxy.org/allow-list: "{{ join "," . }}" {{- end }} {{- end }} - -- 2.52.0