feat(cilium): Deploy on Ekman

helmfile.d/cilium.yaml.gotmpl

@@ -16,7 +16,7 @@ releases:
 - name: cilium
   namespace: kube-system
   chart: cilium/cilium
-  version: {{ if eq (requiredEnv "ARGOCD_ENV_CLUSTER_NAME") "hel1" }}1.18.6{{ else }}1.16.19{{ end }}
+  version: {{ if eq (requiredEnv "ARGOCD_ENV_CLUSTER_NAME") "hel1" }}1.18.6{{ else if eq (requiredEnv "ARGOCD_ENV_CLUSTER_NAME") "ekman" }}1.19.1{{ else }}1.16.19{{ end }}
   condition: cilium.enabled
   values:
   - ../values/cilium/values/cilium.yaml.gotmpl

values/cilium/env-ekman.yaml.gotmpl

@@ -0,0 +1,12 @@
+cilium:
+  enabled: true
+  envoy:
+    enabled: true
+  # kube-proxy stays running during migration; disable replacement until done.
+  # TODO: set to true after migration
+  kubeProxyReplacement: false
+  # Direct apiserver connection -- do not rely on the kubernetes service IP
+  # during migration since we are touching the CNI layer.
+  # TODO: remove k8sServiceHost / k8sServicePort (or keep pointing at apiserver localhost)
+  k8sServiceHost: 10.255.241.99
+  k8sServicePort: 6443

values/cilium/env-rossby.yaml.gotmpl

@@ -0,0 +1,10 @@
+cilium:
+  enabled: true
+  nodePort:
+    enabled: true
+  l2announcement:
+    enabled: true
+  loadbalancerPool:
+    enabled: false
+    cidr:
+     - 10.255.241.11/32

values/cilium/values/cilium-ekman.yaml.gotmpl

@@ -0,0 +1,33 @@
+# Installs Cilium without writing the CNI config file.
+# Flannel keeps serving pods on unmigrated nodes.
+# TODO: Remove after migration
+#cni:
+  #customConf: false
+
+# Routes cross-CNI traffic via the host network stack so Flannel pods
+# and Cilium pods can reach each other during the transition.
+# TODO: Remove after migration
+#bpf:
+  #hostLegacyRouting: false
+
+# NOTE: Use Geneve so Cilium's overlay is distinct from Flannel's VXLAN.
+# This is required for cross-CNI pod connectivity during migration.
+routingMode: tunnel
+tunnelProtocol: geneve
+
+# New pod CIDR exclusively for Cilium (must not overlap):
+# Flannel pod CIDR: 10.100.0.0/16  (clusterCidr in k8s config)
+# Service CIDR:     10.0.0.0/22
+# Node networks:    10.255.241.0/24, 10.255.242.0/24, 10.255.243.0/24, 10.255.244.0/24
+# TODO: after migration switch ipam.mode to kubernetes
+ipam:
+  mode: cluster-pool
+  operator:
+    clusterPoolIPv4PodCIDRList:
+      - "10.100.0.0/16"
+    clusterPoolIPv4MaskSize: 24
+
+# NOTE: Rate limit Cilium's k8s API client ekman has ~30 nodes.
+k8sClientRateLimit:
+  qps: 5
+  burst: 10

values/cilium/values/cilium.yaml.gotmpl

@@ -15,6 +15,14 @@ encryption:
   type: {{ .Values.cilium.encryption.type}}
 envoy:
   enabled: {{ .Values.cilium.envoy.enabled }}
+  securityContext:
+    capabilities:
+      keepCapNetBindService: true
+      envoy:
+        - SYS_ADMIN
+        - NET_ADMIN
+        - BPF
+        - NET_BIND_SERVICE
   prometheus:
     serviceMonitor:
       enabled: {{ .Values.cilium.envoy.enabled }}
@@ -70,15 +78,6 @@ gatewayAPI:
     nodes:
       matchLabels:
         ingresshost: "cilium"
-envoy:
-  enabled: true
-  securityContext:
-    capabilities:
-      keepCapNetBindService: true
-      envoy:
-        - NET_ADMIN
-        - BPF
-        - NET_BIND_SERVICE
 {{- end}}
 ingressController:
   enabled: {{ .Values.cilium.ingressController.enabled }}

values/env-ekman.yaml

@@ -1,7 +1,7 @@
 clusterConfig:
   manifests: https://git.oceanbox.io/platform/manifests.git
   cilium:
-    enabled: false
+    enabled: true
   env: "prod"
   distro: "nixos"
   domain: "ekman.oceanbox.io"