feat: add hel1 cluster

values/env-hel1.yaml

@@ -0,0 +1,24 @@
+clusterConfig:
+  manifests: https://gitlab.com/oceanbox/manifests.git
+  env: "prod"
+  distro: "talos"
+  domain: "hel1.oceanbox.io"
+  initca: ""
+  apiserver: ""
+  apiserverip: ""
+  etcd_nodes: ["10.0.1.2, 10.0.1.4, 10.0.1.5"]
+  k8s_nodes: [""]
+  cluster: "hel1"
+  ingress_nodes: ["controlplane-1, controlplane-2, controlplane-3"]
+  ingress_replica_count: 3
+  fileserver: "10.0.1.1"
+  s3:
+    hosts: []
+    patterns: []
+    cidr: []
+  nodes: []
+  ingress_whitelist:
+    - 10.0.0.0/8
+    - 172.16.0.0/12
+    - 192.168.0.0/16
+    - 100.64.0.0/12

values/system/hel1/hubble-ui-ingress.yaml

@@ -0,0 +1,50 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/auth-signin: https://$host/oauth2/start?rd=$escaped_request_uri
+    nginx.ingress.kubernetes.io/auth-url: https://$host/oauth2/auth
+    oceanbox.io/expose: internal
+  name: hubble-ui
+  namespace: kube-system
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: hubble.hel1.oceanbox.io
+      http:
+        paths:
+          - backend:
+              service:
+                name: hubble-ui
+                port:
+                  number: 80
+            path: /
+            pathType: Prefix
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-production
+    nginx.ingress.kubernetes.io/proxy-buffer-size: 8k
+    nginx.ingress.kubernetes.io/proxy-busy-buffers-size: 16k
+    oceanbox.io/expose: internal
+  name: hubble-ui-oauth2-proxy
+  namespace: kube-system
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: hubble.hel1.oceanbox.io
+      http:
+        paths:
+          - backend:
+              service:
+                name: oauth2-proxy
+                port:
+                  name: http
+            path: /oauth2
+            pathType: Prefix
+  tls:
+    - hosts:
+        - hubble.hel1.oceanbox.io
+      secretName: hubble-tls

values/system/hel1/kyverno/add-openfga-secret.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: add-openfga-secrets
+  namespace: openfga
+spec:
+  admission: true
+  background: true
+  generateExisting: true
+  mutateExistingOnPolicyUpdate: true
+  rules:
+  - name: add-db-uri
+    match:
+      any:
+      - resources:
+          kinds:
+          - Secret
+          names:
+          - prod-openfga-db-superuser
+          - staging-openfga-db-superuser
+    mutate:
+      targets:
+        - apiVersion: v1
+          kind: Secret
+          name: '{{`{{ request.object.metadata.name }}`}}'
+      patchStrategicMerge:
+        stringData:
+          postgres-password: '{{`{{ request.object.data.password | base64_decode(@) }}`}}'
+          uri: '{{`postgres://{{ request.object.data.username | base64_decode(@) }}:{{ request.object.data.password | base64_decode(@) }}@{{ request.object.metadata.labels."cnpg.io/cluster" }}-rw/app?sslmode=disable`}}'
+    skipBackgroundRequests: true
+  validationFailureAction: Audit
+{{- end }}

values/system/hel1/kyverno/sync-atlantis-secrets.yaml

@@ -0,0 +1,177 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: sync-atlantis-secrets
+spec:
+  background: true
+  generateExisting: false
+  rules:
+  - name: sync-prod-rabbitmq-secret
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: prod-rabbitmq
+        namespace: rabbitmq
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - "*-rabbitmq"
+         annotations:
+           kyverno/clone: "true"
+           kyverno/env: "prod"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+  - name: sync-dev-rabbitmq-secret
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: staging-rabbitmq
+        namespace: rabbitmq
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - "*-rabbitmq"
+         annotations:
+           kyverno/clone: "true"
+           kyverno/env: "staging"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+  - name: sync-atlantis-secret
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: staging-atlantis-env
+        namespace: staging-atlantis
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - "*-atlantis-env"
+         annotations:
+           kyverno/clone: "true"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+  - name: sync-azure-keyvault-secret
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: azure-keyvault
+        namespace: prod-atlantis
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - azure-keyvault
+         annotations:
+           kyverno/clone: "true"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+  - name: sync-dapr-api-token
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: dapr-api-token
+        namespace: prod-atlantis
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - dapr-api-token
+         annotations:
+           kyverno/clone: "true"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+  - name: sync-atlantis-db-ca
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: prod-atlantis-db-ca
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        namespace: prod-atlantis
+        name: prod-atlantis-db-ca
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - prod-atlantis-db-ca
+         annotations:
+           kyverno/clone: "true"
+  - name: sync-atlantis-db-replication
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: prod-atlantis-db-replication
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        namespace: prod-atlantis
+        name: prod-atlantis-db-replication
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - prod-atlantis-db-replication
+         annotations:
+           kyverno/clone: "true"
+{{- end }}

values/system/hel1/kyverno/sync-keyvault-secret.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    policies.clusterConfig.kyverno.io/category: Sample
+    policies.clusterConfig.kyverno.io/description: 'Access dataprotection keys from Azure Key Vault'
+  creationTimestamp: "2024-01-15T11:58:24Z"
+  name: sync-keyvault-secrets
+spec:
+  admission: true
+  background: true
+  generateExisting: true
+  rules:
+  - generate:
+      apiVersion: v1
+      clone:
+        name: azure-keyvault
+        namespace: atlantis
+      kind: Secret
+      name: azure-keyvault
+      namespace: '{{`{{request.object.metadata.name}}`}}'
+      synchronize: true
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+          names:
+          - "*-atlantis"
+    name: sync-keyvault-secrets
+    skipBackgroundRequests: true
+{{- end }}

values/system/hel1/kyverno/sync-regcred.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: sync-regcred-secret
+  annotations:
+    policies.clusterConfig.kyverno.io/title: Sync Secrets
+    policies.clusterConfig.kyverno.io/category: Sample
+    policies.clusterConfig.kyverno.io/subject: Secret
+    policies.clusterConfig.kyverno.io/description: >-
+      Secrets like registry credentials often need to exist in multiple
+      Namespaces so Pods there have access. Manually duplicating those Secrets
+      is time consuming and error prone. This policy will copy a
+      Secret called `regcred` which exists in the `default` Namespace to
+      new Namespaces when they are created. It will also push updates to
+      the copied Secrets should the source Secret be changed.            
+spec:
+  rules:
+  - name: sync-image-pull-secret
+    skipBackgroundRequests: true
+    match:
+      resources:
+        kinds:
+        - Namespace
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: regcred
+      namespace: "{{`{{request.object.metadata.name}}`}}"
+      synchronize: true
+      clone:
+        namespace: default
+        name: regcred
+{{- end }}

values/system/hel1/kyverno/sync-s3-secret.yaml

@@ -0,0 +1,34 @@
+# {{- if .Values.clusterConfig.kyverno.enabled }}
+# apiVersion: kyverno.io/v1
+# kind: ClusterPolicy
+# metadata:
+#   annotations:
+#     policies.clusterConfig.kyverno.io/description: 'This policy will sync the s3 secret in kube-system namespace across namespaces'
+#     policies.clusterConfig.kyverno.io/subject: Secret
+#     policies.clusterConfig.kyverno.io/title: Sync s3 Secrets
+#   name: sync-s3-credentials
+# spec:
+#   generateExistingOnPolicyUpdate: true
+#   background: true
+#   rules:
+#   - generate:
+#       apiVersion: v1
+#       clone:
+#         name: s3-credentials
+#         namespace: kube-system
+#       kind: Secret
+#       name: s3-credentials
+#       namespace: '{{`{{request.object.metadata.name}}`}}'
+#       synchronize: true
+#     match:
+#       resources:
+#         kinds:
+#         - Namespace
+#         names:
+#         - "velero"
+#         - "loki"
+#         - "tempo"
+#     name: sync-s3-secret
+#     skipBackgroundRequests: true
+#   validationFailureAction: audit
+# {{- end }}

values/system/hel1/kyverno/sync-slurm-token.yaml

@@ -0,0 +1,35 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: sync-slurm-token
+spec:
+  background: true
+  generateExisting: false
+  rules:
+  - name: sync-slurmrestd-token
+    skipBackgroundRequests: true
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: '{{`{{ request.object.metadata.name }}`}}'
+      namespace: '{{`{{ request.object.metadata.namespace }}`}}'
+      synchronize: true
+      clone:
+        name: slurm-access-token
+        namespace: prod-atlantis
+    match:
+     any:
+     - resources:
+         kinds:
+         - Secret
+         names:
+         - slurm-access-token
+         annotations:
+           kyverno/clone: "true"
+    exclude:
+      any:
+      - resources:
+          annotations:
+            vcluster.loft.sh/controlled-by: secret/v1/GenericImport
+{{- end }}

values/system/hel1/kyverno/whitelist-internal-ingresses.yaml

@@ -0,0 +1,73 @@
+{{- if .Values.clusterConfig.kyverno.enabled }}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: whitelist-internal-ingresses
+  annotations:
+    policies.clusterConfig.kyverno.io/title: Concatenate Ingresss
+    policies.clusterConfig.kyverno.io/category: Other
+    policies.clusterConfig.kyverno.io/severity: medium
+    policies.clusterConfig.kyverno.io/subject: Ingress
+    policies.clusterConfig.kyverno.io/description: >-
+      Ingresses with the annotation "oceanbox.io/expose=internal" should be whitelisted.
+      If no whitelist exists, add the default values, otherwise append
+      whitelist to the already existing ones
+spec:
+  mutateExistingOnPolicyUpdate: false
+  #precondition: has whitelist annotation or
+  rules:
+  - name: ensure-nginx-whitelist-exists
+    skipBackgroundRequests: true
+    match:
+      resources:
+        kinds:
+          - Ingress
+        annotations:
+          oceanbox.io/expose: internal
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          annotations:
+            +(nginx.ingress.kubernetes.io/whitelist-source-range): ""
+  - name: append-existing-whitelist
+    skipBackgroundRequests: true
+    match:
+      resources:
+        kinds:
+          - Ingress
+        annotations:
+          oceanbox.io/expose: internal
+    preconditions:
+      any:
+      - key: "{{`{{request.object.metadata.annotations.\"nginx.ingress.kubernetes.io/whitelist-source-range\"}}`}}"
+        operator: NotEquals
+        value: ""
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          annotations:
+            {{- with .Values.clusterConfig.ingress_whitelist }}
+            nginx.ingress.kubernetes.io/whitelist-source-range: "{{`{{ @ }}`}},{{ join "," . }}"
+            {{- end }}
+  - name: add-nginx-whitelist
+    skipBackgroundRequests: true
+    match:
+      resources:
+        kinds:
+          - Ingress
+        annotations:
+          oceanbox.io/expose: internal
+    preconditions:
+      any:
+      - key: "{{`{{request.object.metadata.annotations.\"nginx.ingress.kubernetes.io/whitelist-source-range\"}}`}}"
+        operator: Equals
+        value: ""
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          annotations:
+            {{- with .Values.clusterConfig.ingress_whitelist }}
+            nginx.ingress.kubernetes.io/whitelist-source-range: "{{ join "," . }}"
+            {{- end }}
+{{- end }}
+

values/system/hel1/network/allow-azure-egress.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-azure-egress
+spec:
+  egress:
+  - toFQDNs:
+    - matchName: atlantis.blob.core.windows.net
+    - matchName: atlantisvault.vault.azure.net
+    toPorts:
+    - ports:
+      - port: "443"
+        protocol: TCP
+  endpointSelector: {}
+
+{{- end }}

values/system/hel1/network/allow-ceph-egress.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-ceph-egress
+spec:
+  egress:
+  - toCIDR:
+    - 10.255.241.30/32
+    - 10.255.241.31/32
+    - 10.255.241.32/32
+    - 10.255.244.0/24
+    # toPorts:
+    # - ports:
+    #   - port: "4443"
+    #     protocol: TCP
+  endpointSelector: {}
+
+{{- end }}

values/system/hel1/network/allow-microsoft-oidc-login.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-microsoft-oidc-login
+spec:
+  endpointSelector: {}
+  egress:
+  - toFQDNs:
+    - matchName: login.microsoftonline.com
+    - matchPattern: '*.microsoftonline.com'
+{{- end }}

values/system/hel1/network/clusterpolicy-allow-api-server.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-kube-apiserver
+spec:
+  endpointSelector:
+    matchLabels: {}
+  egress:
+  - toEntities:
+    - kube-apiserver
+{{- end }}

values/system/hel1/network/clusterpolicy-allow-ekman-egress.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-ekman-egress
+spec:
+  endpointSelector: {}
+  egress:
+  - toCIDR:
+    - 10.255.241.99/32
+    - 10.255.241.100/32
+    - 185.125.160.88/32
+    - 185.125.160.89/32
+    toPorts:
+    - ports:
+      - port: "443"
+        protocol: TCP
+      - port: "4443"
+        protocol: TCP
+      - port: "30443"
+        protocol: TCP
+      - port: "30080"
+        protocol: TCP
+{{- end }}

values/system/hel1/network/clusterpolicy-allow-namespace-traffic.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-namespace-traffic
+spec:
+  description: "Allow all traffic within a namespace, allow dns, allow egress to all entities in cluster"
+  endpointSelector: {}
+  ingress:
+    - fromEndpoints:
+      - matchExpressions:
+        - key: io.kubernetes.pod.namespace
+          operator: Exists
+  egress:
+    - toEndpoints:
+      - matchExpressions:
+        - key: io.kubernetes.pod.namespace
+          operator: Exists
+    - toEndpoints:
+        - matchLabels:
+            io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+          rules:
+            dns:
+              - matchPattern: "*"
+{{- end }}

values/system/hel1/network/clusterpolicy-allow-oceanboxio.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-oceanboxio-egress
+spec:
+  endpointSelector: {}
+  egress:
+  - toFQDNs:
+    - matchName: oceanbox.io
+    - matchPattern: "*oceanbox.io"
+    - matchPattern: "*.oceanbox.io"
+
+{{- end }}

values/system/hel1/network/clusterpolicy-allow-remote-node.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: allow-remote-node-webhooks
+spec:
+  endpointSelector:
+    matchLabels: {}
+  ingress:
+  - fromEntities:
+    - kube-apiserver
+{{- end }}

values/system/hel1/vcluster-rabc.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: vcluster-create-cilium-networkpolicies
+rules:
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  verbs:
+  - '*'

values/system/manifests/oidc-cluster-admin.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+  name: oidc-cluster-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: eb17a659-4ce6-41bc-9153-d9b117c44479