From 33360777c9b7a84fb93bf3b38c34e948cd941e37 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Thu, 19 Jun 2025 15:31:39 +0200 Subject: [PATCH] feat: add system app for cluster level resources --- helmfile.d/system.yaml.gotmpl | 46 +++++ values/sys/values-ekman.yaml | 168 ------------------ values/sys/values-oceanbox.yaml | 157 ---------------- values/system/ekman/kube-flannel-rbac.yaml | 42 +++++ values/system/env.yaml.gotmpl | 0 .../system/manifests/cluster-auth-rbac.yaml | 47 +++++ values/system/manifests/kube-proxy-rbac.yaml | 51 ++++++ values/system/manifests/operator-role.yaml | 12 ++ values/system/manifests/system.yaml | 34 ++++ values/system/oceanbox/empty.yaml | 0 10 files changed, 232 insertions(+), 325 deletions(-) create mode 100644 helmfile.d/system.yaml.gotmpl delete mode 100644 values/sys/values-ekman.yaml delete mode 100644 values/sys/values-oceanbox.yaml create mode 100644 values/system/ekman/kube-flannel-rbac.yaml create mode 100644 values/system/env.yaml.gotmpl create mode 100644 values/system/manifests/cluster-auth-rbac.yaml create mode 100644 values/system/manifests/kube-proxy-rbac.yaml create mode 100644 values/system/manifests/operator-role.yaml create mode 100644 values/system/manifests/system.yaml create mode 100644 values/system/oceanbox/empty.yaml diff --git a/helmfile.d/system.yaml.gotmpl b/helmfile.d/system.yaml.gotmpl new file mode 100644 index 00000000..38dc9f77 --- /dev/null +++ b/helmfile.d/system.yaml.gotmpl @@ -0,0 +1,46 @@ +bases: + - ../envs/environments.yaml.gotmpl + +commonLabels: + tier: system + +releases: +- name: common-system-manifests + namespace: kube-system + chart: _common-system-manifests + missingFileHandler: Info + values: + - ../values/env.yaml + - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml + - ../values/system/env.yaml + - ../values/system/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl + hooks: + - events: [ prepare, cleanup ] + showlogs: true + command: ../bin/helmify + args: + - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}' + - '{{`{{ .Release.Chart }}`}}' + - '{{`{{ .Environment.Name }}`}}' + - ../values/system/manifests + - _common-system-manifests +- name: system-manifests + namespace: kube-system + chart: _system-manifests + missingFileHandler: Info + values: + - ../values/env.yaml + - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml + - ../values/system/env.yaml + - ../values/system/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl + hooks: + - events: [ prepare, cleanup ] + showlogs: true + command: ../bin/helmify + args: + - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}' + - '{{`{{ .Release.Chart }}`}}' + - '{{`{{ .Environment.Name }}`}}' + - ../values/system/{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }} + - _system-manifests + diff --git a/values/sys/values-ekman.yaml b/values/sys/values-ekman.yaml deleted file mode 100644 index cac0b848..00000000 --- a/values/sys/values-ekman.yaml +++ /dev/null @@ -1,168 +0,0 @@ -cluster_config: - env: "prod" - domain: "ekman.oceanbox.io" - initca: "/var/lib/kubernetes/secrets" - apiserver: "frontend" - apiserverip: "10.255.241.99" - etcd_nodes: [ "10.255.241.80, 10.255.241.90, 10.255.241.99" ] - k8s_nodes: [ "10.255.241.80, 10.255.241.90, 10.255.241.99, 10.255.241.100, 10.255.241.101, 10.255.241.102, 10.255.241.103, 10.255.241.104, 10.255.241.105, 10.255.241.106, 10.255.241.107, 10.255.241.108, 10.255.241.109, 10.255.241.110, 10.255.241.111, 10.255.241.112, 10.255.241.113, 10.255.241.114, 10.255.241.116, 10.255.241.121, 10.255.241.122, 10.255.241.123, 10.255.241.124, 10.255.241.125, 10.255.241.126, 10.255.241.127, 10.255.241.128" ] - cluster: "ekman" - ingress_nodes: ["ekman, frontend" ] - ingress_replica_count: 2 - fileserver: "10.255.241.90" - acme_email: "acme@oceanbox.io" - oidc: - - name: serit-oidc - provider: azuread - tenant: "95e5d757-4fb3-4113-a93c-c41393be61cf" - secret_ref: - name: serit-oidc - group_id: "dd2aa2d6-269d-48fe-90cc-04fd5c08bd29" - external_access: - enabled: false - - name: oceanbox-oidc - provider: azuread - tenant: "3f737008-e9a0-4485-9d27-40329d288089" - secret_ref: - name: oceanbox-oidc - group_id: "eb17a659-4ce6-41bc-9153-d9b117c44479" - nodes: - - name: frontend - taints: [] - labels: - - "node-role.kubernetes.io=control-plane" - - name: ekman - taints: [] - labels: - - "node-role.kubernetes.io=control-plane" - - name: nfs1 - taints: - - "workload=data:NoSchedule" - labels: - - "node-role.kubernetes.io=control-plane" - - "nfs=data" - - name: fs2 - taints: - - "workload=data:NoSchedule" - labels: - - "node-role.kubernetes.io=control-plane" - - "nfs=data" - - name: c0-1 - taints: - - "workload=compute:NoSchedule" - - name: c0-2 - taints: - - "workload=compute:NoSchedule" - - name: c0-3 - taints: - - "workload=compute:NoSchedule" - - name: c0-4 - taints: - - "workload=compute:NoSchedule" - - name: c0-5 - taints: - - "workload=compute:NoSchedule" - - name: c0-6 - taints: - - "workload=compute:NoSchedule" - - name: c0-7 - taints: - - "workload=compute:NoSchedule" - - name: c0-8 - taints: - - "workload=compute:NoSchedule" - - name: c0-9 - taints: - - "workload=compute:NoSchedule" - - name: c0-10 - taints: - - "workload=compute:NoSchedule" - - name: c0-11 - taints: - - "workload=compute:NoSchedule" - - name: c0-12 - taints: - - "workload=compute:NoSchedule" - - name: c0-13 - taints: - - "workload=compute:NoSchedule" - - name: c0-14 - taints: - - "workload=compute:NoSchedule" - - name: c0-15 - taints: - - "workload=compute:NoSchedule" - - name: c0-16 - taints: - - "workload=compute:NoSchedule" - - name: c1-1 - taints: - - "workload=compute:NoSchedule" - - name: c1-2 - taints: - - "workload=compute:NoSchedule" - - name: c1-3 - taints: - - "workload=compute:NoSchedule" - - name: c1-4 - taints: - - "workload=compute:NoSchedule" - - name: c1-5 - taints: - - "workload=compute:NoSchedule" - - name: c1-6 - taints: - - "workload=compute:NoSchedule" - - name: c1-7 - taints: - - "workload=compute:NoSchedule" - - name: c1-8 - taints: - - "workload=compute:NoSchedule" -argocd: - adminLogin: false - additional_rbac_settings: - - g, "eb17a659-4ce6-41bc-9153-d9b117c44479", role:org-admin -linkerd: - trustAnchorPEM: | - -----BEGIN CERTIFICATE----- - MIIBtDCCAVqgAwIBAgIQRlhbOLj9zw+QTGHqbOBaozAKBggqhkjOPQQDAjAlMSMw - IQYDVQQDExpyb290LmxpbmtlcmQuY2x1c3Rlci5sb2NhbDAeFw0yMTA0MDkxNDAy - NTFaFw0zMTA0MDcxNDAyNTFaMCUxIzAhBgNVBAMTGnJvb3QubGlua2VyZC5jbHVz - dGVyLmxvY2FsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEljOLtSPSi6XIEdFP - VCGa4BKoQ0X5dBSZvHRLt/IzHRzAbIVIjgjvyRQc7EQlRKvZ8P9um/WG1ypyyA2l - C9MWz6NsMGowDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYD - VR0OBBYEFHz4UuVKCNX8/hsZCcdTlmWnSCGXMCUGA1UdEQQeMByCGnJvb3QubGlu - a2VyZC5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGAiz3yNhboVdze1 - sNFcFL2GF5WwW9z53u03UkPkiuBTAiEA4ZHWZJVGV5VAQArL5v32HeH/IjC1ssGl - 7Y8D0rQqkis= - -----END CERTIFICATE----- - webhookPEM: | - -----BEGIN CERTIFICATE----- - MIIBlDCCATqgAwIBAgIRAP9aY0pRwkDnXqi3FwKmfZowCgYIKoZIzj0EAwIwKDEm - MCQGA1UEAxMdd2ViaG9vay5saW5rZXJkLmNsdXN0ZXIubG9jYWwwHhcNMjIxMDI3 - MDUxNTE0WhcNMjQxMDI1MDkxNTE0WjAoMSYwJAYDVQQDEx13ZWJob29rLmxpbmtl - cmQuY2x1c3Rlci5sb2NhbDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABIGSt6Th - 62wgjM5dRbZLa9YwPQAm/T2QnTzzrAUm+GeqvKfBhpPMGX6+91/x20X0uV26LvKz - YV1wVMs7tuPZioijRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/ - AgEBMB0GA1UdDgQWBBQWV6+eqRWOPyLWz9s0HT96MOr01zAKBggqhkjOPQQDAgNI - ADBFAiBTBFuIJUBEI5T2unrnFhM+Bj0rZFfuxQqEwD6+z2YRzwIhAOINkH5u7Z8M - zIVl06Biq2N+MO4TJ+CSS1C1w/22CDru - -----END CERTIFICATE----- - multicluster: - enabled: false -prometheus: - version: 39.6.0 - snitchUrl: "https://nosnch.in/bceb803932" -nfs_provisioner: - version: 4.0.17 - extraMountOpts: - - soft -cert_manager: - version: 1.9.1 -gitlab_runner: - enabled: false -velero: - enabled: false -kyverno: - enabled: true diff --git a/values/sys/values-oceanbox.yaml b/values/sys/values-oceanbox.yaml deleted file mode 100644 index 8010aa17..00000000 --- a/values/sys/values-oceanbox.yaml +++ /dev/null @@ -1,157 +0,0 @@ -cluster_config: - env: "prod" - distro: "talos" - domain: "adm.oceanbox.io" - initca: "" - apiserver: "" - apiserverip: "" - etcd_nodes: [ "10.255.241.201, 10.255.241.202, 10.255.241.203" ] - k8s_nodes: [ "" ] - cluster: "oceanbox" - ingress_nodes: ["oceanbox-controlplane-1, oceanbox-controlplane-2, oceanbox-controlplane-3" ] - ingress_replica_count: 3 - fileserver: "10.255.241.210" - acme_email: "acme@oceanbox.io" - oidc: - - name: serit-oidc - provider: azuread - tenant: "95e5d757-4fb3-4113-a93c-c41393be61cf" - secret_ref: - name: serit-oidc - group_id: "dd2aa2d6-269d-48fe-90cc-04fd5c08bd29" - external_access: - enabled: false - - name: oceanbox-oidc - provider: azuread - tenant: "3f737008-e9a0-4485-9d27-40329d288089" - secret_ref: - name: oceanbox-oidc - group_id: "eb17a659-4ce6-41bc-9153-d9b117c44479" - nodes: [] - ingress_whitelist_ips: - #itp internal - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - - 172.19.255.0/24 -argocd: - adminLogin: false - version: 7.5.2 - additional_rbac_settings: - - g, "eb17a659-4ce6-41bc-9153-d9b117c44479", role:org-admin - resources: - controller: - memory: 2000Mi - repoServer: - cmp: - enabled: true - name: "kustomize-helm-with-rewrite" - image: "registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest" - helmTokenSecret: oceanbox-helm - imagePullSecret: - - name: gitlab-pull-secret - initContainers: - - command: - - /bin/sh - - /plugin/init-helm-repos.sh - image: registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest - imagePullPolicy: Always - name: init-helm-repos - resources: {} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 999 - seccompProfile: - type: RuntimeDefault - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - env: - - name: OCEANBOX_HELM_ACCESS_TOKEN - valueFrom: - secretKeyRef: - key: token - name: oceanbox-helm - optional: false -linkerd: - enabled: false -prometheus: - snitchUrl: "https://nosnch.in/136c1b564f" - pagerdutyRoutingKey: a5cff1fc46414d0bc02851e4af159ee7 - certRenewCronEnabled: false - fullname: prom - enableFeatures: - - otlp-write-reciever - #- remote-write-reciever - grafana: - persistence: true - thanos: - enabled: true - coredns: - targetPort: 9153 - scheduler: - targetPort: 10259 - kubelet: - enabled: true - https: true -nfs_provisioner: - extraMountOpts: - - soft -gitlab_runner: - enabled: false -kyverno: - enabled: true -cilium: - enabled: true - kubeProxyReplacement: true - upgradeCompatability: 1.15 - nodePort: - enabled: true - l2announcement: - enabled: true - policyAuditMode: false - encryption: - type: wireguard - ingressController: - enabled: false - defaultClass: false - loadbalancerMode: shared - loadbalancerPool: - enabled: true - cidr: - - 10.255.241.11/32 - - 10.255.241.12/32 - - 10.255.241.13/32 - - 10.255.241.14/32 - - 10.255.241.15/32 -velero: - enabled: true - # Opt-in or opt-out pvc backup - # https://velero.io/docs/main/file-system-backup/#to-back-up - backupAllVolumes: false - credentials: - secretName: "velero-s3" - s3: - region: us-east-1 - url: "http://10.255.241.30:30080" - insecureSkipTLSVerify: true - bsl: default - bucket: velero - kubeletRootDir: "/var/lib/kubelet/pods" - resources: - velero: - request: - cpu: 20m - memory: 1Gi - limit: - memory: 2Gi - nodeAgent: - request: - cpu: 20m - memory: 1Gi - limit: - memory: 2Gi diff --git a/values/system/ekman/kube-flannel-rbac.yaml b/values/system/ekman/kube-flannel-rbac.yaml new file mode 100644 index 00000000..b14b53a8 --- /dev/null +++ b/values/system/ekman/kube-flannel-rbac.yaml @@ -0,0 +1,42 @@ +# Create the clusterrole and clusterrolebinding: +# $ kubectl create -f kube-flannel-rbac.yml +# Create the pod using the same namespace used by the flannel serviceaccount: +# $ kubectl create --namespace kube-system -f kube-flannel-legacy.yml +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel-client +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel-client +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel-client +subjects: + - kind: User + name: flannel-client + apiGroup: rbac.authorization.k8s.io diff --git a/values/system/env.yaml.gotmpl b/values/system/env.yaml.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/values/system/manifests/cluster-auth-rbac.yaml b/values/system/manifests/cluster-auth-rbac.yaml new file mode 100644 index 00000000..ee655308 --- /dev/null +++ b/values/system/manifests/cluster-auth-rbac.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cluster-admin + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-admin +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + namespace: kube-system + name: cluster-admin +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: 'system:masters' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + namespace: kube-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubernetes diff --git a/values/system/manifests/kube-proxy-rbac.yaml b/values/system/manifests/kube-proxy-rbac.yaml new file mode 100644 index 00000000..0b53b301 --- /dev/null +++ b/values/system/manifests/kube-proxy-rbac.yaml @@ -0,0 +1,51 @@ +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-proxy +subjects: + - kind: User + name: kube-proxy + apiGroup: rbac.authorization.k8s.io + - kind: ServiceAccount + name: kube-proxy + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-proxy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-proxy +rules: + - apiGroups: + - "" + resources: + - endpoints + - events + - services + - nodes + verbs: ["get", "watch", "list"] + - nonResourceURLs: ["*"] + verbs: ["get", "watch", "list"] + + - apiGroups: + - "" + - "events.k8s.io" + resources: + - events + verbs: ["*"] + + - nonResourceURLs: ["*"] + verbs: ["*"] + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + + diff --git a/values/system/manifests/operator-role.yaml b/values/system/manifests/operator-role.yaml new file mode 100644 index 00000000..2e630e77 --- /dev/null +++ b/values/system/manifests/operator-role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: az-kubernetes-operators-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: dd2aa2d6-269d-48fe-90cc-04fd5c08bd29 diff --git a/values/system/manifests/system.yaml b/values/system/manifests/system.yaml new file mode 100644 index 00000000..b8abc68c --- /dev/null +++ b/values/system/manifests/system.yaml @@ -0,0 +1,34 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: system + namespace: argocd +spec: + destination: + namespace: kube-system + server: 'https://kubernetes.default.svc' + sources: + - repoURL: {{ .Values.clusterConfig.manifests }} + targetRevision: HEAD + path: helmfile.d + plugin: + name: helmfile-cmp + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterConfig.cluster }} + - name: HELMFILE_ENVIRONMENT + value: default + - name: HELMFILE_FILE_PATH + value: system.yaml.gotmpl + project: sys + syncPolicy: + managedNamespaceMetadata: + labels: + component: sys + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - ServerSideApply=true + automated: + prune: true + # selfHeal: false diff --git a/values/system/oceanbox/empty.yaml b/values/system/oceanbox/empty.yaml new file mode 100644 index 00000000..e69de29b