feat: add system app for cluster level resources

helmfile.d/system.yaml.gotmpl

@@ -0,0 +1,46 @@
+bases:
+ - ../envs/environments.yaml.gotmpl
+
+commonLabels:
+  tier: system
+
+releases:
+- name: common-system-manifests
+  namespace: kube-system
+  chart: _common-system-manifests
+  missingFileHandler: Info
+  values:
+  - ../values/env.yaml
+  - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
+  - ../values/system/env.yaml
+  - ../values/system/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
+  hooks:
+  - events: [ prepare, cleanup ]
+    showlogs: true
+    command: ../bin/helmify
+    args:
+    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
+    - '{{`{{ .Release.Chart }}`}}'
+    - '{{`{{ .Environment.Name }}`}}'
+    - ../values/system/manifests
+    - _common-system-manifests
+- name: system-manifests
+  namespace: kube-system
+  chart: _system-manifests
+  missingFileHandler: Info
+  values:
+  - ../values/env.yaml
+  - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
+  - ../values/system/env.yaml
+  - ../values/system/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
+  hooks:
+  - events: [ prepare, cleanup ]
+    showlogs: true
+    command: ../bin/helmify
+    args:
+    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
+    - '{{`{{ .Release.Chart }}`}}'
+    - '{{`{{ .Environment.Name }}`}}'
+    - ../values/system/{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}
+    - _system-manifests
+

values/sys/values-ekman.yaml

@@ -1,168 +0,0 @@
-cluster_config:
-  env: "prod"
-  domain: "ekman.oceanbox.io"
-  initca: "/var/lib/kubernetes/secrets"
-  apiserver: "frontend"
-  apiserverip: "10.255.241.99"
-  etcd_nodes: [ "10.255.241.80, 10.255.241.90, 10.255.241.99" ]
-  k8s_nodes: [ "10.255.241.80, 10.255.241.90, 10.255.241.99, 10.255.241.100, 10.255.241.101, 10.255.241.102, 10.255.241.103, 10.255.241.104, 10.255.241.105, 10.255.241.106, 10.255.241.107, 10.255.241.108, 10.255.241.109, 10.255.241.110, 10.255.241.111, 10.255.241.112, 10.255.241.113, 10.255.241.114, 10.255.241.116, 10.255.241.121, 10.255.241.122, 10.255.241.123, 10.255.241.124, 10.255.241.125, 10.255.241.126, 10.255.241.127, 10.255.241.128" ]
-  cluster: "ekman"
-  ingress_nodes: ["ekman, frontend" ]
-  ingress_replica_count: 2
-  fileserver: "10.255.241.90"
-  acme_email: "acme@oceanbox.io"
-  oidc:
-  - name: serit-oidc
-    provider: azuread
-    tenant: "95e5d757-4fb3-4113-a93c-c41393be61cf"
-    secret_ref:
-      name: serit-oidc
-    group_id: "dd2aa2d6-269d-48fe-90cc-04fd5c08bd29"
-    external_access:
-      enabled: false
-  - name: oceanbox-oidc
-    provider: azuread
-    tenant: "3f737008-e9a0-4485-9d27-40329d288089"
-    secret_ref:
-      name: oceanbox-oidc
-    group_id: "eb17a659-4ce6-41bc-9153-d9b117c44479"
-  nodes:
-    - name: frontend
-      taints: []
-      labels:
-        - "node-role.kubernetes.io=control-plane"
-    - name: ekman
-      taints: []
-      labels:
-        - "node-role.kubernetes.io=control-plane"
-    - name: nfs1
-      taints:
-        - "workload=data:NoSchedule"
-      labels:
-        - "node-role.kubernetes.io=control-plane"
-        - "nfs=data"
-    - name: fs2
-      taints:
-        - "workload=data:NoSchedule"
-      labels:
-        - "node-role.kubernetes.io=control-plane"
-        - "nfs=data"
-    - name: c0-1
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-2
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-3
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-4
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-5
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-6
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-7
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-8
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-9
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-10
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-11
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-12
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-13
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-14
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-15
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c0-16
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-1
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-2
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-3
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-4
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-5
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-6
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-7
-      taints:
-        - "workload=compute:NoSchedule"
-    - name: c1-8
-      taints:
-        - "workload=compute:NoSchedule"
-argocd:
-  adminLogin: false
-  additional_rbac_settings:
-    - g, "eb17a659-4ce6-41bc-9153-d9b117c44479", role:org-admin
-linkerd:
-  trustAnchorPEM: |
-    -----BEGIN CERTIFICATE-----
-    MIIBtDCCAVqgAwIBAgIQRlhbOLj9zw+QTGHqbOBaozAKBggqhkjOPQQDAjAlMSMw
-    IQYDVQQDExpyb290LmxpbmtlcmQuY2x1c3Rlci5sb2NhbDAeFw0yMTA0MDkxNDAy
-    NTFaFw0zMTA0MDcxNDAyNTFaMCUxIzAhBgNVBAMTGnJvb3QubGlua2VyZC5jbHVz
-    dGVyLmxvY2FsMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEljOLtSPSi6XIEdFP
-    VCGa4BKoQ0X5dBSZvHRLt/IzHRzAbIVIjgjvyRQc7EQlRKvZ8P9um/WG1ypyyA2l
-    C9MWz6NsMGowDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYD
-    VR0OBBYEFHz4UuVKCNX8/hsZCcdTlmWnSCGXMCUGA1UdEQQeMByCGnJvb3QubGlu
-    a2VyZC5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGAiz3yNhboVdze1
-    sNFcFL2GF5WwW9z53u03UkPkiuBTAiEA4ZHWZJVGV5VAQArL5v32HeH/IjC1ssGl
-    7Y8D0rQqkis=
-    -----END CERTIFICATE-----
-  webhookPEM: |
-    -----BEGIN CERTIFICATE-----
-    MIIBlDCCATqgAwIBAgIRAP9aY0pRwkDnXqi3FwKmfZowCgYIKoZIzj0EAwIwKDEm
-    MCQGA1UEAxMdd2ViaG9vay5saW5rZXJkLmNsdXN0ZXIubG9jYWwwHhcNMjIxMDI3
-    MDUxNTE0WhcNMjQxMDI1MDkxNTE0WjAoMSYwJAYDVQQDEx13ZWJob29rLmxpbmtl
-    cmQuY2x1c3Rlci5sb2NhbDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABIGSt6Th
-    62wgjM5dRbZLa9YwPQAm/T2QnTzzrAUm+GeqvKfBhpPMGX6+91/x20X0uV26LvKz
-    YV1wVMs7tuPZioijRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/
-    AgEBMB0GA1UdDgQWBBQWV6+eqRWOPyLWz9s0HT96MOr01zAKBggqhkjOPQQDAgNI
-    ADBFAiBTBFuIJUBEI5T2unrnFhM+Bj0rZFfuxQqEwD6+z2YRzwIhAOINkH5u7Z8M
-    zIVl06Biq2N+MO4TJ+CSS1C1w/22CDru
-    -----END CERTIFICATE-----
-  multicluster:
-    enabled: false
-prometheus:
-  version: 39.6.0
-  snitchUrl: "https://nosnch.in/bceb803932"
-nfs_provisioner:
-  version: 4.0.17
-  extraMountOpts:
-    - soft
-cert_manager:
-  version: 1.9.1
-gitlab_runner:
-  enabled: false
-velero:
-  enabled: false
-kyverno:
-  enabled: true

values/sys/values-oceanbox.yaml

@@ -1,157 +0,0 @@
-cluster_config:
-  env: "prod"
-  distro: "talos"
-  domain: "adm.oceanbox.io"
-  initca: ""
-  apiserver: ""
-  apiserverip: ""
-  etcd_nodes: [ "10.255.241.201, 10.255.241.202, 10.255.241.203" ]
-  k8s_nodes: [ "" ]
-  cluster: "oceanbox"
-  ingress_nodes: ["oceanbox-controlplane-1, oceanbox-controlplane-2, oceanbox-controlplane-3" ]
-  ingress_replica_count: 3
-  fileserver: "10.255.241.210"
-  acme_email: "acme@oceanbox.io"
-  oidc:
-  - name: serit-oidc
-    provider: azuread
-    tenant: "95e5d757-4fb3-4113-a93c-c41393be61cf"
-    secret_ref:
-      name: serit-oidc
-    group_id: "dd2aa2d6-269d-48fe-90cc-04fd5c08bd29"
-    external_access:
-      enabled: false
-  - name: oceanbox-oidc
-    provider: azuread
-    tenant: "3f737008-e9a0-4485-9d27-40329d288089"
-    secret_ref:
-      name: oceanbox-oidc
-    group_id: "eb17a659-4ce6-41bc-9153-d9b117c44479"
-  nodes: []
-  ingress_whitelist_ips:
-    #itp internal
-    - 10.0.0.0/8
-    - 172.16.0.0/12
-    - 192.168.0.0/16
-    - 172.19.255.0/24
-argocd:
-  adminLogin: false
-  version: 7.5.2
-  additional_rbac_settings:
-    - g, "eb17a659-4ce6-41bc-9153-d9b117c44479", role:org-admin
-  resources:
-    controller:
-      memory: 2000Mi
-  repoServer:
-    cmp:
-      enabled: true
-      name: "kustomize-helm-with-rewrite"
-      image: "registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest"
-      helmTokenSecret: oceanbox-helm
-      imagePullSecret:
-        - name: gitlab-pull-secret
-      initContainers:
-      - command:
-        - /bin/sh
-        - /plugin/init-helm-repos.sh
-        image: registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest
-        imagePullPolicy: Always
-        name: init-helm-repos
-        resources: {}
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          readOnlyRootFilesystem: true
-          runAsNonRoot: true
-          runAsUser: 999
-          seccompProfile:
-            type: RuntimeDefault
-        terminationMessagePath: /dev/termination-log
-        terminationMessagePolicy: File
-        env:
-        - name: OCEANBOX_HELM_ACCESS_TOKEN
-          valueFrom:
-            secretKeyRef:
-              key: token
-              name: oceanbox-helm
-              optional: false
-linkerd:
-  enabled: false
-prometheus:
-  snitchUrl: "https://nosnch.in/136c1b564f"
-  pagerdutyRoutingKey: a5cff1fc46414d0bc02851e4af159ee7
-  certRenewCronEnabled: false
-  fullname: prom
-  enableFeatures:
-    - otlp-write-reciever
-    #- remote-write-reciever
-  grafana:
-    persistence: true
-  thanos:
-    enabled: true
-  coredns:
-    targetPort: 9153
-  scheduler:
-    targetPort: 10259
-  kubelet:
-    enabled: true
-    https: true
-nfs_provisioner:
-  extraMountOpts:
-    - soft
-gitlab_runner:
-  enabled: false
-kyverno:
-  enabled: true
-cilium:
-  enabled: true
-  kubeProxyReplacement: true
-  upgradeCompatability: 1.15
-  nodePort:
-    enabled: true
-  l2announcement:
-    enabled: true
-  policyAuditMode: false
-  encryption:
-    type: wireguard
-  ingressController:
-    enabled: false
-    defaultClass: false
-    loadbalancerMode: shared
-  loadbalancerPool:
-    enabled: true
-    cidr:
-     - 10.255.241.11/32
-     - 10.255.241.12/32
-     - 10.255.241.13/32
-     - 10.255.241.14/32
-     - 10.255.241.15/32
-velero:
-  enabled: true
-  # Opt-in or opt-out pvc backup
-  # https://velero.io/docs/main/file-system-backup/#to-back-up
-  backupAllVolumes: false
-  credentials:
-    secretName: "velero-s3"
-  s3:
-    region: us-east-1
-    url: "http://10.255.241.30:30080"
-    insecureSkipTLSVerify: true
-  bsl: default
-  bucket: velero
-  kubeletRootDir: "/var/lib/kubelet/pods"
-  resources:
-    velero:
-      request:
-        cpu: 20m
-        memory: 1Gi
-      limit:
-        memory: 2Gi
-    nodeAgent:
-      request:
-        cpu: 20m
-        memory: 1Gi
-      limit:
-        memory: 2Gi

values/system/ekman/kube-flannel-rbac.yaml

@@ -0,0 +1,42 @@
+# Create the clusterrole and clusterrolebinding:
+# $ kubectl create -f kube-flannel-rbac.yml
+# Create the pod using the same namespace used by the flannel serviceaccount:
+# $ kubectl create --namespace kube-system -f kube-flannel-legacy.yml
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: flannel-client
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: flannel-client
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel-client
+subjects:
+  - kind: User
+    name: flannel-client
+    apiGroup: rbac.authorization.k8s.io

values/system/manifests/cluster-auth-rbac.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cluster-admin
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-admin
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: cluster-admin
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: 'system:masters'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system-default
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: kubernetes

values/system/manifests/kube-proxy-rbac.yaml

@@ -0,0 +1,51 @@
+# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace.
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-proxy
+subjects:
+  - kind: User
+    name: kube-proxy
+    apiGroup: rbac.authorization.k8s.io
+  - kind: ServiceAccount
+    name: kube-proxy
+    namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-proxy
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-proxy
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - events
+      - services
+      - nodes
+    verbs: ["get", "watch", "list"]
+  - nonResourceURLs: ["*"]
+    verbs: ["get", "watch", "list"]
+
+  - apiGroups:
+      - ""
+      - "events.k8s.io"
+    resources:
+      - events
+    verbs: ["*"]
+      
+  - nonResourceURLs: ["*"]
+    verbs: ["*"]
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - list
+    - watch
+
+

values/system/manifests/operator-role.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: az-kubernetes-operators-cluster-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: dd2aa2d6-269d-48fe-90cc-04fd5c08bd29

values/system/manifests/system.yaml

@@ -0,0 +1,34 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: system
+  namespace: argocd
+spec:
+  destination:
+    namespace: kube-system
+    server: 'https://kubernetes.default.svc'
+  sources:
+  - repoURL: {{ .Values.clusterConfig.manifests }}
+    targetRevision: HEAD
+    path: helmfile.d
+    plugin:
+      name: helmfile-cmp
+      env:
+      - name: CLUSTER_NAME
+        value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: system.yaml.gotmpl
+  project: sys
+  syncPolicy:
+    managedNamespaceMetadata:
+      labels:
+        component: sys
+    syncOptions:
+      - CreateNamespace=true
+      - ApplyOutOfSyncOnly=true
+      - ServerSideApply=true
+    automated:
+      prune: true
+      # selfHeal: false