wip: Match chart to k8s state

apps/charts/sys-cilium-policies/templates/plausible/CiliumNetworkPolicy-allow-gravatar.yaml

@@ -0,0 +1,14 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-plausible-secure-gravatar
+  namespace: prometheus
+spec:
+  description: Allow Plausible Gravatar
+  egress:
+    - toFQDNs:
+        - matchName: secure.gravatar.com
+        - matchName: gravatar.com
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: plausible-analytics

apps/charts/sys-cilium-policies/templates/robusta/CiliumNetworkPolicy-allow-slack.yaml

@@ -2,10 +2,11 @@ apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
   name: allow-slack
-  namespace: robusta
+  namespace: prometheus
 spec:
   egress:
     - toFQDNs:
         - matchPattern: slack.com
+        - matchName: hooks.slack.com
   endpointSelector:
     matchLabels: {}

apps/templates/argocd-apps.yaml

@@ -21,6 +21,7 @@ spec:
             sourceRepos:
             - '{{ .Values.cluster_config.manifests }}'
             - 'https://argoproj.github.io/argo-helm'
+            - 'https://gitlab.com/oceanbox/manifests.git'
             - 'https://kubernetes-sigs.github.io/metrics-server/'
             - 'https://kubernetes.github.io/ingress-nginx'
             - 'https://cloudnative-pg.github.io/charts'

apps/templates/resources/pre-cert-manager.yaml

@@ -10,7 +10,7 @@ spec:
     # The ACME server URL
     server: https://acme-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: {{ .Values.cluster_config.acme_email }}
+    email: {{ .Values.clusterConfig.acme_email }}
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-production
@@ -30,7 +30,7 @@ spec:
     # The ACME server URL
     server: https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: {{ .Values.cluster_config.acme_email }}
+    email: {{ .Values.clusterConfig.acme_email }}
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-staging
@@ -93,7 +93,7 @@ rules:
   - watch
 ---
 
-{{ if .Values.cluster_config.initca }}
+{{ if .Values.clusterConfig.initca }}
 
 # Pod to update certificates from master nodes
 # only runs on control plane nodes (etcd)
@@ -153,11 +153,11 @@ spec:
       volumes:
         - name: ca-pem
           hostPath:
-            path: {{.Values.cluster_config.initca}}/ca.pem
+            path: {{.Values.clusterConfig.initca}}/ca.pem
             type: File
         - name: ca-key-pem
           hostPath:
-            path: {{.Values.cluster_config.initca}}/ca-key.pem
+            path: {{.Values.clusterConfig.initca}}/ca-key.pem
             type: File
         - name: certs-script
           configMap:

argocd/helmfile-cmp/generate.sh

@@ -10,9 +10,7 @@ export HELM_CONFIG_HOME=/tmp/helm/config
 export HELMFILE_CACHE_HOME=/tmp/helmfile/cache
 export HELMFILE_TEMPDIR=/tmp/helmfile/tmp
 
-[[ -v ARGOCD_ENV_HELMFILE_ENVIRONMENT ]] && export HELMFILE_ENVIRONMENT=$ARGOCD_ENV_HELMFILE_ENVIRONMENT
+test -n ARGOCD_ENV_HELMFILE_ENVIRONMENT && export HELMFILE_ENVIRONMENT=$ARGOCD_ENV_HELMFILE_ENVIRONMENT
+test -n ARGOCD_ENV_HELMFILE_FILE_PATH && export HELMFILE_FILE_PATH=$ARGOCD_ENV_HELMFILE_FILE_PATH
 
-env > /tmp/$ARGOCD_APP_NAME.env
+helmfile -n "$ARGOCD_APP_NAMESPACE" $ARGS template --include-crds -q
-
-# helmfile -n "$ARGOCD_APP_NAMESPACE" $ARGS template --include-crds -q
-helmfile -n "$ARGOCD_APP_NAMESPACE" $ARGS template --include-crds --debug

argocd/helmfile-cmp/plugin.yaml

@@ -4,7 +4,8 @@ metadata:
   name: helmfile-cmp
 spec:
   generate:
-    command: [ /bin/sh ]
+    command: [ "/bin/sh" ]
     args:
       - /plugin/generate.sh
   lockRepo: false
+  preserveFileMode: true

helmfile.d/argo.yaml.gotmpl

@@ -3,16 +3,19 @@ bases:
 
 repositories:
 - name: argo
-  url: https://argoproj.github.io/argo-helm
+  url: 'https://argoproj.github.io/argo-helm'
 
 commonLabels:
   tier: system
 
+apiVersions:
+- monitoring.coreos.com/v1
+
 releases:
 - name: argocd
   namespace: argocd
   chart: argo/argo-cd
-  version: 7.5.2
+  version: 7.8.0
   condition: argo.enabled
   values:
   - ../values/argo/values/argocd.yaml.gotmpl
@@ -49,9 +52,9 @@ releases:
   condition: argo.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/argo/values.yaml.gotmpl
-  - ../values/argo/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/argo/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/cert-manager.yaml.gotmpl

@@ -6,7 +6,7 @@ repositories:
     url: 'https://charts.jetstack.io'
 
 commonLabels:
-  tier: sys
+  tier: system
 
 releases:
 - name: cert-manager
@@ -27,9 +27,9 @@ releases:
   condition: cert_manager.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/cert-manager/values.yaml.gotmpl
-  - ../values/cert-manager/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/cert-manager/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/cilium.yaml.gotmpl

@@ -41,9 +41,9 @@ releases:
   condition: cilium.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/cilium/values.yaml.gotmpl
-  - ../values/cilium/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/cilium/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/ingress-nginx.yaml.gotmpl

@@ -2,11 +2,11 @@ bases:
  - ../envs/environments.yaml.gotmpl
 
 repositories:
-  - name: ingress-nginx
+- name: ingress-nginx
   url: 'https://kubernetes.github.io/ingress-nginx'
 
 commonLabels:
-  tier: sys
+  tier: system
 
 releases:
 - name: ingress-nginx
@@ -27,9 +27,9 @@ releases:
   condition: nginx.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/ingress-nginx/values.yaml.gotmpl
-  - ../values/ingress-nginx/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/ingress-nginx/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/kyverno.yaml.gotmpl

@@ -6,7 +6,7 @@ repositories:
   url: 'https://kyverno.github.io/kyverno/'
 
 commonLabels:
-  tier: sys
+  tier: system
 
 apiVersions:
 - monitoring.coreos.com/v1
@@ -30,9 +30,9 @@ releases:
   condition: kyverno.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/kyverno/values.yaml.gotmpl
-  - ../values/kyverno/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/kyverno/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/metrics-server.yaml.gotmpl

@@ -0,0 +1,42 @@
+bases:
+ - ../envs/environments.yaml.gotmpl
+
+repositories:
+  - name: metrics-server
+    url: 'https://kubernetes-sigs.github.io/metrics-server/'
+
+commonLabels:
+  tier: system
+
+releases:
+- name: metrics-server
+  namespace: kube-system
+  chart: metrics-server/metrics-server
+  version: 3.8.2
+  condition: metrics_server.enabled
+  values:
+  - ../values/metrics-server/values/metrics-server.yaml.gotmpl
+  - ../values/metrics-server/values/metrics-server-{{ .Environment.Name }}.yaml.gotmpl
+  postRenderer: ../bin/kustomizer
+  postRendererArgs:
+  - ../values/metrics-server/kustomize/{{ .Environment.Name }}
+  missingFileHandler: Info
+- name: metrics-server-manifests
+  namespace: kube-system
+  chart: _metrics-server-manifests
+  condition: metrics_server.enabled
+  missingFileHandler: Info
+  values:
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
+  - ../values/metrics-server/values.yaml.gotmpl
+  - ../values/metrics-server/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
+  hooks:
+  - events: [ prepare, cleanup ]
+    showlogs: true
+    command: ../bin/helmify
+    args:
+    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
+    - '{{`{{ .Release.Chart }}`}}'
+    - '{{`{{ .Environment.Name }}`}}'
+    - ../values/metrics-server/manifests
+    - _metrics-server-manifests

helmfile.d/nfs-provisioner.yaml.gotmpl

@@ -5,9 +5,8 @@ repositories:
   - name: nfs-provisioner
     url: 'https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/'
 
-
 commonLabels:
-  tier: sys
+  tier: system
 
 releases:
 - name: nfs-provisioner
@@ -28,9 +27,9 @@ releases:
   condition: nfs_provisioner.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/nfs-provisioner/values.yaml.gotmpl
-  - ../values/nfs-provisioner/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/nfs-provisioner/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/postgres-operator.yaml.gotmpl

@@ -30,9 +30,9 @@ releases:
   condition: postgres_operator.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/postgres-operator/values.yaml.gotmpl
-  - ../values/postgres-operator/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/postgres-operator/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/prometheus.yaml.gotmpl

@@ -27,9 +27,9 @@ releases:
   condition: prometheus.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/prometheus/values.yaml.gotmpl
-  - ../values/prometheus/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/prometheus/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

helmfile.d/velero.yaml.gotmpl

@@ -8,6 +8,9 @@ repositories:
 commonLabels:
   tier: system
 
+apiVersions:
+- monitoring.coreos.com/v1
+
 releases:
 - name: velero
   namespace: velero

helmfile.d/x509-exporter.yaml.gotmpl

@@ -12,6 +12,7 @@ releases:
 - name: x509-exporter
   namespace: x509-exporter
   chart: x509-exporter/x509-certificate-exporter
+  version: 3.6.0
   condition: x509_exporter.enabled
   values:
   - ../values/x509-exporter/values/x509-exporter.yaml.gotmpl
@@ -26,9 +27,9 @@ releases:
   condition: x509_exporter.enabled
   missingFileHandler: Info
   values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
+  - ../values/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
   - ../values/x509-exporter/values.yaml.gotmpl
-  - ../values/x509-exporter/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
+  - ../values/x509-exporter/values-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
   hooks:
   - events: [ prepare, cleanup ]
     showlogs: true

justfile

@@ -2,12 +2,7 @@
 default:
   just --list -u
 
-# Lint a specifc helm chart
+# Render a specifc helm chart
-l HELMFILE ENV="default":
-  # helmfile --environment={{ENV}} lint --args --quiet --skip-deps --skip-refresh -f helmfile.d/{{HELMFILE}}.yaml.gotmpl
-  helmfile --environment={{ENV}} lint --args --quiet -f helmfile.d/{{HELMFILE}}.yaml.gotmpl
-
-# NOTE: Render a specifc helm chart
 r HELMFILE ENV="default":
+  helmfile --environment={{ENV}} lint --args --quiet -f helmfile.d/{{HELMFILE}}.yaml.gotmpl
   helmfile --environment={{ENV}} template -q -f helmfile.d/{{HELMFILE}}.yaml.gotmpl --output-dir-template="../_manifests/{{HELMFILE}}/{{ENV}}"
-  # helmfile --environment={{ENV}} template -q -f helmfile.d/{{HELMFILE}}.yaml.gotmpl #--output-dir-template="../_manifests/{{HELMFILE}}/{{ENV}}/{{{{.Release.Name }}"

shell.nix

@@ -21,6 +21,7 @@ pkgs.mkShellNoCC {
   name = "clstr";
 
   packages = with pkgs; [
+    # nix helpers
     npins
     nix-converter
 
@@ -29,6 +30,7 @@ pkgs.mkShellNoCC {
     helmfileWrap
     helmfile-nix
 
+    # kubectl tools
     kubectl-cnpg
     kubectl-neat
   ];

temp/metricsserver.yaml.gotmpl

@@ -1,43 +0,0 @@
-bases:
- - ../envs/environments.yaml.gotmpl
-
-repositories:
-  - name: metricsserver
-    url: 'https://kubernetes-sigs.github.io/metrics-server/'
-
-
-commonLabels:
-  tier: sys
-
-releases:
-- name: metricsserver
-  namespace: kube-system
-  chart: metricsserver/metricsserver
-  version: 3.8.2
-  condition: metrics_server.enabled
-  values:
-  - ../values/metricsserver/values/metricsserver.yaml.gotmpl
-  - ../values/metricsserver/values/metricsserver-{{ .Environment.Name }}.yaml.gotmpl
-  postRenderer: ../bin/kustomizer
-  postRendererArgs:
-  - ../values/metricsserver/kustomize/{{ .Environment.Name }}
-  missingFileHandler: Info
-- name: metricsserver-manifests
-  namespace: kube-system
-  chart: _metricsserver-manifests
-  condition: metrics_server.enabled
-  missingFileHandler: Info
-  values:
-  - ../values/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml
-  - ../values/metricsserver/values.yaml.gotmpl
-  - ../values/metricsserver/values-{{ requiredEnv "CLUSTER_NAME" }}.yaml.gotmpl
-  hooks:
-  - events: [ prepare, cleanup ]
-    showlogs: true
-    command: ../bin/helmify
-    args:
-    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
-    - '{{`{{ .Release.Chart }}`}}'
-    - '{{`{{ .Environment.Name }}`}}'
-    - ../values/metricsserver/manifests
-    - _metricsserver-manifests

values/argo/manifests/argo.yaml

@@ -12,13 +12,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/argocd
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: argo.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -27,6 +32,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
+      - ServerSideApply=true
     {{- if .Values.argocd.autosync }}
     automated:
       prune: true

values/argo/manifests/nodeport.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: argocd-server-nodeport
+  namespace: argocd
+spec:
+  clusterIP: 10.102.84.163
+  clusterIPs:
+  - 10.102.84.163
+  internalTrafficPolicy: Cluster
+  ipFamilies:
+  - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+  - name: http
+    nodePort: 30290
+    port: 80
+    targetPort: 8080
+  - name: https
+    nodePort: 31261
+    port: 443
+    targetPort: 8080
+  selector:
+    app.kubernetes.io/instance: argocd
+    app.kubernetes.io/name: argocd-server
+  type: NodePort

values/argo/manifests/sys-project.yaml

@@ -0,0 +1,84 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: sys
+  namespace: argocd
+spec:
+  clusterResourceWhitelist:
+  - group: '*'
+    kind: '*'
+  description: sys components project
+  destinations:
+  - namespace: argocd
+    server: https://kubernetes.default.svc
+  - namespace: kube-system
+    server: https://kubernetes.default.svc
+  - namespace: ingress-nginx
+    server: https://kubernetes.default.svc
+  - namespace: serit-operator
+    server: https://kubernetes.default.svc
+  - namespace: prometheus
+    server: https://kubernetes.default.svc
+  - namespace: cnpg
+    server: https://kubernetes.default.svc
+  - namespace: cert-manager
+    server: https://kubernetes.default.svc
+  - namespace: kubernetes-dashboard
+    server: https://kubernetes.default.svc
+  - namespace: rabbitmq
+    server: https://kubernetes.default.svc
+  - namespace: sealed-secrets
+    server: https://kubernetes.default.svc
+  - namespace: gitlab
+    server: https://kubernetes.default.svc
+  - namespace: thanos
+    server: https://kubernetes.default.svc
+  - namespace: linkerd
+    server: https://kubernetes.default.svc
+  - namespace: linkerd-multicluster
+    server: https://kubernetes.default.svc
+  - namespace: observability
+    server: https://kubernetes.default.svc
+  - namespace: kyverno
+    server: https://kubernetes.default.svc
+  - namespace: velero
+    server: https://kubernetes.default.svc
+  - namespace: loki
+    server: https://kubernetes.default.svc
+  - namespace: x509-exporter
+    server: https://kubernetes.default.svc
+  - namespace: mariadb-operator
+    server: https://kubernetes.default.svc
+  - namespace: cilium-spire
+    server: https://kubernetes.default.svc
+  - namespace: cilium-test
+    server: https://kubernetes.default.svc
+  - namespace: cilium-secrets
+    server: https://kubernetes.default.svc
+  sourceRepos:
+  - https://argoproj.github.io/argo-helm
+  - https://kubernetes-sigs.github.io/metrics-server/
+  - https://gitlab.com/oceanbox/manifests.git
+  - https://gitlab.com/serit/k8s/serit-platform.git
+  - https://gitlab.com/serit/k8s/serit-platform-values.git
+  - https://gitlab.com/serit/k8s/serit-platform-manifests.git
+  - https://gitlab.com/serit/k8s/serit-operator.git
+  - https://kubernetes.github.io/ingress-nginx
+  - https://cloudnative-pg.github.io/charts
+  - https://charts.jetstack.io
+  - https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
+  - https://github.com/kubernetes/dashboard
+  - https://bitnami-labs.github.io/sealed-secrets
+  - https://prometheus-community.github.io/helm-charts
+  - https://github.com/prometheus-community/helm-charts.git
+  - https://charts.gitlab.io/
+  - https://charts.bitnami.com/bitnami
+  - https://helm.linkerd.io/stable
+  - https://github.com/jaegertracing/jaeger-operator
+  - https://kyverno.github.io/kyverno/
+  - https://vmware-tanzu.github.io/helm-charts
+  - https://grafana.github.io/helm-charts
+  - https://charts.enix.io
+  - https://helm.mariadb.com/mariadb-operator
+  - https://helm.cilium.io
+  - https://chartmuseum.github.io/charts

values/argo/values.yaml.gotmpl

@@ -8,12 +8,28 @@ argo:
     enabled: false
 
 argocd:
+  autosync: true
+  ingress:
+    enabled: true
+  adminLogin: false
+  kustomizeHelmSupport: false
+  applicationset_webhook:
+    enabled: false
   anyNamespaces:
     enabled: false
     glob: ""
+  resources:
+    controller:
+      memory: 2000Mi
+      cpu: 250m
   repoServers:
   - name: "helmfile-cmp"
     image: "registry.gitlab.com/oceanbox/manifests/helmfile-cmp:latest"
-    imagePullSecret: []
+    imagePullSecrets:
+      - gitlab-pull-secret
+  - name: "kustomize-helm-with-rewrite"
+    image: "registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest"
+    imagePullSecrets:
+      - gitlab-pull-secret
   additional_rbac_settings:
   - g, "eb17a659-4ce6-41bc-9153-d9b117c44479", role:org-admin

values/argo/values/argocd.yaml.gotmpl

@@ -135,10 +135,10 @@ controller:
       enabled: true
   resources:
     limits:
-      memory: "1000Mi"
+      memory: {{ .Values.argocd.resources.controller.memory  | default "1000Mi" }}
     requests:
-      cpu: "250m"
+      cpu: {{ .Values.argocd.resources.controller.cpu  | default "250m" }}
-      memory: "1000Mi"
+      memory: {{ .Values.argocd.resources.controller.memory  | default "1000Mi" }}
 
 # Mount azure ca as file for SAML auth
 dex:
@@ -175,6 +175,25 @@ repoServer:
       enabled: true
   {{- range .Values.argocd.repoServers }}
   extraContainers:
+  - command:
+    - /var/run/argocd/argocd-cmp-server
+    image: registry.gitlab.com/oceanbox/manifests/helmfile-cmp:latest
+    imagePullPolicy: Always
+    name: helmfile-cmp
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 999
+    terminationMessagePath: /dev/termination-log
+    terminationMessagePolicy: File
+    volumeMounts:
+    - mountPath: /var/run/argocd
+      name: var-files
+    - mountPath: /home/argocd/cmp-server/plugins
+      name: plugins
+    - mountPath: /tmp
+      name: cmp-tmp
+    - mountPath: /helm-working-dir
+      name: helm-working-dir
   - command:
     - /var/run/argocd/argocd-cmp-server
     image: {{ .image }}
@@ -196,10 +215,40 @@ repoServer:
   - name: cmp-tmp
     emptyDir: {}
   imagePullSecrets:
-    {{- range .imagePullSecret }}
+    {{- range .imagePullSecrets }}
-    - name: {{ .name }}
+    - name: {{ . }}
     {{- end }}
   {{- end }}
+  initContainers:
+  - command:
+    - /bin/sh
+    - /plugin/init-helm-repos.sh
+    env:
+    - name: OCEANBOX_HELM_ACCESS_TOKEN
+      valueFrom:
+        secretKeyRef:
+          key: token
+          name: oceanbox-helm
+          optional: false
+    image: registry.gitlab.com/oceanbox/manifests/kustomize-helm-with-rewrite:latest
+    imagePullPolicy: Always
+    name: init-helm-repos
+    resources: {}
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      readOnlyRootFilesystem: true
+      runAsNonRoot: true
+      runAsUser: 999
+      seccompProfile:
+        type: RuntimeDefault
+    terminationMessagePath: /dev/termination-log
+    terminationMessagePolicy: File
+    volumeMounts:
+    - mountPath: /helm-working-dir
+      name: helm-working-dir
 
 # Configuration for argocd server instance
 server:

values/cert-manager/manifests/cert-manager.yaml

@@ -4,8 +4,6 @@ kind: Application
 metadata:
   name: cert-manager
   namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
   finalizers:
   - resources-finalizer.argocd.argoproj.io
 spec:
@@ -14,13 +12,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/cert-manager
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: cert-manager.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -29,7 +32,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
-      # - ServerSideApply=true
+      - ServerSideApply=true
     {{- if .Values.cert_manager.autosync }}
     automated:
       prune: true

values/cert-manager/manifests/pre-cert-manager.yaml

@@ -10,7 +10,7 @@ spec:
     # The ACME server URL
     server: https://acme-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: {{ .Values.cluster_config.acme_email }}
+    email: {{ .Values.clusterConfig.acme_email }}
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-production
@@ -30,7 +30,7 @@ spec:
     # The ACME server URL
     server: https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: {{ .Values.cluster_config.acme_email }}
+    email: {{ .Values.clusterConfig.acme_email }}
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-staging
@@ -93,7 +93,7 @@ rules:
   - watch
 ---
 
-{{ if .Values.cluster_config.initca }}
+{{ if .Values.clusterConfig.initca }}
 
 # Pod to update certificates from master nodes
 # only runs on control plane nodes (etcd)
@@ -153,11 +153,11 @@ spec:
       volumes:
         - name: ca-pem
           hostPath:
-            path: {{.Values.cluster_config.initca}}/ca.pem
+            path: {{.Values.clusterConfig.initca}}/ca.pem
             type: File
         - name: ca-key-pem
           hostPath:
-            path: {{.Values.cluster_config.initca}}/ca-key.pem
+            path: {{.Values.clusterConfig.initca}}/ca-key.pem
             type: File
         - name: certs-script
           configMap:

values/cilium/cilium-manifests/cilium-test/CiliumNetworkPolicy-allow-api-server.yaml

@@ -1,22 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: allow-api-server
-  namespace: cilium-test
-spec:
-  egress:
-    - toEndpoints:
-        - {}
-    - toEntities:
-        - cluster
-    - toEntities:
-        - remote-node
-    - toEntities:
-        - world
-  endpointSelector:
-    matchLabels: {}
-  ingress:
-    - fromEntities:
-        - cluster
-    - fromEntities:
-        - world

values/cilium/cilium-manifests/cilium.yaml

@@ -12,13 +12,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/cilium
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
         - name: CLUSTER_NAME
           value: {{ .Values.clusterConfig.cluster }}
+        - name: HELMFILE_ENVIRONMENT
+          value: default
+        - name: HELMFILE_FILE_PATH
+          value: cilium.yaml.gotmpl
   project: sys
   syncPolicy:
     syncOptions:

values/cilium/cilium-manifests/policies/CiliumClusterwideNetworkPolicy-allow-s3.yaml

@@ -1,20 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumClusterwideNetworkPolicy
-metadata:
-  name: allow-s3-traffic
-spec:
-  description: Policy for egress for CNPG Backups.
-  egress:
-    - toFQDNs:
-      {{- range .Values.clusterConfig.s3.hosts }}
-        - matchName: {{ . | quote }}
-      {{- end }}
-      {{- range .Values.clusterConfig.s3.patterns }}
-        - matchPattern: {{ . | quote }}
-      {{- end }}
-    - toCIDR:
-      {{- range .Values.clusterConfig.s3.cidr }}
-        - {{ . | quote }}
-      {{- end }}
-  endpointSelector:
-    matchLabels: {}

values/cilium/values-oceanbox.yaml.gotmpl

@@ -1,9 +1,9 @@
 cilium:
   enabled: true
   nodePort:
-    enable: true
+    enabled: true
   l2announcement:
-    enable: true
+    enabled: true
   loadbalancerPool:
     enabled: true
     cidr:
@@ -12,4 +12,3 @@ cilium:
      - 10.255.241.13/32
      - 10.255.241.14/32
      - 10.255.241.15/32
-

values/cilium/values.yaml.gotmpl

@@ -16,6 +16,7 @@ cilium:
     enabled: false
   nodePort:
     enabled: false
+  # NOTE: Requires that ingresscontroller is also enabled 
   gatewayAPI:
     enabled: false
   ingressController:
@@ -23,7 +24,7 @@ cilium:
     defaultClass: false
     loadbalancerMode: shared
   policyAuditMode: false
-  upgradeCompatability: 1.15
+  upgradeCompatability: 1.16
   k8sServiceHost: localhost
   k8sServicePort: 7445
   loadbalancerPool:

values/cilium/values/cilium.yaml.gotmpl

@@ -83,6 +83,7 @@ prometheus:
   serviceMonitor:
     enabled: true
 rollOutCiliumPods: true
+podSecurityContext: null
 securityContext:
   capabilities:
     ciliumAgent:

values/ingress-nginx/manifests/ingress-nginx.yaml

@@ -12,19 +12,26 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/ingress-nginx
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
         - name: CLUSTER_NAME
           value: {{ .Values.clusterConfig.cluster }}
+        - name: HELMFILE_ENVIRONMENT
+          value: default
+        - name: HELMFILE_FILE_PATH
+          value: ingress-nginx.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
       labels:
         component: sys
     syncOptions:
+      - CreateNamespace=true
+      - ApplyOutOfSyncOnly=true
       - ServerSideApply=true
     {{- if .Values.nginx.autosync }}
     automated:

values/ingress-nginx/values/ingress-nginx.yaml.gotmpl

@@ -7,8 +7,8 @@
 fullnameOverride: main-ingress-nginx
 controller:
   resources:
-    limits:
+    #limits:
-      memory: {{ .Values.nginx.resources.controller.memory }}
+    #  memory: {{ .Values.nginx.resources.controller.memory }}
     requests:
       cpu: {{ .Values.nginx.resources.controller.cpu }}
       memory: {{ .Values.nginx.resources.controller.memory }}

values/kyverno/manifests/kyverno.yaml

@@ -10,13 +10,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/kyverno
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: kyverno.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -25,7 +30,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
-      # - ServerSideApply=true
+      - ServerSideApply=true
     {{- if .Values.kyverno.autosync }}
     automated:
       prune: true

values/kyverno/manifests/policies/sync-gitlab.yaml

@@ -16,6 +16,7 @@ metadata:
 spec:
   rules:
   - name: sync-image-pull-secret
+    skipBackgroundRequests: true
     match:
       resources:
         kinds:

values/kyverno/manifests/policies/sync-regcred.yaml

@@ -17,6 +17,7 @@ metadata:
 spec:
   rules:
   - name: sync-image-pull-secret
+    skipBackgroundRequests: true
     match:
       resources:
         kinds:

values/kyverno/manifests/policies/sync-s3-secret.yaml

@@ -29,5 +29,6 @@ spec:
         - "loki"
         - "tempo"
     name: sync-s3-secret
+    skipBackgroundRequests: true
   validationFailureAction: audit
 {{- end }}

values/kyverno/manifests/policies/whitelist-internal-ingresses.yaml

@@ -17,6 +17,7 @@ spec:
   #precondition: has whitelist annotation or 
   rules:
   - name: ensure-nginx-whitelist-exists
+    skipBackgroundRequests: true
     match:
       resources:
         kinds:
@@ -30,6 +31,7 @@ spec:
           annotations:
             +(nginx.ingress.kubernetes.io/whitelist-source-range): ""
   - name: append-existing-whitelist
+    skipBackgroundRequests: true
     match:
       resources:
         kinds:
@@ -46,10 +48,11 @@ spec:
       patchStrategicMerge:
         metadata:
           annotations:
-            {{- with .Values.cluster_config.ingress_whitelist_ips }}
+            {{- with .Values.clusterConfig.ingress_whitelist_ips }}
             nginx.ingress.kubernetes.io/whitelist-source-range: "{{`{{ @ }}`}},{{ join "," . }}"
             {{- end }}
   - name: add-nginx-whitelist
+    skipBackgroundRequests: true
     match:
       resources:
         kinds:
@@ -66,7 +69,7 @@ spec:
       patchStrategicMerge:
         metadata:
           annotations:
-            {{- with .Values.cluster_config.ingress_whitelist_ips }}
+            {{- with .Values.clusterConfig.ingress_whitelist_ips }}
             nginx.ingress.kubernetes.io/whitelist-source-range: "{{ join "," . }}"
             {{- end }}
 {{- end }}

values/kyverno/values.yaml.gotmpl

@@ -1,6 +1,6 @@
 kyverno:
   enabled: true
-  autosync: false
+  autosync: true
   metrics: false
   resources:
     cleanupController:

values/metrics-server/manifests/metrics-server.yaml

@@ -2,7 +2,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: metricsserver
+  name: metrics-server
   namespace: argocd
   finalizers:
   - resources-finalizer.argocd.argoproj.io
@@ -12,13 +12,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/metricsserver
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: metrics-server.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -27,7 +32,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
-      # - ServerSideApply=true
+      - ServerSideApply=true
     {{- if .Values.metrics_server.autosync }}
     automated:
       prune: true

values/metrics-server/values.yaml.gotmpl

@@ -1,4 +1,4 @@
-metricsserver:
+metrics_server:
     enabled: true
     autosync: true
     ignoreTLS: false

values/nfs-provisioner/manifests/nfs-provisioner.yaml

@@ -4,23 +4,24 @@ kind: Application
 metadata:
   name: nfs-provisioner
   namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-  finalizers:
-  - resources-finalizer.argocd.argoproj.io
 spec:
   destination:
     namespace: kube-system 
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/nfs-provisioner
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: nfs-provisioner.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -29,7 +30,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
-      # - ServerSideApply=true
+      - ServerSideApply=true
     {{- if .Values.nfs_provisioner.autosync }}
     automated:
       prune: true

values/nfs-provisioner/values-oceanbox.yaml.gotmpl

@@ -0,0 +1,8 @@
+nfs_provisioner:
+  enabled: true
+  autosync: true
+  archiveOnDelete: true
+  defaultClass: true
+  path: "oceanbox"
+  extraMountOpts:
+    - soft

values/nfs-provisioner/values.yaml.gotmpl

@@ -3,4 +3,5 @@ nfs_provisioner:
   autosync: true
   archiveOnDelete: true
   defaultClass: true
+  path: ""
   extraMountOpts: []

values/nfs-provisioner/values/nfs-provisioner.yaml.gotmpl

@@ -1,5 +1,5 @@
 nfs:
-  server: {{ .Values.cluster_config.fileserver }}
+  server: {{ .Values.clusterConfig.fileserver }}
   path: /{{ default (.Values.clusterConfig.cluster) .Values.nfs_provisioner.path }}
   mountOptions:
     - nfsvers=4.2

values/postgres-operator/manifests/postgres-operator.yaml

@@ -10,13 +10,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/postgres-operator
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: postgres-operator.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -25,7 +30,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ApplyOutOfSyncOnly=true
-      # - ServerSideApply=true
+      - ServerSideApply=true
     {{- if .Values.postgres_operator.autosync }}
     automated:
       prune: true

values/postgres-operator/values/postgres-operator.yaml.gotmpl

@@ -0,0 +1,6 @@
+additionalArgs: []
+config:
+  data:
+    INHERITED_ANNOTATIONS: "linkerd.io/*"
+    INHERITED_LABELS: "velero.io/*"
+

values/prometheus/manifests/policies/CiliumNetworkPolicy-allow-slack.yaml

@@ -0,0 +1,12 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-slack
+  namespace: prometheus
+spec:
+  egress:
+    - toFQDNs:
+        - matchPattern: slack.com
+        - matchName: hooks.slack.com
+  endpointSelector:
+    matchLabels: {}

values/prometheus/manifests/policies/prometheus-add-folder-to-default-dashboards.yaml

@@ -1,4 +1,4 @@
-{{- if and (.Values.kyverno.enabled) (.Values.prometheus.enabled) }}
+{{- if .Values.prometheus.enabled }}
 apiVersion: kyverno.io/v1
 kind: Policy
 metadata:

values/prometheus/manifests/prometheus.yaml

@@ -10,13 +10,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/prometheus
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
         - name: CLUSTER_NAME
           value: {{ .Values.clusterConfig.cluster }}
+        - name: HELMFILE_ENVIRONMENT
+          value: default
+        - name: HELMFILE_FILE_PATH
+          value: prometheus.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:
@@ -24,6 +29,8 @@ spec:
         component: sys
     syncOptions:
     - ServerSideApply=true
+    - CreateNamespace=true
+    - ApplyOutOfSyncOnly=true
     {{- if .Values.prometheus.autosync }}
     automated:
       prune: true

values/prometheus/values-oceanbox.yaml.gotmpl

@@ -1,15 +1,19 @@
-cilium:
+prometheus:
+  snitchUrl: "https://nosnch.in/136c1b564f"
+  pagerdutyRoutingKey: a5cff1fc46414d0bc02851e4af159ee7
+  certRenewCronEnabled: false
+  fullname: prom
+  enableFeatures:
+    - otlp-write-reciever
+    - remote-write-reciever
+  grafana:
+    persistence: true
+  thanos:
     enabled: true
-  nodePort:
+  coredns:
-    enable: true
+    targetPort: 9153
-  l2announcement:
+  scheduler:
-    enable: true
+    targetPort: 10259
-  loadbalancerPool:
+  kubelet:
     enabled: true
-    cidr:
+    https: true
-     - 10.255.241.11/32
-     - 10.255.241.12/32
-     - 10.255.241.13/32
-     - 10.255.241.14/32
-     - 10.255.241.15/32
-

values/prometheus/values.yaml.gotmpl

@@ -13,6 +13,10 @@ prometheus:
   additionalScrapeConfigs: []
   additionalDataSources: []
   enableFeatures: []
+  #alert_receiver: oncall
+  #alert_group_by: [ "cluster", "namespace", "alertname" ]
+  alert_group_by: []
+  alert_receiver: ""
   storage:
     size: 50Gi
   grafana:

values/prometheus/values/prometheus.yaml.gotmpl

@@ -1,106 +1,545 @@
-authentication:
+crds:
-  mutual:
-    spire:
-      enabled: {{ .Values.cilium.spire.enabled }}
-cgroup:
-  autoMount:
   enabled: false
-  hostRoot: /sys/fs/cgroup
+
-dashboards:
+fullnameOverride: {{ .Values.prometheus.fullname | default "prometheus-kube-prometheus" }}
-  enabled: true
+
-  namespace: prometheus
+{{- with .Values.prometheus.defaultRules }}
-enableXTSocketFallback: false
+defaultRules:
-encryption:
+{{- . | toYaml | nindent 10 }}
-  enabled: {{ .Values.cilium.encryption.enabled }}
+{{- end}}
-  type: {{ .Values.cilium.encryption.type}}
+
-envoy:
+## Configuration for alertmanager
-  enabled: {{ .Values.cilium.envoy.enabled }}
+## ref: https://prometheus.io/docs/alerting/alertmanager/
-  prometheus:
+##
-    serviceMonitor:
+alertmanager:
-      enabled: {{ .Values.cilium.envoy.enabled }}
+  config:
-extraConfig:
+    route:
-  enable-envoy-config: "true"
+      {{- if .Values.prometheus.alert_group_by }}
-hubble:
+      group_by:
+      {{- range .Values.prometheus.alert_group_by }}
+      - {{ . | quote }}
+      {{- end }}
+      {{- else }}
+      group_by: ["alertname"]
+      {{- end }}
+      group_wait: 60s
+      group_interval: 15m
+      repeat_interval: 24h
+      receiver: {{ .Values.prometheus.alert_receiver | default "pagerduty" }}
+      routes:
+      - match:
+          alertname: Watchdog
+        group_wait: 0s
+        group_interval: 1m
+        repeat_interval: 50s
+        receiver: snitch
+      {{- if .Values.prometheus.oncallUrl}}
+      - match:
+          alertname: .*
+        receiver: oncall
+        continue: true
+      {{- end }}
+    receivers:
+    - name: pagerduty
+      pagerduty_configs:
+      - routing_key: {{ default "key"  .Values.prometheus.pagerdutyRoutingKey }}
+        url: "https://events.pagerduty.com/v2/enqueue"
+        severity: {{`'{{ if .CommonLabels.severity }}{{ .CommonLabels.severity | toLower }}{{ else }}critical{{ end }}'`}}
+    {{- if .Values.prometheus.snitchUrl}}
+    - name: snitch
+      webhook_configs:
+      - url: "{{ .Values.prometheus.snitchUrl }}"
+        send_resolved: false
+    {{- end }}
+    - name: teams
+      webhook_configs:
+      - url: "https://prometheus-msteams.{{ .Values.clusterConfig.domain }}/{{ .Values.clusterConfig.cluster }}"
+        http_config:
+          tls_config:
+            insecure_skip_verify: true
+    {{- if .Values.prometheus.oncallUrl}}
+    - name: oncall
+      webhook_configs:
+      - url: "{{ .Values.prometheus.oncallUrl }}"
+        send_resolved: true
+    {{- end }}
+
+  storage: {}
+
+  ingress:
     enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }}
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      {{- with .Values.clusterConfig.ingress_whitelist_ips }}
+      nginx.ingress.kubernetes.io/whitelist-source-range: {{ join "," . }}
+      {{- end }}
+    hosts:
+      - alertmanager.{{ .Values.clusterConfig.domain }}
+    paths:
+    - /
+    pathType: ImplementationSpecific
     tls:
-    auto:
+    - secretName: alertmanager-general-tls
-      method: cronJob
+      hosts:
-  metrics:
+      - alertmanager.{{ .Values.clusterConfig.domain }}
+
+  ingressPerReplica:
+    pathType: ImplementationSpecific
+
+  alertmanagerSpec:
+    affinity: {}
+      # nodeAffinity:
+      #   requiredDuringSchedulingIgnoredDuringExecution:
+      #     nodeSelectorTerms:
+      #     - matchExpressions:
+      #       - key: kubernetes.io/hostname
+      #         operator: In
+      #         values:
+      #         - {{ .Values.clusterConfig.cluster }}-0.itpartner.intern
+
+    tolerations: []
+      # - key: unschedulable
+      #   operator: Exists
+      #   effect: NoSchedule
+
+grafana:
+  defaultDashboardsEnabled: {{ .Values.prometheus.grafana.defaultDashboardsEnabled }}
+  {{- if .Values.prometheus.grafana.plugins }}
+  plugins:
+  {{- range .Values.prometheus.grafana.plugins }}
+  - {{ . }}
+  {{- end }}
+  {{- end }}
+  grafana.ini:
+    server:
+      root_url: "https://grafana.{{.Values.clusterConfig.domain}}:443"
+    security:
+      allow_embedding: "true"
+    auth:
+      disable_login_form: "{{ .Values.prometheus.grafana.disable_login_form }}"
+    users:
+      auto_assign_org_role: "Admin"
+    {{- range .Values.clusterConfig.oidc }}
+    {{- if eq .provider "azuread" }}
+    auth.{{ .provider }}:
+      enabled: true
+      name: {{ .name }}
+      client_id: $__file{/etc/secrets/oauth/{{ .name }}/client_id}
+      client_secret: $__file{/etc/secrets/oauth/{{ .name }}/client_secret}
+      scopes: openid email profile
+      auth_url: https://login.microsoftonline.com/{{ .tenant }}/oauth2/v2.0/authorize
+      token_url: https://login.microsoftonline.com/{{ .tenant }}/oauth2/v2.0/token
+      allowed_groups: {{ .group_id }}
+      allow_sign_up: true
+      role_attribute_strict: false
+      allow_assign_grafana_admin: true
+    {{- else if eq .provider "github" }}
+    auth.{{ .provider }}:
+      name: {{ .name }}
+      enabled: true
+      client_id: $__file{/etc/secrets/oauth/{{ .name }}/client_id}
+      client_secret: $__file{/etc/secrets/oauth/{{ .name }}/client_secret}
+      allowed_organizations: {{ .allowed_organizations }}
+      {{- if .allowed_teams }}
+      allowed_teams: "{{ .allowed_teams }}"
+      {{- end }}
+      scopes: user:email,read:org
+      auth_url: https://github.com/login/oauth/authorize
+      token_url: https://github.com/login/oauth/access_token
+      allow_sign_up: true
+      role_attribute_strict: false
+      allow_assign_grafana_admin: true
+    {{- end }}
+    {{- end }}
+  extraSecretMounts:
+  {{- range .Values.clusterConfig.oidc }}
+  - name: {{ .name }}
+    secretName: {{ .secret_ref.name }}
+    defaultMode: 0440
+    mountPath: /etc/secrets/oauth/{{ .name }}
+    readOnly: true
+  {{- end }}
+
+  {{- if .Values.prometheus.grafana.persistence }}
+  persistence:
+    enabled: true
+    size: 10Gi
+  {{- end }}
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }}
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      {{- with .Values.clusterConfig.ingress_whitelist_ips}}
+      nginx.ingress.kubernetes.io/whitelist-source-range: {{ join "," . }}
+      {{- end }}
+    hosts:
+      - grafana.{{ .Values.clusterConfig.domain }}
+    path: /
+    tls:
+    - secretName: grafana-general-tls
+      hosts:
+        - grafana.{{ .Values.clusterConfig.domain }}
+  sidecar:
     dashboards:
       enabled: true
-      namespace: prometheus
+      label: grafana_dashboard
-    enabled:
+      folderAnnotation: grafana_folder
-    - dns:query;ignoreAAAA
+      annotations: {}
-    - drop
+      multicluster:
-    - tcp
+        global:
-    - flow
+          enabled: true
-    - icmp
+        etcd:
-    - policy:sourceContext=app|workload-name|pod|reserved-identity;destinationContext=app|workload-name|pod|dns|reserved-identity;labelsContext=source_namespace,destination_namespace
+          enabled: false
-    - httpV2:exemplars=false;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction
+      provider:
-    port: 12304
+        allowUiUpdates: false
+        foldersFromFilesStructure: true
+    {{- if .Values.prometheus.thanos.datasource.enabled }}
+    datasources:
+      enabled: true
+      defaultDatasourceEnabled: true
+      url: http://thanos-query-frontend.thanos.svc:9090/
+      # defaultDatasourceScrapeInterval: 15s
+      annotations: {}
+
+      ## Create datasource for each Pod of Prometheus StatefulSet;
+      ## this uses headless service `prometheus-operated` which is
+      ## created by Prometheus Operator
+      ## ref: https://git.io/fjaBS
+      createPrometheusReplicasDatasources: false
+      label: grafana_datasource
+    {{ end }}
+  {{- if or .Values.loki.enabled .Values.prometheus.additionalDataSources }}
+  additionalDataSources:
+  {{- end }}
+  {{- if .Values.tempo.enabled }}
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    orgId: 1
+    url: http://tempo.tempo:3100
+    isDefault: false
+    version: 1
+    access: proxy
+    jsonData:
+        nodeGraph:
+          enabled: true
+        serviceMap:
+          datasourceUid: 'Prometheus'
+        tracesToLogs:
+          datasourceUid: loki
+          filterByTraceID: false
+          spanEndTimeShift: "500ms"
+          spanStartTimeShift: "-500ms"
+        timeInterval: 30s
+  {{- end }}
+  {{- if .Values.loki.enabled }}
+  - name: loki
+    type: loki
+    uid: loki
+    access: proxy
+    basicAuth: false
+    editable: false
+    jsonData:
+        tlsSkipVerify: false
+        {{- if .Values.tempo.enabled }}
+        derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: trace_id
+          matcherType: label
+          name: Trace ID
+          url: $${__value.raw}
+          urlDisplayLabel: 'Trace ID: $${__value.raw}'
+      {{- end }}
+    orgId: 1
+    url: http://loki-read-headless.loki:3100
+    version: 1
+  {{- end }}
+  {{- with .Values.prometheus.additionalDataSources }}
+  {{- toYaml . | nindent 10 }}
+  {{- end }}
+
+kubeApiServer:
+  tlsConfig:
+    serverName: kubernetes
+    insecureSkipVerify: true
+
+kubelet:
+  serviceMonitor:
+    https: {{ .Values.prometheus.kubelet.https }}
+    cAdvisor: true
+
+    # NOTE(simkir): Including throttling seconds by setting this. We use that in some dashboards, and could be useful
+    ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
+    ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
+    ##
+    cAdvisorMetricRelabelings:
+      # Drop less useful container CPU metrics.
+      - sourceLabels: [__name__]
+        action: drop
+        regex: 'container_cpu_(load_average_10s|system_seconds_total|user_seconds_total)'
+      # Drop less useful container / always zero filesystem metrics.
+      - sourceLabels: [__name__]
+        action: drop
+        regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)'
+      # Drop less useful / always zero container memory metrics.
+      - sourceLabels: [__name__]
+        action: drop
+        regex: 'container_memory_(mapped_file|swap)'
+      # Drop less useful container process metrics.
+      - sourceLabels: [__name__]
+        action: drop
+        regex: 'container_(file_descriptors|tasks_state|threads_max)'
+      # Drop container_memory_failures_total{scope="hierarchy"} metrics,
+      # we only need the container scope.
+      - sourceLabels: [__name__, scope]
+        action: drop
+        regex: 'container_memory_failures_total;hierarchy'
+      # Drop container_network_... metrics that match various interfaces that
+      # correspond to CNI and similar interfaces. This avoids capturing network
+      # metrics for host network containers.
+      - sourceLabels: [__name__, interface]
+        action: drop
+        regex: 'container_network_.*;(cali|cilium|cni|lxc|nodelocaldns|tunl).*'
+      # Drop container spec metrics that overlap with kube-state-metrics.
+      - sourceLabels: [__name__]
+        action: drop
+        regex: 'container_spec.*'
+      # Drop cgroup metrics with no pod.
+      - sourceLabels: [id, pod]
+        action: drop
+        regex: '.+;'
+    # - sourceLabels: [__name__, image]
+    #   separator: ;
+    #   regex: container_([a-z_]+);
+    #   replacement: $1
+    #   action: drop
+    # - sourceLabels: [__name__]
+    #   separator: ;
+    #   regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
+    #   replacement: $1
+    #   action: drop
+
+kubeControllerManager:
+  enabled: false
+  {{- if .Values.clusterConfig.apiserverip }}
+  endpoints:
+  - {{ .Values.clusterConfig.apiserverip }}
+  {{- end }}
+  service:
+    port: 10252
+    selector:
+      k8s-app: kube-controller-manager
   serviceMonitor:
     enabled: true
-  redact:
+    https: true
+    insecureSkipVerify: true
+
+coreDns:
   enabled: true
-  relay:
+  service:
-    enabled: true
+    targetPort: {{ .Values.prometheus.coredns.targetPort | default 10055 }}
-    prometheus:
+    selector:
+      k8s-app: kube-dns
+
+kubeEtcd:
   enabled: true
+  {{- if .Values.clusterConfig.etcd_nodes }}
+  endpoints:  {{ .Values.clusterConfig.etcd_nodes }}
+  {{- end }}
+  service:
+    port: {{ .Values.prometheus.etcd.targetPort | default 2379 }}
+    targetPort: {{ .Values.prometheus.etcd.targetPort | default 2379 }}
   serviceMonitor:
     enabled: true
-  ui:
+    scheme: https
-    enabled: {{ .Values.cilium.hubble.ui }}
+    insecureSkipVerify: true
-ipam:
+    caFile: /etc/prometheus/secrets/etcd-client-cert/ca.pem
-  mode: kubernetes
+    certFile: /etc/prometheus/secrets/etcd-client-cert/etcd.pem
-kubeProxyReplacement: {{ .Values.cilium.kubeProxyReplacement }}
+    keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-key.pem
-l2announcements:
+
-  enabled: {{ .Values.cilium.l2announcement.enabled }}
+kubeScheduler:
-k8sServiceHost: {{ .Values.cilium.k8sServiceHost }}
+  enabled: false
-k8sServicePort: {{ .Values.cilium.k8sServicePort }}
+  {{- if .Values.clusterConfig.apiserverip }}
-nodePort:
+  endpoints:
-  enabled: {{ .Values.cilium.nodePort.enabled }}
+  - {{ .Values.clusterConfig.apiserverip }}
-gatewayAPI:
+  {{- end }}
-  enabled: {{ .Values.cilium.gatewayAPI.enabled }}
+  service:
-ingressController:
+    port: {{ .Values.prometheus.scheduler.targetPort | default 10251 }}
-  enabled: {{ .Values.cilium.ingressController.enabled }}
+    targetPort: {{ .Values.prometheus.scheduler.targetPort | default 10251 }}
-  default: {{ .Values.cilium.ingressController.defaultClass }}
+    selector:
-  loadbalancerMode: {{ .Values.cilium.ingressController.loadbalancerMode }}
+      k8s-app: kube-scheduler
-operator:
+
-  dashboards:
+kubeProxy:
+  enabled: false
+  {{- if .Values.clusterConfig.k8s_nodes }}
+  endpoints:  {{ .Values.clusterConfig.k8s_nodes }}
+  {{- else }}
+  service:
+    selector:
+      k8s-app: kube-proxy
+  {{- end }}
+
+prometheusOperator:
   enabled: true
-    namespace: prometheus
+
-  prometheus:
+  admissionWebhooks:
+    certManager:
       enabled: true
-    port: 12301
+      issuerRef:
-    serviceMointor:
+        name: "ca-issuer"
-      enabled: true
+        kind: "ClusterIssuer"
-      port: 12302
+
-  rollOutPods: true
+  kubeletService:
-policyAuditMode: {{ .Values.cilium.policyAuditMode }}
+    enabled: {{ .Values.prometheus.kubelet.enabled }}
+
+
 prometheus:
   enabled: true
-  port: 12300
+
-  serviceMonitor:
+  thanosService:
+    enabled: false
+    type: ClusterIP
+
+    ## gRPC port config
+    portName: grpc
+    port: 10901
+    targetPort: "grpc"
+
+    ## HTTP port config (for metrics)
+    httpPortName: http
+    httpPort: 10902
+    targetHttpPort: "http"
+
+    # Default is to make this a headless service ("None")
+    # clusterIP: "None"
+
+    ## Port to expose on each node, if service type is NodePort
+    ##
+    nodePort: 30901
+    httpNodePort: 30902
+
+  {{- if .Values.prometheus.thanos.enabled }}
+  # ServiceMonitor to scrape Sidecar metrics
+  # Needs thanosService to be enabled as well
+  thanosServiceMonitor:
     enabled: true
-rollOutCiliumPods: true
+    interval: ""
-securityContext:
+
-  capabilities:
+  thanosIngress:
-    ciliumAgent:
+    enabled: true
-    - CHOWN
+    servicePort: 10901
-    - KILL
+    ingressClassName: nginx
-    - NET_ADMIN
+    annotations:
-    - NET_RAW
+      cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }}
-    - IPC_LOCK
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
-    - SYS_ADMIN
+      nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
-    - SYS_RESOURCE
+      {{- with .Values.clusterConfig.ingress_whitelist_ips }}
-    - DAC_OVERRIDE
+      nginx.ingress.kubernetes.io/whitelist-source-range: {{ join "," . }}
-    - FOWNER
+      {{- end }}
-    - SETGID
+      kubernetes.io/ingress.allow-http: "false"
-    - SETUID
+    hosts:
-    cleanCiliumState:
+      - thanos-gateway.{{ .Values.clusterConfig.domain }}
-    - NET_ADMIN
+    paths:
-    - SYS_ADMIN
+      - /
-    - SYS_RESOURCE
+    pathType: ImplementationSpecific
-{{- with .Values.cilium.upgradeCompatability}}
+    tls:
-upgradeCompatability:  {{ . }}
+      - secretName: thanos-gateway-tls
-{{- end }}
+        hosts:
+        - thanos-gateway.{{ .Values.clusterConfig.domain }}
+    {{- end }}
+
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }}
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      {{- with .Values.clusterConfig.ingress_whitelist_ips }}
+      nginx.ingress.kubernetes.io/whitelist-source-range: {{ join "," . }}
+      {{- end }}
+    hosts:
+      - prometheus.{{ .Values.clusterConfig.domain }}
+    paths:
+    - /
+    pathType: ImplementationSpecific
+    tls:
+      - secretName: prometheus-general-tls
+        hosts:
+          - prometheus.{{ .Values.clusterConfig.domain }}
+
+  ingressPerReplica:
+    enabled: false
+    pathType: ImplementationSpecific
+
+  prometheusSpec:
+    #{{- if .Values.install.otel.enabled }}
+    enableRemoteWriteReceiver: true
+    #{{- end }}
+
+    tolerations: []
+      # - key: unschedulable
+      #   operator: Exists
+      #   effect: NoSchedule
+    secrets:
+      - etcd-client-cert
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: {{ .Values.prometheus.storage.size }}
+    {{- with .Values.prometheus.enableFeatures}}
+    enableFeatures:
+    {{- range . }}
+    - {{ . }}
+    {{- end }}
+    {{- end }}
+
+    ## External labels to add to any time series or alerts when communicating with external systems
+    ##
+    externalLabels:
+      cluster: {{ .Values.clusterConfig.cluster }}
+
+    ## Name of the external label used to denote replica name
+    ##
+    replicaExternalLabelName: ""
+
+    ## If true, the Operator won't add the external label used to denote replica name
+    ##
+    replicaExternalLabelNameClear: true
+
+    ## Name of the external label used to denote Prometheus instance name
+    ##
+    prometheusExternalLabelName: ""
+
+    ## If true, the Operator won't add the external label used to denote Prometheus instance name
+    ##
+    prometheusExternalLabelNameClear: true
+
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+
+    ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment.
+    ## This section is experimental, it may change significantly without deprecation notice in any release.
+    ## This is experimental and may change significantly without backward compatibility in any release.
+    ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#thanosspec
+    ##
+    {{- if .Values.prometheus.thanos.enabled }}
+    thanos:
+      objectStorageConfig:
+        key: thanos.yaml
+        name: thanos-objstore-config
+    {{- end }}
+    # remoteWrite:
+    #  - url: https://thanos-receive.k1.itpartner.no/api/v1/receive
+    #    name: {{ .Values.clusterConfig.cluster }}
+    {{- with .Values.prometheus.additionalScrapeConfigs}}
+    additionalScrapeConfigs:
+      {{- toYaml . | nindent 12 }}
+    {{- end }}

values/values-oceanbox.yaml

@@ -41,4 +41,6 @@ clusterConfig:
     - 172.16.0.0/12
     - 192.168.0.0/16
     - 172.19.255.0/24
-
+install:
+  otel:
+    enabled: true

values/velero/manifests/velero.yaml

@@ -20,6 +20,8 @@ spec:
         value: {{ .Values.clusterConfig.cluster }}
       - name: HELMFILE_ENVIRONMENT
         value: default
+      - name: HELMFILE_FILE_PATH
+        value: velero.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata:

values/velero/values.yaml.gotmpl

@@ -1,12 +1,12 @@
 velero:
   enabled: true
   autosync: true
-  kubeletRootDir: "/var/lib/kubernetes/pods"
+  kubeletRootDir: "/var/lib/kubelet/pods"
-  bucket: backup
+  bucket: velero
   bsl: default
   # Opt-in or opt-out pvc backup
   # https://velero.io/docs/main/file-system-backup/#to-back-up
-  backupAllVolumes: true
+  backupAllVolumes: false
   credentials:
     secretName: "velero-s3"
   s3:
@@ -16,13 +16,13 @@ velero:
   resources:
     velero:
       request:
-        cpu: 500m
+        cpu: 20m
         memory: 1Gi
       limit:
         memory: 2Gi
     nodeAgent:
       request:
-        cpu: 500m
+        cpu: 20m
         memory: 1Gi
       limit:
         memory: 2Gi

values/x509-exporter/manifests/policies/CiliumNetworkPolicy-allow-api-server.yaml

@@ -0,0 +1,16 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-api-server
+  namespace: x509-exporter
+spec:
+  egress:
+    - toEntities:
+        - kube-apiserver
+      toPorts:
+        - ports:
+            - port: "6443"
+              protocol: TCP
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/instance: x509-exporter

values/x509-exporter/manifests/policies/CiliumNetworkPolicy-allow-prometheus-metrics.yaml

@@ -0,0 +1,17 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-prometheus-metrics
+  namespace: x509-exporter
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/instance: x509-exporter
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            io.kubernetes.pod.namespace: prometheus
+    - toPorts:
+        - ports:
+            - port: "9793"
+              protocol: TCP

values/x509-exporter/manifests/x509-exporter.yaml

@@ -14,13 +14,18 @@ spec:
     server: 'https://kubernetes.default.svc'
   sources:
   - repoURL: {{ .Values.clusterConfig.manifests }}
-    targetRevision: HEAD
+    # targetRevision: HEAD
-    path: helmfiles/x509-exporter
+    targetRevision: mrtz/helmify
+    path: helmfile.d
     plugin:
-      name: helmfile
+      name: helmfile-cmp
       env:
       - name: CLUSTER_NAME
         value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: x509-exporter.yaml.gotmpl
   project: sys
   syncPolicy:
     managedNamespaceMetadata: