feat: add helmified and argofied vcluster setup

vcluster/.envrc

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use nix
+fi

vcluster/.gitignore

@@ -0,0 +1,2 @@
+~
+_*

vcluster/README.md

@@ -0,0 +1,7 @@
+# Oceanbox k8s vcluster setup
+
+The script `./create-vcluster.sh` provisions a personal vcluster on a Kubernetes cluster, for usage
+with Tilt. It also automatically provisions a local `Dapr` installation on the cluster, and sets up a
+CNPG psql database cluster on the host system, and tunnels it to the vcluster for Archmeister. In
+addition, it sets up an ingress and a kubeconfig.yaml for convenient access, if `vcluster connect` isn't
+available.

vcluster/chart/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

vcluster/chart/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: atlantis-vcluster
+description: vClusters for Atlantis
+type: application
+version: v0.18.1
+appVersion: v0.18.1

vcluster/chart/templates/cnp.yaml

@@ -0,0 +1,13 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-external-services
+  namespace: {{ .Release.Namespace }}
+spec:
+  egress:
+  - toFQDNs:
+    - matchName: dapr.github.io
+    - matchPattern: "*.k1.itpartner.no"
+    - matchName: analytics.loft.rocks
+  endpointSelector:
+    matchLabels: {}

vcluster/chart/templates/cnpg.yaml

@@ -0,0 +1,69 @@
+{{- if .Values.persistence -}}
+{{- $fullname := include "vCluster.fullname" . -}}
+{{- $name := include "vCluster.releaseName" . -}}
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: {{ $fullname }}-db
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    linkerd.io/inject: disabled
+spec:
+  instances: 1
+  bootstrap:
+    initdb:
+      database: k3s
+      owner: k3s
+  primaryUpdateStrategy: unsupervised
+  backup:
+    retentionPolicy: "7d"
+  storage:
+    size: "5Gi"
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: {{ $name }}-archmeister
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    linkerd.io/inject: disabled
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgis:15-3.3
+  bootstrap:
+    initdb:
+      postInitTemplateSQL:
+      - CREATE EXTENSION postgis;
+      - CREATE EXTENSION postgis_topology;
+      - CREATE EXTENSION fuzzystrmatch;
+      - CREATE EXTENSION postgis_tiger_geocoder;
+  # Example of rolling update strategy:
+  # - unsupervised: automated update of the primary once all
+  #                 replicas have been upgraded (default)
+  # - supervised: requires manual supervision to perform
+  #               the switchover of the primary
+  primaryUpdateStrategy: unsupervised
+  backup:
+    retentionPolicy: "7d"
+  storage:
+    size: "5Gi"
+  bootstrap:
+    pg_basebackup:
+      source: prod-archmeister
+  externalClusters:
+  - name: prod-archmeister
+    connectionParameters:
+      host: prod-archmeister-rw.atlantis.svc
+      user: streaming_replica
+      sslmode: verify-full
+    sslKey:
+      name: prod-archmeister-replication
+      key: tls.key
+    sslCert:
+      name: prod-archmeister-replication
+      key: tls.crt
+    sslRootCert:
+      name: prod-archmeister-ca
+      key: ca.crt
+{{- end -}}
+

vcluster/chart/templates/generate-cnp-rules.yaml

@@ -0,0 +1,54 @@
+{{- $fullname := include "vCluster.fullname" . -}}
+{{- $name := include "vCluster.fullname" . -}}
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    kyverno.io/kyverno-version: 1.7.0
+    policies.kyverno.io/description:  Allow egress to vcluster kube-apiserver
+    policies.kyverno.io/minversion: 1.7.0
+    policies.kyverno.io/subject: Namespace, NetworkPolicy
+    policies.kyverno.io/title: Generate NetworkPolicy to Existing Namespaces
+  name: generate-vcluster-apiserver-networkpolicy
+  namespace: {{ .Release.Namespace }}
+spec:
+  background: true
+  generateExistingOnPolicyUpdate: true
+  validationFailureAction: audit
+  rules:
+  - name: generate-vcluster-apiserver-networkpolicy
+    generate:
+      apiVersion: cilium.io/v2
+      kind: CiliumNetworkPolicy
+      name: allow-vcluster-apiserver-access
+      namespace: {{ printf "{{request.object.metadata.name}}" | quote }}
+      synchronize: true
+      data:
+        metadata:
+          labels:
+            created-by: kyverno
+        spec:
+          description: Allow egress to vcluster kube-apiserver
+          egress:
+          - toEndpoints:
+            - matchLabels:
+               app: vcluster
+            toPorts:
+            - ports:
+              - port: "443"
+                protocol: TCP
+          endpointSelector: {}
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+          names:
+          - {{ $fullname }}
+      - resources:
+          kinds:
+          - Namespace
+          selector:
+            matchLabels:
+              vcluster.loft.sh/vcluster-name: {{ $fullname }}
+

vcluster/chart/templates/jaeger.yaml

@@ -0,0 +1,14 @@
+apiVersion: jaegertracing.io/v1
+kind: "Jaeger"
+metadata:
+  name: jaeger
+  namespace: {{ .Release.Namespace }}
+spec:
+  strategy: allInOne
+  ingress:
+    enabled: false
+  allInOne:
+    image: jaegertracing/all-in-one:1.22
+    options:
+      query:
+        base-path: /jaeger

vcluster/chart/templates/rbac.yaml

@@ -0,0 +1,50 @@
+{{- $fullname := include "vCluster.fullname" . -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: vcluster-create-cilium-networkpolicies
+rules:
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: vcluster-cilium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: vcluster-create-cilium-networkpolicies
+subjects:
+- kind: ServiceAccount
+  namespace: {{ $fullname }}
+  name: {{ $fullname }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: vcluster-jaegers
+rules:
+- apiGroups:
+  - jaegertracing.io
+  resources:
+  - jaegers
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: vcluster-jaegers
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: vcluster-jaegers
+subjects:
+- kind: ServiceAccount
+  namespace: {{ $fullname }}
+  name: {{ $fullname }}

vcluster/chart/templates/vcluster.yaml

@@ -0,0 +1,153 @@
+{{- $fullname := include "vCluster.fullname" . -}}
+{{- $name := include "vCluster.releaseName" . -}}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ $fullname }}
+  namespace: argocd
+spec:
+  project: atlantis
+  syncPolicy:
+    automated: {}
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: {{ .Release.Namespace }}
+  source:
+    repoURL: https://charts.loft.sh
+    targetRevision: 0.18.1
+    chart: vcluster
+    helm:
+      values: |-
+          vcluster:
+            env:
+        {{ if .Values.persistence }}
+            - name: PG_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ $fullname }}-db-app"
+                  key: password
+            - name: K3S_DATASTORE_ENDPOINT
+              value: "postgres://k3s:$(PG_PASSWORD)@{{ $fullname }}-db-rw:5432/k3s"
+        {{ end }}
+
+          ingress:
+            enabled: true
+            ingressClassName: nginx
+            annotations:
+              cert-manager.io/cluster-issuer: letsencrypt-staging
+              nginx.ingress.kubernetes.io/backend-protocol: HTTPS
+              nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+              nginx.ingress.kubernetes.io/ssl-redirect: "true"
+              nginx.ingress.kubernetes.io/whitelist-source-range: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+            host: "{{ $fullname }}.beta.oceanbox.io"
+            tls:
+            - hosts:
+              - "{{ $fullname }}.beta.oceanbox.io"
+              secretName: "{{ $fullname }}-tls"
+
+          storage:
+            persistence: false
+
+          # coredns:
+          #   image: coredns/coredns:1.10.1
+          fallbackHostDns: true
+
+          multiNamespaceMode:
+            enabled: true
+
+          mapServices:
+            fromHost:
+            - from: "redis/{{ .Values.environment }}-redis-master"
+              to: "redis/{{ .Values.environment }}-redis-master"
+            - from: "rabbitmq/{{ .Values.environment }}-rabbitmq"
+              to: "rabbitmq/{{ .Values.environment }}-rabbitmq"
+            - from: "{{ .Release.Namespace }}/{{ $name }}-archmeister-rw"
+              to: "atlantis/{{ $name }}-archmeister-rw"
+            - from: "{{ .Release.Namespace }}/jaeger-collector"
+              to: "atlantis/jaeger-collector"
+
+          sync:
+            secrets:
+              all: true
+            configmaps:
+              all: true
+            ingresses:
+              enabled: true
+            generic:
+              clusterRole:
+                extraRules:
+                  - apiGroups: [ "apiextensions.k8s.io" ]
+                    resources: [ "customresourcedefinitions" ]
+                    verbs: [ "get", "list", "watch" ]
+              role:
+                extraRules:
+                  - apiGroups: ["postgresql.cnpg.io"]
+                    resources: ["backups", "clusters", "poolers", "scheduledbackups" ]
+                    verbs: ["create", "delete", "patch", "update", "get", "list", "watch"]
+                  - apiGroups: [ "cilium.io" ]
+                    resources: [ "ciliumnetworkpolicies" ]
+                    verbs: [ "get", "list", "watch", "create", "patch" ]
+              config: |-
+                version: v1beta1
+                import:
+                - kind: CiliumNetworkPolicy
+                  apiVersion: cilium.io/v2
+                - kind: Cluster
+                  apiVersion: postgresql.cnpg.io/v1
+                - kind: Secret
+                  apiVersion: v1
+                export:
+                - kind: CiliumNetworkPolicy
+                  apiVersion: cilium.io/v2
+
+          init:
+            manifests: |-
+              ---
+              apiVersion: v1
+              kind: ServiceAccount
+              metadata:
+                name: admin
+                namespace: kube-system
+              ---
+              apiVersion: rbac.authorization.k8s.io/v1
+              kind: ClusterRoleBinding
+              metadata:
+                name: admin
+              roleRef:
+                kind: ClusterRole
+                name: cluster-admin
+                apiGroup: rbac.authorization.k8s.io
+              subjects:
+              - kind: ServiceAccount
+                namespace: kube-system
+                name: admin
+              ---
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: admin-token
+                namespace: kube-system
+                annotations:
+                  kubernetes.io/service-account.name: admin
+              type: kubernetes.io/service-account-token
+
+            # The contents of manifests-template will be templated using helm
+            # this allows you to use helm values inside, e.g.: {{ .Release.Name }}
+            manifestsTemplate: ''
+
+            helm:
+              - chart:
+                  name: dapr
+                  version: 1.12.4
+                  repo: https://dapr.github.io/helm-charts/
+                release:
+                  name: dapr
+                  namespace: dapr-system
+                  timeout: 180
+                values: |-
+                  ha.enabled: false
+
+          # plugin:
+          #   secret-syncer:
+          #     image: registry.gitlab.com/oceanbox/vcluster-secret-syncer:v1.0.1
+          #     imagePullPolicy: IfNotPresent

vcluster/chart/values.yaml

@@ -0,0 +1,2 @@
+environment: staging
+persistence: true

vcluster/create-vcluster.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+
+if [ $# != 1 ]; then
+    echo "usage: $0 cluster"
+    exit 1
+fi
+
+if [ ! -d chart ]; then
+    echo "error: must be run from toplevel directory"
+    exit 1
+fi
+
+cluster=$1
+ns=$cluster-vcluster
+
+kubectl get ns $ns 2>&1 > /dev/null || kubectl create ns $ns
+
+helm template -n $ns $1 ./chart --set persistence=false | kubectl --context oceanbox apply -f -
+
+# configure_cluster_files () {
+#     mkdir -p _$cluster/host _$cluster/vcluster
+
+#     cd templates
+#     for i in values.yaml host/*.yaml vcluster/*.yaml; do
+#         sed "s/<x>/$cluster/g" $i > ../_$cluster/$i
+#     done
+#     cd ..
+# }
+
+# configure_cluster_files
+
+# kubectl apply -n $ns -f _$cluster/host/cnpg.yaml
+# echo "Waiting for databases to become ready..."
+# kubectl wait -n $ns --for=condition=ready=true cluster $ns-db --timeout=60s
+
+# kubectl apply -n $ns -f _$cluster/host/generate-cnp-rules.yaml
+# kubectl apply -n $ns -f _$cluster/host/rbac.yaml
+# kubectl apply -n $ns -f _$cluster/host/jaeger.yaml
+
+# vcluster create $cluster -n $ns --distro k3s --connect=false -f _$cluster/values.yaml
+# # helm template $cluster -n $ns -f _$cluster/values.yaml > k11n/_manifest.sh
+# # kubectl kustomize k11n > _$cluster/manifest.yaml
+# # kubectl apply -f _$cluster/manifest.yaml
+
+# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/rbac.yaml
+
+# vcluster connect $cluster -- dapr init -k
+# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/tracing.yaml
+
+# ./update-kubeconfig.sh $cluster

vcluster/shell.nix

@@ -0,0 +1,9 @@
+with import <nixpkgs> {};
+mkShell rec {
+  nativeBuildInputs = [
+    dapr-cli
+    vcluster
+  ];
+
+  shellHook = '' '';
+}

vcluster/update-kubeconfig.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+vcluster connect $1-vcluster -n $1-vcluster \
+    --context oceanbox \
+    --update-current=true \
+    --insecure --cluster-role cluster-admin \
+    --service-account kube-system/admin \
+    --kube-config-context-name $1-vcluster \
+    --server https://$1-vcluster.beta.oceanbox.io
+