From 1a4301e6e3f3c9d2dfcac741bc9f69c53193ade0 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Fri, 9 Feb 2024 14:36:46 +0100 Subject: [PATCH] feat: add helmified and argofied vcluster setup --- vcluster/.envrc | 9 ++ vcluster/.gitignore | 2 + vcluster/README.md | 7 + vcluster/chart/.helmignore | 22 +++ vcluster/chart/Chart.yaml | 6 + vcluster/chart/templates/cnp.yaml | 13 ++ vcluster/chart/templates/cnpg.yaml | 69 ++++++++ .../chart/templates/generate-cnp-rules.yaml | 54 +++++++ vcluster/chart/templates/jaeger.yaml | 14 ++ vcluster/chart/templates/rbac.yaml | 50 ++++++ vcluster/chart/templates/vcluster.yaml | 153 ++++++++++++++++++ vcluster/chart/values.yaml | 2 + vcluster/create-vcluster.sh | 51 ++++++ vcluster/shell.nix | 9 ++ vcluster/update-kubeconfig.sh | 10 ++ 15 files changed, 471 insertions(+) create mode 100644 vcluster/.envrc create mode 100644 vcluster/.gitignore create mode 100644 vcluster/README.md create mode 100644 vcluster/chart/.helmignore create mode 100644 vcluster/chart/Chart.yaml create mode 100644 vcluster/chart/templates/cnp.yaml create mode 100644 vcluster/chart/templates/cnpg.yaml create mode 100644 vcluster/chart/templates/generate-cnp-rules.yaml create mode 100644 vcluster/chart/templates/jaeger.yaml create mode 100644 vcluster/chart/templates/rbac.yaml create mode 100644 vcluster/chart/templates/vcluster.yaml create mode 100644 vcluster/chart/values.yaml create mode 100755 vcluster/create-vcluster.sh create mode 100644 vcluster/shell.nix create mode 100755 vcluster/update-kubeconfig.sh diff --git a/vcluster/.envrc b/vcluster/.envrc new file mode 100644 index 00000000..82b2b9ef --- /dev/null +++ b/vcluster/.envrc @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use nix +fi diff --git a/vcluster/.gitignore b/vcluster/.gitignore new file mode 100644 index 00000000..34da8a92 --- /dev/null +++ b/vcluster/.gitignore @@ -0,0 +1,2 @@ +~ +_* diff --git a/vcluster/README.md b/vcluster/README.md new file mode 100644 index 00000000..be2998e6 --- /dev/null +++ b/vcluster/README.md @@ -0,0 +1,7 @@ +# Oceanbox k8s vcluster setup + +The script `./create-vcluster.sh` provisions a personal vcluster on a Kubernetes cluster, for usage +with Tilt. It also automatically provisions a local `Dapr` installation on the cluster, and sets up a +CNPG psql database cluster on the host system, and tunnels it to the vcluster for Archmeister. In +addition, it sets up an ingress and a kubeconfig.yaml for convenient access, if `vcluster connect` isn't +available. diff --git a/vcluster/chart/.helmignore b/vcluster/chart/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/vcluster/chart/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/vcluster/chart/Chart.yaml b/vcluster/chart/Chart.yaml new file mode 100644 index 00000000..44df2ea0 --- /dev/null +++ b/vcluster/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: atlantis-vcluster +description: vClusters for Atlantis +type: application +version: v0.18.1 +appVersion: v0.18.1 diff --git a/vcluster/chart/templates/cnp.yaml b/vcluster/chart/templates/cnp.yaml new file mode 100644 index 00000000..52139aa0 --- /dev/null +++ b/vcluster/chart/templates/cnp.yaml @@ -0,0 +1,13 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-external-services + namespace: {{ .Release.Namespace }} +spec: + egress: + - toFQDNs: + - matchName: dapr.github.io + - matchPattern: "*.k1.itpartner.no" + - matchName: analytics.loft.rocks + endpointSelector: + matchLabels: {} diff --git a/vcluster/chart/templates/cnpg.yaml b/vcluster/chart/templates/cnpg.yaml new file mode 100644 index 00000000..b542063f --- /dev/null +++ b/vcluster/chart/templates/cnpg.yaml @@ -0,0 +1,69 @@ +{{- if .Values.persistence -}} +{{- $fullname := include "vCluster.fullname" . -}} +{{- $name := include "vCluster.releaseName" . -}} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ $fullname }}-db + namespace: {{ .Release.Namespace }} + annotations: + linkerd.io/inject: disabled +spec: + instances: 1 + bootstrap: + initdb: + database: k3s + owner: k3s + primaryUpdateStrategy: unsupervised + backup: + retentionPolicy: "7d" + storage: + size: "5Gi" +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ $name }}-archmeister + namespace: {{ .Release.Namespace }} + annotations: + linkerd.io/inject: disabled +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgis:15-3.3 + bootstrap: + initdb: + postInitTemplateSQL: + - CREATE EXTENSION postgis; + - CREATE EXTENSION postgis_topology; + - CREATE EXTENSION fuzzystrmatch; + - CREATE EXTENSION postgis_tiger_geocoder; + # Example of rolling update strategy: + # - unsupervised: automated update of the primary once all + # replicas have been upgraded (default) + # - supervised: requires manual supervision to perform + # the switchover of the primary + primaryUpdateStrategy: unsupervised + backup: + retentionPolicy: "7d" + storage: + size: "5Gi" + bootstrap: + pg_basebackup: + source: prod-archmeister + externalClusters: + - name: prod-archmeister + connectionParameters: + host: prod-archmeister-rw.atlantis.svc + user: streaming_replica + sslmode: verify-full + sslKey: + name: prod-archmeister-replication + key: tls.key + sslCert: + name: prod-archmeister-replication + key: tls.crt + sslRootCert: + name: prod-archmeister-ca + key: ca.crt +{{- end -}} + diff --git a/vcluster/chart/templates/generate-cnp-rules.yaml b/vcluster/chart/templates/generate-cnp-rules.yaml new file mode 100644 index 00000000..effc77f3 --- /dev/null +++ b/vcluster/chart/templates/generate-cnp-rules.yaml @@ -0,0 +1,54 @@ +{{- $fullname := include "vCluster.fullname" . -}} +{{- $name := include "vCluster.fullname" . -}} +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + annotations: + kyverno.io/kyverno-version: 1.7.0 + policies.kyverno.io/description: Allow egress to vcluster kube-apiserver + policies.kyverno.io/minversion: 1.7.0 + policies.kyverno.io/subject: Namespace, NetworkPolicy + policies.kyverno.io/title: Generate NetworkPolicy to Existing Namespaces + name: generate-vcluster-apiserver-networkpolicy + namespace: {{ .Release.Namespace }} +spec: + background: true + generateExistingOnPolicyUpdate: true + validationFailureAction: audit + rules: + - name: generate-vcluster-apiserver-networkpolicy + generate: + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + name: allow-vcluster-apiserver-access + namespace: {{ printf "{{request.object.metadata.name}}" | quote }} + synchronize: true + data: + metadata: + labels: + created-by: kyverno + spec: + description: Allow egress to vcluster kube-apiserver + egress: + - toEndpoints: + - matchLabels: + app: vcluster + toPorts: + - ports: + - port: "443" + protocol: TCP + endpointSelector: {} + match: + any: + - resources: + kinds: + - Namespace + names: + - {{ $fullname }} + - resources: + kinds: + - Namespace + selector: + matchLabels: + vcluster.loft.sh/vcluster-name: {{ $fullname }} + diff --git a/vcluster/chart/templates/jaeger.yaml b/vcluster/chart/templates/jaeger.yaml new file mode 100644 index 00000000..cbc93aa4 --- /dev/null +++ b/vcluster/chart/templates/jaeger.yaml @@ -0,0 +1,14 @@ +apiVersion: jaegertracing.io/v1 +kind: "Jaeger" +metadata: + name: jaeger + namespace: {{ .Release.Namespace }} +spec: + strategy: allInOne + ingress: + enabled: false + allInOne: + image: jaegertracing/all-in-one:1.22 + options: + query: + base-path: /jaeger diff --git a/vcluster/chart/templates/rbac.yaml b/vcluster/chart/templates/rbac.yaml new file mode 100644 index 00000000..c4d02679 --- /dev/null +++ b/vcluster/chart/templates/rbac.yaml @@ -0,0 +1,50 @@ +{{- $fullname := include "vCluster.fullname" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vcluster-create-cilium-networkpolicies +rules: +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vcluster-cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vcluster-create-cilium-networkpolicies +subjects: +- kind: ServiceAccount + namespace: {{ $fullname }} + name: {{ $fullname }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vcluster-jaegers +rules: +- apiGroups: + - jaegertracing.io + resources: + - jaegers + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vcluster-jaegers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vcluster-jaegers +subjects: +- kind: ServiceAccount + namespace: {{ $fullname }} + name: {{ $fullname }} diff --git a/vcluster/chart/templates/vcluster.yaml b/vcluster/chart/templates/vcluster.yaml new file mode 100644 index 00000000..07d72c67 --- /dev/null +++ b/vcluster/chart/templates/vcluster.yaml @@ -0,0 +1,153 @@ +{{- $fullname := include "vCluster.fullname" . -}} +{{- $name := include "vCluster.releaseName" . -}} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ $fullname }} + namespace: argocd +spec: + project: atlantis + syncPolicy: + automated: {} + destination: + server: https://kubernetes.default.svc + namespace: {{ .Release.Namespace }} + source: + repoURL: https://charts.loft.sh + targetRevision: 0.18.1 + chart: vcluster + helm: + values: |- + vcluster: + env: + {{ if .Values.persistence }} + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ $fullname }}-db-app" + key: password + - name: K3S_DATASTORE_ENDPOINT + value: "postgres://k3s:$(PG_PASSWORD)@{{ $fullname }}-db-rw:5432/k3s" + {{ end }} + + ingress: + enabled: true + ingressClassName: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-staging + nginx.ingress.kubernetes.io/backend-protocol: HTTPS + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/whitelist-source-range: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 + host: "{{ $fullname }}.beta.oceanbox.io" + tls: + - hosts: + - "{{ $fullname }}.beta.oceanbox.io" + secretName: "{{ $fullname }}-tls" + + storage: + persistence: false + + # coredns: + # image: coredns/coredns:1.10.1 + fallbackHostDns: true + + multiNamespaceMode: + enabled: true + + mapServices: + fromHost: + - from: "redis/{{ .Values.environment }}-redis-master" + to: "redis/{{ .Values.environment }}-redis-master" + - from: "rabbitmq/{{ .Values.environment }}-rabbitmq" + to: "rabbitmq/{{ .Values.environment }}-rabbitmq" + - from: "{{ .Release.Namespace }}/{{ $name }}-archmeister-rw" + to: "atlantis/{{ $name }}-archmeister-rw" + - from: "{{ .Release.Namespace }}/jaeger-collector" + to: "atlantis/jaeger-collector" + + sync: + secrets: + all: true + configmaps: + all: true + ingresses: + enabled: true + generic: + clusterRole: + extraRules: + - apiGroups: [ "apiextensions.k8s.io" ] + resources: [ "customresourcedefinitions" ] + verbs: [ "get", "list", "watch" ] + role: + extraRules: + - apiGroups: ["postgresql.cnpg.io"] + resources: ["backups", "clusters", "poolers", "scheduledbackups" ] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + - apiGroups: [ "cilium.io" ] + resources: [ "ciliumnetworkpolicies" ] + verbs: [ "get", "list", "watch", "create", "patch" ] + config: |- + version: v1beta1 + import: + - kind: CiliumNetworkPolicy + apiVersion: cilium.io/v2 + - kind: Cluster + apiVersion: postgresql.cnpg.io/v1 + - kind: Secret + apiVersion: v1 + export: + - kind: CiliumNetworkPolicy + apiVersion: cilium.io/v2 + + init: + manifests: |- + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: admin + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: admin + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + subjects: + - kind: ServiceAccount + namespace: kube-system + name: admin + --- + apiVersion: v1 + kind: Secret + metadata: + name: admin-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: admin + type: kubernetes.io/service-account-token + + # The contents of manifests-template will be templated using helm + # this allows you to use helm values inside, e.g.: {{ .Release.Name }} + manifestsTemplate: '' + + helm: + - chart: + name: dapr + version: 1.12.4 + repo: https://dapr.github.io/helm-charts/ + release: + name: dapr + namespace: dapr-system + timeout: 180 + values: |- + ha.enabled: false + + # plugin: + # secret-syncer: + # image: registry.gitlab.com/oceanbox/vcluster-secret-syncer:v1.0.1 + # imagePullPolicy: IfNotPresent diff --git a/vcluster/chart/values.yaml b/vcluster/chart/values.yaml new file mode 100644 index 00000000..45832fbe --- /dev/null +++ b/vcluster/chart/values.yaml @@ -0,0 +1,2 @@ +environment: staging +persistence: true diff --git a/vcluster/create-vcluster.sh b/vcluster/create-vcluster.sh new file mode 100755 index 00000000..c3105d1a --- /dev/null +++ b/vcluster/create-vcluster.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# + +if [ $# != 1 ]; then + echo "usage: $0 cluster" + exit 1 +fi + +if [ ! -d chart ]; then + echo "error: must be run from toplevel directory" + exit 1 +fi + +cluster=$1 +ns=$cluster-vcluster + +kubectl get ns $ns 2>&1 > /dev/null || kubectl create ns $ns + +helm template -n $ns $1 ./chart --set persistence=false | kubectl --context oceanbox apply -f - + +# configure_cluster_files () { +# mkdir -p _$cluster/host _$cluster/vcluster + +# cd templates +# for i in values.yaml host/*.yaml vcluster/*.yaml; do +# sed "s//$cluster/g" $i > ../_$cluster/$i +# done +# cd .. +# } + +# configure_cluster_files + +# kubectl apply -n $ns -f _$cluster/host/cnpg.yaml +# echo "Waiting for databases to become ready..." +# kubectl wait -n $ns --for=condition=ready=true cluster $ns-db --timeout=60s + +# kubectl apply -n $ns -f _$cluster/host/generate-cnp-rules.yaml +# kubectl apply -n $ns -f _$cluster/host/rbac.yaml +# kubectl apply -n $ns -f _$cluster/host/jaeger.yaml + +# vcluster create $cluster -n $ns --distro k3s --connect=false -f _$cluster/values.yaml +# # helm template $cluster -n $ns -f _$cluster/values.yaml > k11n/_manifest.sh +# # kubectl kustomize k11n > _$cluster/manifest.yaml +# # kubectl apply -f _$cluster/manifest.yaml + +# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/rbac.yaml + +# vcluster connect $cluster -- dapr init -k +# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/tracing.yaml + +# ./update-kubeconfig.sh $cluster diff --git a/vcluster/shell.nix b/vcluster/shell.nix new file mode 100644 index 00000000..b220dd8e --- /dev/null +++ b/vcluster/shell.nix @@ -0,0 +1,9 @@ +with import {}; +mkShell rec { + nativeBuildInputs = [ + dapr-cli + vcluster + ]; + + shellHook = '' ''; +} diff --git a/vcluster/update-kubeconfig.sh b/vcluster/update-kubeconfig.sh new file mode 100755 index 00000000..98ee7074 --- /dev/null +++ b/vcluster/update-kubeconfig.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +vcluster connect $1-vcluster -n $1-vcluster \ + --context oceanbox \ + --update-current=true \ + --insecure --cluster-role cluster-admin \ + --service-account kube-system/admin \ + --kube-config-context-name $1-vcluster \ + --server https://$1-vcluster.beta.oceanbox.io +