feat: add helmified and argofied vcluster setup
This commit is contained in:
Jonas Juselius
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# the shebang is ignored, but nice for editors
|
||||
|
||||
if type -P lorri &>/dev/null; then
|
||||
eval "$(lorri direnv)"
|
||||
else
|
||||
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
|
||||
use nix
|
||||
fi
|
||||
@@ -0,0 +1,2 @@
|
||||
~
|
||||
_*
|
||||
@@ -0,0 +1,7 @@
|
||||
# Oceanbox k8s vcluster setup
|
||||
|
||||
The script `./create-vcluster.sh` provisions a personal vcluster on a Kubernetes cluster, for usage
|
||||
with Tilt. It also automatically provisions a local `Dapr` installation on the cluster, and sets up a
|
||||
CNPG psql database cluster on the host system, and tunnels it to the vcluster for Archmeister. In
|
||||
addition, it sets up an ingress and a kubeconfig.yaml for convenient access, if `vcluster connect` isn't
|
||||
available.
|
||||
@@ -0,0 +1,22 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v2
|
||||
name: atlantis-vcluster
|
||||
description: vClusters for Atlantis
|
||||
type: application
|
||||
version: v0.18.1
|
||||
appVersion: v0.18.1
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: allow-external-services
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
egress:
|
||||
- toFQDNs:
|
||||
- matchName: dapr.github.io
|
||||
- matchPattern: "*.k1.itpartner.no"
|
||||
- matchName: analytics.loft.rocks
|
||||
endpointSelector:
|
||||
matchLabels: {}
|
||||
@@ -0,0 +1,69 @@
|
||||
{{- if .Values.persistence -}}
|
||||
{{- $fullname := include "vCluster.fullname" . -}}
|
||||
{{- $name := include "vCluster.releaseName" . -}}
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: {{ $fullname }}-db
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
linkerd.io/inject: disabled
|
||||
spec:
|
||||
instances: 1
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: k3s
|
||||
owner: k3s
|
||||
primaryUpdateStrategy: unsupervised
|
||||
backup:
|
||||
retentionPolicy: "7d"
|
||||
storage:
|
||||
size: "5Gi"
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: {{ $name }}-archmeister
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
linkerd.io/inject: disabled
|
||||
spec:
|
||||
instances: 1
|
||||
imageName: ghcr.io/cloudnative-pg/postgis:15-3.3
|
||||
bootstrap:
|
||||
initdb:
|
||||
postInitTemplateSQL:
|
||||
- CREATE EXTENSION postgis;
|
||||
- CREATE EXTENSION postgis_topology;
|
||||
- CREATE EXTENSION fuzzystrmatch;
|
||||
- CREATE EXTENSION postgis_tiger_geocoder;
|
||||
# Example of rolling update strategy:
|
||||
# - unsupervised: automated update of the primary once all
|
||||
# replicas have been upgraded (default)
|
||||
# - supervised: requires manual supervision to perform
|
||||
# the switchover of the primary
|
||||
primaryUpdateStrategy: unsupervised
|
||||
backup:
|
||||
retentionPolicy: "7d"
|
||||
storage:
|
||||
size: "5Gi"
|
||||
bootstrap:
|
||||
pg_basebackup:
|
||||
source: prod-archmeister
|
||||
externalClusters:
|
||||
- name: prod-archmeister
|
||||
connectionParameters:
|
||||
host: prod-archmeister-rw.atlantis.svc
|
||||
user: streaming_replica
|
||||
sslmode: verify-full
|
||||
sslKey:
|
||||
name: prod-archmeister-replication
|
||||
key: tls.key
|
||||
sslCert:
|
||||
name: prod-archmeister-replication
|
||||
key: tls.crt
|
||||
sslRootCert:
|
||||
name: prod-archmeister-ca
|
||||
key: ca.crt
|
||||
{{- end -}}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
{{- $fullname := include "vCluster.fullname" . -}}
|
||||
{{- $name := include "vCluster.fullname" . -}}
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
annotations:
|
||||
kyverno.io/kyverno-version: 1.7.0
|
||||
policies.kyverno.io/description: Allow egress to vcluster kube-apiserver
|
||||
policies.kyverno.io/minversion: 1.7.0
|
||||
policies.kyverno.io/subject: Namespace, NetworkPolicy
|
||||
policies.kyverno.io/title: Generate NetworkPolicy to Existing Namespaces
|
||||
name: generate-vcluster-apiserver-networkpolicy
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
background: true
|
||||
generateExistingOnPolicyUpdate: true
|
||||
validationFailureAction: audit
|
||||
rules:
|
||||
- name: generate-vcluster-apiserver-networkpolicy
|
||||
generate:
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
name: allow-vcluster-apiserver-access
|
||||
namespace: {{ printf "{{request.object.metadata.name}}" | quote }}
|
||||
synchronize: true
|
||||
data:
|
||||
metadata:
|
||||
labels:
|
||||
created-by: kyverno
|
||||
spec:
|
||||
description: Allow egress to vcluster kube-apiserver
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
app: vcluster
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "443"
|
||||
protocol: TCP
|
||||
endpointSelector: {}
|
||||
match:
|
||||
any:
|
||||
- resources:
|
||||
kinds:
|
||||
- Namespace
|
||||
names:
|
||||
- {{ $fullname }}
|
||||
- resources:
|
||||
kinds:
|
||||
- Namespace
|
||||
selector:
|
||||
matchLabels:
|
||||
vcluster.loft.sh/vcluster-name: {{ $fullname }}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: jaegertracing.io/v1
|
||||
kind: "Jaeger"
|
||||
metadata:
|
||||
name: jaeger
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
strategy: allInOne
|
||||
ingress:
|
||||
enabled: false
|
||||
allInOne:
|
||||
image: jaegertracing/all-in-one:1.22
|
||||
options:
|
||||
query:
|
||||
base-path: /jaeger
|
||||
@@ -0,0 +1,50 @@
|
||||
{{- $fullname := include "vCluster.fullname" . -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: vcluster-create-cilium-networkpolicies
|
||||
rules:
|
||||
- apiGroups:
|
||||
- cilium.io
|
||||
resources:
|
||||
- ciliumnetworkpolicies
|
||||
verbs:
|
||||
- '*'
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: vcluster-cilium
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: vcluster-create-cilium-networkpolicies
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
namespace: {{ $fullname }}
|
||||
name: {{ $fullname }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: vcluster-jaegers
|
||||
rules:
|
||||
- apiGroups:
|
||||
- jaegertracing.io
|
||||
resources:
|
||||
- jaegers
|
||||
verbs:
|
||||
- '*'
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: vcluster-jaegers
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: vcluster-jaegers
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
namespace: {{ $fullname }}
|
||||
name: {{ $fullname }}
|
||||
@@ -0,0 +1,153 @@
|
||||
{{- $fullname := include "vCluster.fullname" . -}}
|
||||
{{- $name := include "vCluster.releaseName" . -}}
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: {{ $fullname }}
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: atlantis
|
||||
syncPolicy:
|
||||
automated: {}
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: {{ .Release.Namespace }}
|
||||
source:
|
||||
repoURL: https://charts.loft.sh
|
||||
targetRevision: 0.18.1
|
||||
chart: vcluster
|
||||
helm:
|
||||
values: |-
|
||||
vcluster:
|
||||
env:
|
||||
{{ if .Values.persistence }}
|
||||
- name: PG_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: "{{ $fullname }}-db-app"
|
||||
key: password
|
||||
- name: K3S_DATASTORE_ENDPOINT
|
||||
value: "postgres://k3s:$(PG_PASSWORD)@{{ $fullname }}-db-rw:5432/k3s"
|
||||
{{ end }}
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
ingressClassName: nginx
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-staging
|
||||
nginx.ingress.kubernetes.io/backend-protocol: HTTPS
|
||||
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
|
||||
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
|
||||
host: "{{ $fullname }}.beta.oceanbox.io"
|
||||
tls:
|
||||
- hosts:
|
||||
- "{{ $fullname }}.beta.oceanbox.io"
|
||||
secretName: "{{ $fullname }}-tls"
|
||||
|
||||
storage:
|
||||
persistence: false
|
||||
|
||||
# coredns:
|
||||
# image: coredns/coredns:1.10.1
|
||||
fallbackHostDns: true
|
||||
|
||||
multiNamespaceMode:
|
||||
enabled: true
|
||||
|
||||
mapServices:
|
||||
fromHost:
|
||||
- from: "redis/{{ .Values.environment }}-redis-master"
|
||||
to: "redis/{{ .Values.environment }}-redis-master"
|
||||
- from: "rabbitmq/{{ .Values.environment }}-rabbitmq"
|
||||
to: "rabbitmq/{{ .Values.environment }}-rabbitmq"
|
||||
- from: "{{ .Release.Namespace }}/{{ $name }}-archmeister-rw"
|
||||
to: "atlantis/{{ $name }}-archmeister-rw"
|
||||
- from: "{{ .Release.Namespace }}/jaeger-collector"
|
||||
to: "atlantis/jaeger-collector"
|
||||
|
||||
sync:
|
||||
secrets:
|
||||
all: true
|
||||
configmaps:
|
||||
all: true
|
||||
ingresses:
|
||||
enabled: true
|
||||
generic:
|
||||
clusterRole:
|
||||
extraRules:
|
||||
- apiGroups: [ "apiextensions.k8s.io" ]
|
||||
resources: [ "customresourcedefinitions" ]
|
||||
verbs: [ "get", "list", "watch" ]
|
||||
role:
|
||||
extraRules:
|
||||
- apiGroups: ["postgresql.cnpg.io"]
|
||||
resources: ["backups", "clusters", "poolers", "scheduledbackups" ]
|
||||
verbs: ["create", "delete", "patch", "update", "get", "list", "watch"]
|
||||
- apiGroups: [ "cilium.io" ]
|
||||
resources: [ "ciliumnetworkpolicies" ]
|
||||
verbs: [ "get", "list", "watch", "create", "patch" ]
|
||||
config: |-
|
||||
version: v1beta1
|
||||
import:
|
||||
- kind: CiliumNetworkPolicy
|
||||
apiVersion: cilium.io/v2
|
||||
- kind: Cluster
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
- kind: Secret
|
||||
apiVersion: v1
|
||||
export:
|
||||
- kind: CiliumNetworkPolicy
|
||||
apiVersion: cilium.io/v2
|
||||
|
||||
init:
|
||||
manifests: |-
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: admin
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: admin
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cluster-admin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
namespace: kube-system
|
||||
name: admin
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: admin-token
|
||||
namespace: kube-system
|
||||
annotations:
|
||||
kubernetes.io/service-account.name: admin
|
||||
type: kubernetes.io/service-account-token
|
||||
|
||||
# The contents of manifests-template will be templated using helm
|
||||
# this allows you to use helm values inside, e.g.: {{ .Release.Name }}
|
||||
manifestsTemplate: ''
|
||||
|
||||
helm:
|
||||
- chart:
|
||||
name: dapr
|
||||
version: 1.12.4
|
||||
repo: https://dapr.github.io/helm-charts/
|
||||
release:
|
||||
name: dapr
|
||||
namespace: dapr-system
|
||||
timeout: 180
|
||||
values: |-
|
||||
ha.enabled: false
|
||||
|
||||
# plugin:
|
||||
# secret-syncer:
|
||||
# image: registry.gitlab.com/oceanbox/vcluster-secret-syncer:v1.0.1
|
||||
# imagePullPolicy: IfNotPresent
|
||||
@@ -0,0 +1,2 @@
|
||||
environment: staging
|
||||
persistence: true
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
|
||||
if [ $# != 1 ]; then
|
||||
echo "usage: $0 cluster"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d chart ]; then
|
||||
echo "error: must be run from toplevel directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cluster=$1
|
||||
ns=$cluster-vcluster
|
||||
|
||||
kubectl get ns $ns 2>&1 > /dev/null || kubectl create ns $ns
|
||||
|
||||
helm template -n $ns $1 ./chart --set persistence=false | kubectl --context oceanbox apply -f -
|
||||
|
||||
# configure_cluster_files () {
|
||||
# mkdir -p _$cluster/host _$cluster/vcluster
|
||||
|
||||
# cd templates
|
||||
# for i in values.yaml host/*.yaml vcluster/*.yaml; do
|
||||
# sed "s/<x>/$cluster/g" $i > ../_$cluster/$i
|
||||
# done
|
||||
# cd ..
|
||||
# }
|
||||
|
||||
# configure_cluster_files
|
||||
|
||||
# kubectl apply -n $ns -f _$cluster/host/cnpg.yaml
|
||||
# echo "Waiting for databases to become ready..."
|
||||
# kubectl wait -n $ns --for=condition=ready=true cluster $ns-db --timeout=60s
|
||||
|
||||
# kubectl apply -n $ns -f _$cluster/host/generate-cnp-rules.yaml
|
||||
# kubectl apply -n $ns -f _$cluster/host/rbac.yaml
|
||||
# kubectl apply -n $ns -f _$cluster/host/jaeger.yaml
|
||||
|
||||
# vcluster create $cluster -n $ns --distro k3s --connect=false -f _$cluster/values.yaml
|
||||
# # helm template $cluster -n $ns -f _$cluster/values.yaml > k11n/_manifest.sh
|
||||
# # kubectl kustomize k11n > _$cluster/manifest.yaml
|
||||
# # kubectl apply -f _$cluster/manifest.yaml
|
||||
|
||||
# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/rbac.yaml
|
||||
|
||||
# vcluster connect $cluster -- dapr init -k
|
||||
# vcluster connect $cluster -- kubectl apply -f _$cluster/vcluster/tracing.yaml
|
||||
|
||||
# ./update-kubeconfig.sh $cluster
|
||||
@@ -0,0 +1,9 @@
|
||||
with import <nixpkgs> {};
|
||||
mkShell rec {
|
||||
nativeBuildInputs = [
|
||||
dapr-cli
|
||||
vcluster
|
||||
];
|
||||
|
||||
shellHook = '' '';
|
||||
}
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
vcluster connect $1-vcluster -n $1-vcluster \
|
||||
--context oceanbox \
|
||||
--update-current=true \
|
||||
--insecure --cluster-role cluster-admin \
|
||||
--service-account kube-system/admin \
|
||||
--kube-config-context-name $1-vcluster \
|
||||
--server https://$1-vcluster.beta.oceanbox.io
|
||||
|
||||
Reference in New Issue
Block a user