diff --git a/helmfile.d/temporal.yaml.gotmpl b/helmfile.d/temporal.yaml.gotmpl new file mode 100644 index 00000000..d1fb0915 --- /dev/null +++ b/helmfile.d/temporal.yaml.gotmpl @@ -0,0 +1,39 @@ +bases: + - ../envs/environments.yaml.gotmpl + +repositories: +- name: temporal + url: https://go.temporal.io/helm-charts + +commonLabels: + tier: system + +releases: +- name: temporal + namespace: temporal + chart: temporal/temporal + version: 1.2.0 + condition: temporal.enabled + missingFileHandler: Info + values: + - ../values/temporal/values/temporal.yaml +- name: manifests + namespace: temporal + chart: manifests + condition: temporal.enabled + missingFileHandler: Info + values: + - ../values/env.yaml + - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml + - ../values/temporal/env.yaml.gotmpl + - ../values/temporal/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl + hooks: + - events: [ prepare, cleanup ] + showlogs: true + command: ../bin/helmify + args: + - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}' + - '{{`{{ .Release.Chart }}`}}' + - '{{`{{ .Environment.Name }}`}}' + - ../values/temporal/manifests + - manifests diff --git a/values/argo/manifests/sys-project.yaml b/values/argo/manifests/sys-project.yaml index 3481094e..4a9e2344 100644 --- a/values/argo/manifests/sys-project.yaml +++ b/values/argo/manifests/sys-project.yaml @@ -104,6 +104,8 @@ spec: server: https://kubernetes.default.svc - namespace: niks3 server: https://kubernetes.default.svc + - namespace: temporal + server: https://kubernetes.default.svc sourceRepos: - https://argoproj.github.io/argo-helm - https://kubernetes-sigs.github.io/metrics-server/ @@ -154,3 +156,4 @@ spec: - https://charts.dexidp.io - public.ecr.aws/diagrid/catalyst - ghcr.io/haproxytech/helm-charts + - https://go.temporal.io/helm-charts diff --git a/values/atlantis/kustomize/beta/actor-config.yaml b/values/atlantis/kustomize/beta/actor-config.yaml index 03852f56..92e97058 100644 --- a/values/atlantis/kustomize/beta/actor-config.yaml +++ b/values/atlantis/kustomize/beta/actor-config.yaml @@ -8,3 +8,6 @@ data: XTRACT_QUEUE: "prod-queue" PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.4.4" PLUME_QUEUE: "prod-queue" + TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233" + TEMPORAL_NAMESPACE: "beta-atlantis" + TEMPORAL_TASK_QUEUE: "atlantis" diff --git a/values/atlantis/kustomize/prod/actor-config.yaml b/values/atlantis/kustomize/prod/actor-config.yaml index e91a1f90..4bc547d6 100644 --- a/values/atlantis/kustomize/prod/actor-config.yaml +++ b/values/atlantis/kustomize/prod/actor-config.yaml @@ -8,3 +8,6 @@ data: XTRACT_QUEUE: "prod-queue" PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.2.8" PLUME_QUEUE: "prod-queue" + TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233" + TEMPORAL_NAMESPACE: "prod-atlantis" + TEMPORAL_TASK_QUEUE: "atlantis" diff --git a/values/atlantis/kustomize/staging/actor-config.yaml b/values/atlantis/kustomize/staging/actor-config.yaml index f92a326d..3297ca96 100644 --- a/values/atlantis/kustomize/staging/actor-config.yaml +++ b/values/atlantis/kustomize/staging/actor-config.yaml @@ -7,3 +7,6 @@ data: XTRACT_QUEUE: "dev-queue" PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.4.0" PLUME_QUEUE: "dev-queue" + TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233" + TEMPORAL_NAMESPACE: "staging-atlantis" + TEMPORAL_TASK_QUEUE: "atlantis" diff --git a/values/temporal/env-ekman.yaml.gotmpl b/values/temporal/env-ekman.yaml.gotmpl new file mode 100644 index 00000000..48870b2f --- /dev/null +++ b/values/temporal/env-ekman.yaml.gotmpl @@ -0,0 +1,5 @@ +temporal: + # NOTE: Re-enable for the fapr migration + enabled: false + autosync: false + ingress: false diff --git a/values/temporal/env-oceanbox.yaml.gotmpl b/values/temporal/env-oceanbox.yaml.gotmpl new file mode 100644 index 00000000..d5a66617 --- /dev/null +++ b/values/temporal/env-oceanbox.yaml.gotmpl @@ -0,0 +1,4 @@ +temporal: + enabled: true + autosync: false + ingress: true diff --git a/values/temporal/env.yaml.gotmpl b/values/temporal/env.yaml.gotmpl new file mode 100644 index 00000000..68bc3818 --- /dev/null +++ b/values/temporal/env.yaml.gotmpl @@ -0,0 +1,5 @@ +temporal: + enabled: false + autosync: false + ingress: false + cluster: {{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }} diff --git a/values/temporal/manifests/dashboards/temporal.yaml b/values/temporal/manifests/dashboards/temporal.yaml new file mode 100644 index 00000000..b9284bb2 --- /dev/null +++ b/values/temporal/manifests/dashboards/temporal.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: temporal-dashboard + namespace: prometheus + labels: + grafana_dashboard: "1" +data: + temporal.json: |- + { + "title": "Temporal Server", + "editable": true, + "schemaVersion": 39, + "timezone": "", + "time": { "from": "now-6h", "to": "now" }, + "templating": { "list": [] }, + "panels": [ + { + "type": "timeseries", + "title": "Frontend request rate", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(service_requests{service_name=\"frontend\"}[5m])) by (operation)", + "legendFormat": "{{`{{operation}}`}}" + } + ] + }, + { + "type": "timeseries", + "title": "Persistence latency p99", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(persistence_latency_bucket[5m])) by (le, operation))", + "legendFormat": "{{`{{operation}}`}}" + } + ] + } + ] + } diff --git a/values/temporal/manifests/ingress.yaml b/values/temporal/manifests/ingress.yaml new file mode 100644 index 00000000..eda4550e --- /dev/null +++ b/values/temporal/manifests/ingress.yaml @@ -0,0 +1,32 @@ +{{- if .Values.temporal.ingress }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: temporal-web + namespace: temporal + annotations: + cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }} + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/ssl-redirect: "true" + oceanbox.io/expose: internal + labels: + app.kubernetes.io/name: temporal + app.kubernetes.io/component: web +spec: + ingressClassName: nginx + rules: + - host: temporal.{{ .Values.clusterConfig.domain }} + http: + paths: + - backend: + service: + name: temporal-web + port: + number: 8080 + path: / + pathType: Prefix + tls: + - hosts: + - temporal.{{ .Values.clusterConfig.domain }} + secretName: temporal.{{ .Values.clusterConfig.domain }}-tls +{{- end }} diff --git a/values/temporal/manifests/namespaces-setup.yaml b/values/temporal/manifests/namespaces-setup.yaml new file mode 100644 index 00000000..1eabaade --- /dev/null +++ b/values/temporal/manifests/namespaces-setup.yaml @@ -0,0 +1,38 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: temporal-namespaces-setup + namespace: temporal + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 10 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/name: temporal + app.kubernetes.io/component: namespaces-setup + spec: + restartPolicy: OnFailure + containers: + - name: tctl + image: temporalio/admin-tools:1.25.2 + env: + - name: TEMPORAL_ADDRESS + value: temporal-frontend.temporal:7233 + command: + - /bin/sh + - -c + - | + set -eu + until temporal operator cluster health 2>/dev/null; do + echo "waiting for temporal-frontend..."; sleep 5; + done + for ns in prod-atlantis staging-atlantis beta-atlantis; do + echo "ensuring namespace ${ns}"; + temporal operator namespace create --namespace "${ns}" --retention 30d \ + || echo "namespace ${ns} already exists"; + done + temporal operator namespace list diff --git a/values/temporal/manifests/network/temporal-policies.yaml b/values/temporal/manifests/network/temporal-policies.yaml new file mode 100644 index 00000000..f97e5897 --- /dev/null +++ b/values/temporal/manifests/network/temporal-policies.yaml @@ -0,0 +1,63 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-temporal-frontend + namespace: temporal +spec: + endpointSelector: + matchLabels: {} + ingress: + - fromEntities: + - cluster + toPorts: + - ports: + - port: "7233" + protocol: TCP + - port: "7234" + protocol: TCP + - port: "7235" + protocol: TCP + - port: "7239" + protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-metrics-scrape + namespace: temporal +spec: + endpointSelector: + matchLabels: {} + ingress: + - fromEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: prometheus + toPorts: + - ports: + - port: "9090" + protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-temporal-egress + namespace: temporal +spec: + endpointSelector: + matchLabels: {} + egress: + - toEntities: + - cluster + - kube-apiserver + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP +{{- end }} diff --git a/values/temporal/manifests/temporal-db.yaml b/values/temporal/manifests/temporal-db.yaml new file mode 100644 index 00000000..2a43abe2 --- /dev/null +++ b/values/temporal/manifests/temporal-db.yaml @@ -0,0 +1,72 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: temporal-db + namespace: temporal +spec: + instances: 2 + imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie + bootstrap: + initdb: + database: temporal + owner: temporal + storage: + resizeInUseVolumes: true + size: 10Gi +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: temporal-visibility + namespace: temporal +spec: + name: temporal_visibility + owner: temporal + cluster: + name: temporal-db +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: temporal-db-monitor + namespace: temporal +spec: + selector: + matchLabels: + cnpg.io/cluster: temporal-db + podMetricsEndpoints: + - port: metrics +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: tos-store + namespace: temporal +spec: + retentionPolicy: "7d" + configuration: + destinationPath: s3://cnpg/temporal-db + endpointURL: http://10.255.241.30:30080 + s3Credentials: + accessKeyId: + name: cnpg-s3 + key: access_key + secretAccessKey: + name: cnpg-s3 + key: access_secret + wal: + compression: snappy +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: temporal-db + namespace: temporal +spec: + schedule: "0 0 1 * * *" + backupOwnerReference: self + cluster: + name: temporal-db + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io diff --git a/values/temporal/manifests/temporal.yaml b/values/temporal/manifests/temporal.yaml new file mode 100644 index 00000000..4402f85f --- /dev/null +++ b/values/temporal/manifests/temporal.yaml @@ -0,0 +1,48 @@ +{{- if .Values.clusterConfig.argo.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: temporal + namespace: argocd + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + argocd.argoproj.io/compare-options: ServerSideDiff=true + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: temporal + server: 'https://kubernetes.default.svc' + sources: + - repoURL: {{ .Values.clusterConfig.manifests }} + targetRevision: HEAD + path: helmfile.d + plugin: + name: helmfile-cmp + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterConfig.cluster }} + - name: HELMFILE_ENVIRONMENT + value: default + - name: HELMFILE_FILE_PATH + value: temporal.yaml.gotmpl + project: sys + ignoreDifferences: + - kind: Secret + jsonPointers: + - /metadata/labels + - /data + syncPolicy: + managedNamespaceMetadata: + labels: + component: sys + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - ServerSideApply=true + {{- if .Values.temporal.autosync }} + automated: + prune: true + # selfHeal: false + {{- end }} +{{- end }} diff --git a/values/temporal/values/temporal.yaml b/values/temporal/values/temporal.yaml new file mode 100644 index 00000000..1e5e6d80 --- /dev/null +++ b/values/temporal/values/temporal.yaml @@ -0,0 +1,45 @@ +server: + config: + persistence: + defaultStore: default + visibilityStore: visibility + numHistoryShards: 512 + datastores: + default: + sql: + createDatabase: false + manageSchema: true + pluginName: postgres12 + driverName: postgres12 + databaseName: temporal + connectAddr: "temporal-db-rw.temporal:5432" + connectProtocol: tcp + user: temporal + existingSecret: temporal-db-app + secretKey: password + maxConns: 20 + maxIdleConns: 20 + maxConnLifetime: "1h" + # TODO: migrate visibility to Elasticsearch for advanced visibility search. + visibility: + sql: + createDatabase: false + manageSchema: true + pluginName: postgres12 + driverName: postgres12 + databaseName: temporal_visibility + connectAddr: "temporal-db-rw.temporal:5432" + connectProtocol: tcp + user: temporal + existingSecret: temporal-db-app + secretKey: password + maxConns: 10 + maxIdleConns: 10 + maxConnLifetime: "1h" + metrics: + prometheus: + timerType: histogram + listenAddress: "0.0.0.0:9090" + +web: + enabled: true