temporal: Initial Temporal Setup

helmfile.d/temporal.yaml.gotmpl

@@ -0,0 +1,39 @@
+bases:
+ - ../envs/environments.yaml.gotmpl
+
+repositories:
+- name: temporal
+  url: https://go.temporal.io/helm-charts
+
+commonLabels:
+  tier: system
+
+releases:
+- name: temporal
+  namespace: temporal
+  chart: temporal/temporal
+  version: 1.2.0
+  condition: temporal.enabled
+  missingFileHandler: Info
+  values:
+  - ../values/temporal/values/temporal.yaml
+- name: manifests
+  namespace: temporal
+  chart: manifests
+  condition: temporal.enabled
+  missingFileHandler: Info
+  values:
+  - ../values/env.yaml
+  - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
+  - ../values/temporal/env.yaml.gotmpl
+  - ../values/temporal/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
+  hooks:
+  - events: [ prepare, cleanup ]
+    showlogs: true
+    command: ../bin/helmify
+    args:
+    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
+    - '{{`{{ .Release.Chart }}`}}'
+    - '{{`{{ .Environment.Name }}`}}'
+    - ../values/temporal/manifests
+    - manifests

values/argo/manifests/sys-project.yaml

@@ -104,6 +104,8 @@ spec:
     server: https://kubernetes.default.svc
   - namespace: niks3
     server: https://kubernetes.default.svc
+  - namespace: temporal
+    server: https://kubernetes.default.svc
   sourceRepos:
   - https://argoproj.github.io/argo-helm
   - https://kubernetes-sigs.github.io/metrics-server/
@@ -154,3 +156,4 @@ spec:
   - https://charts.dexidp.io
   - public.ecr.aws/diagrid/catalyst
   - ghcr.io/haproxytech/helm-charts
+  - https://go.temporal.io/helm-charts

values/atlantis/kustomize/beta/actor-config.yaml

@@ -8,3 +8,6 @@ data:
   XTRACT_QUEUE: "prod-queue"
   PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.4.4"
   PLUME_QUEUE: "prod-queue"
+  TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233"
+  TEMPORAL_NAMESPACE: "beta-atlantis"
+  TEMPORAL_TASK_QUEUE: "atlantis"

values/atlantis/kustomize/prod/actor-config.yaml

@@ -8,3 +8,6 @@ data:
   XTRACT_QUEUE: "prod-queue"
   PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.2.8"
   PLUME_QUEUE: "prod-queue"
+  TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233"
+  TEMPORAL_NAMESPACE: "prod-atlantis"
+  TEMPORAL_TASK_QUEUE: "atlantis"

values/atlantis/kustomize/staging/actor-config.yaml

@@ -7,3 +7,6 @@ data:
   XTRACT_QUEUE: "dev-queue"
   PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.4.0"
   PLUME_QUEUE: "dev-queue"
+  TEMPORAL_ADDRESS: "temporal-frontend.temporal:7233"
+  TEMPORAL_NAMESPACE: "staging-atlantis"
+  TEMPORAL_TASK_QUEUE: "atlantis"

values/temporal/env-ekman.yaml.gotmpl

@@ -0,0 +1,5 @@
+temporal:
+  # NOTE: Re-enable for the fapr migration
+  enabled: false
+  autosync: false
+  ingress: false

values/temporal/env-oceanbox.yaml.gotmpl

@@ -0,0 +1,4 @@
+temporal:
+  enabled: true
+  autosync: false
+  ingress: true

values/temporal/env.yaml.gotmpl

@@ -0,0 +1,5 @@
+temporal:
+  enabled: false
+  autosync: false
+  ingress: false
+  cluster: {{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}

values/temporal/manifests/dashboards/temporal.yaml

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: temporal-dashboard
+  namespace: prometheus
+  labels:
+    grafana_dashboard: "1"
+data:
+  temporal.json: |-
+    {
+      "title": "Temporal Server",
+      "editable": true,
+      "schemaVersion": 39,
+      "timezone": "",
+      "time": { "from": "now-6h", "to": "now" },
+      "templating": { "list": [] },
+      "panels": [
+        {
+          "type": "timeseries",
+          "title": "Frontend request rate",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "sum(rate(service_requests{service_name=\"frontend\"}[5m])) by (operation)",
+              "legendFormat": "{{`{{operation}}`}}"
+            }
+          ]
+        },
+        {
+          "type": "timeseries",
+          "title": "Persistence latency p99",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, sum(rate(persistence_latency_bucket[5m])) by (le, operation))",
+              "legendFormat": "{{`{{operation}}`}}"
+            }
+          ]
+        }
+      ]
+    }

values/temporal/manifests/ingress.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.temporal.ingress }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: temporal-web
+  namespace: temporal
+  annotations:
+    cert-manager.io/cluster-issuer: {{ .Values.clusterConfig.ingress_clusterissuer }}
+    nginx.ingress.kubernetes.io/backend-protocol: HTTP
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    oceanbox.io/expose: internal
+  labels:
+    app.kubernetes.io/name: temporal
+    app.kubernetes.io/component: web
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: temporal.{{ .Values.clusterConfig.domain }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: temporal-web
+            port:
+              number: 8080
+        path: /
+        pathType: Prefix
+  tls:
+  - hosts:
+    - temporal.{{ .Values.clusterConfig.domain }}
+    secretName: temporal.{{ .Values.clusterConfig.domain }}-tls
+{{- end }}

values/temporal/manifests/namespaces-setup.yaml

@@ -0,0 +1,38 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: temporal-namespaces-setup
+  namespace: temporal
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 10
+  ttlSecondsAfterFinished: 86400
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: temporal
+        app.kubernetes.io/component: namespaces-setup
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: tctl
+          image: temporalio/admin-tools:1.25.2
+          env:
+            - name: TEMPORAL_ADDRESS
+              value: temporal-frontend.temporal:7233
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -eu
+              until temporal operator cluster health 2>/dev/null; do
+                echo "waiting for temporal-frontend..."; sleep 5;
+              done
+              for ns in prod-atlantis staging-atlantis beta-atlantis; do
+                echo "ensuring namespace ${ns}";
+                temporal operator namespace create --namespace "${ns}" --retention 30d \
+                  || echo "namespace ${ns} already exists";
+              done
+              temporal operator namespace list

values/temporal/manifests/network/temporal-policies.yaml

@@ -0,0 +1,63 @@
+{{- if .Values.clusterConfig.cilium.enabled }}
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-temporal-frontend
+  namespace: temporal
+spec:
+  endpointSelector:
+    matchLabels: {}
+  ingress:
+  - fromEntities:
+    - cluster
+    toPorts:
+    - ports:
+      - port: "7233"
+        protocol: TCP
+      - port: "7234"
+        protocol: TCP
+      - port: "7235"
+        protocol: TCP
+      - port: "7239"
+        protocol: TCP
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-metrics-scrape
+  namespace: temporal
+spec:
+  endpointSelector:
+    matchLabels: {}
+  ingress:
+  - fromEndpoints:
+    - matchLabels:
+        k8s:io.kubernetes.pod.namespace: prometheus
+    toPorts:
+    - ports:
+      - port: "9090"
+        protocol: TCP
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-temporal-egress
+  namespace: temporal
+spec:
+  endpointSelector:
+    matchLabels: {}
+  egress:
+  - toEntities:
+    - cluster
+    - kube-apiserver
+  - toEndpoints:
+    - matchLabels:
+        k8s:io.kubernetes.pod.namespace: kube-system
+        k8s-app: kube-dns
+    toPorts:
+    - ports:
+      - port: "53"
+        protocol: UDP
+      - port: "53"
+        protocol: TCP
+{{- end }}

values/temporal/manifests/temporal-db.yaml

@@ -0,0 +1,72 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: temporal-db
+  namespace: temporal
+spec:
+  instances: 2
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  bootstrap:
+    initdb:
+      database: temporal
+      owner: temporal
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: temporal-visibility
+  namespace: temporal
+spec:
+  name: temporal_visibility
+  owner: temporal
+  cluster:
+    name: temporal-db
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: temporal-db-monitor
+  namespace: temporal
+spec:
+  selector:
+    matchLabels:
+      cnpg.io/cluster: temporal-db
+  podMetricsEndpoints:
+    - port: metrics
+---
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: tos-store
+  namespace: temporal
+spec:
+  retentionPolicy: "7d"
+  configuration:
+    destinationPath: s3://cnpg/temporal-db
+    endpointURL: http://10.255.241.30:30080
+    s3Credentials:
+      accessKeyId:
+        name: cnpg-s3
+        key: access_key
+      secretAccessKey:
+        name: cnpg-s3
+        key: access_secret
+    wal:
+      compression: snappy
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: temporal-db
+  namespace: temporal
+spec:
+  schedule: "0 0 1 * * *"
+  backupOwnerReference: self
+  cluster:
+    name: temporal-db
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

values/temporal/manifests/temporal.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.clusterConfig.argo.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: temporal
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/compare-options: ServerSideDiff=true
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  destination:
+    namespace: temporal
+    server: 'https://kubernetes.default.svc'
+  sources:
+  - repoURL: {{ .Values.clusterConfig.manifests }}
+    targetRevision: HEAD
+    path: helmfile.d
+    plugin:
+      name: helmfile-cmp
+      env:
+      - name: CLUSTER_NAME
+        value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: temporal.yaml.gotmpl
+  project: sys
+  ignoreDifferences:
+  - kind: Secret
+    jsonPointers:
+    - /metadata/labels
+    - /data
+  syncPolicy:
+    managedNamespaceMetadata:
+      labels:
+        component: sys
+    syncOptions:
+      - CreateNamespace=true
+      - ApplyOutOfSyncOnly=true
+      - ServerSideApply=true
+    {{- if .Values.temporal.autosync }}
+    automated:
+      prune: true
+      # selfHeal: false
+    {{- end }}
+{{- end }}

values/temporal/values/temporal.yaml

@@ -0,0 +1,45 @@
+server:
+  config:
+    persistence:
+      defaultStore: default
+      visibilityStore: visibility
+      numHistoryShards: 512
+      datastores:
+        default:
+          sql:
+            createDatabase: false
+            manageSchema: true
+            pluginName: postgres12
+            driverName: postgres12
+            databaseName: temporal
+            connectAddr: "temporal-db-rw.temporal:5432"
+            connectProtocol: tcp
+            user: temporal
+            existingSecret: temporal-db-app
+            secretKey: password
+            maxConns: 20
+            maxIdleConns: 20
+            maxConnLifetime: "1h"
+        # TODO: migrate visibility to Elasticsearch for advanced visibility search.
+        visibility:
+          sql:
+            createDatabase: false
+            manageSchema: true
+            pluginName: postgres12
+            driverName: postgres12
+            databaseName: temporal_visibility
+            connectAddr: "temporal-db-rw.temporal:5432"
+            connectProtocol: tcp
+            user: temporal
+            existingSecret: temporal-db-app
+            secretKey: password
+            maxConns: 10
+            maxIdleConns: 10
+            maxConnLifetime: "1h"
+    metrics:
+      prometheus:
+        timerType: histogram
+        listenAddress: "0.0.0.0:9090"
+
+web:
+  enabled: true