diff --git a/helmfile.d/clickhouse.yaml.gotmpl b/helmfile.d/clickhouse.yaml.gotmpl new file mode 100644 index 00000000..16d427c1 --- /dev/null +++ b/helmfile.d/clickhouse.yaml.gotmpl @@ -0,0 +1,53 @@ +bases: + - ../envs/environments.yaml.gotmpl + +repositories: +- name: clickhouse + url: ghcr.io/clickhouse + oci: true + +commonLabels: + tier: system + +releases: +- name: clickhouse-operator + namespace: clickhouse + chart: clickhouse/clickhouse-operator-helm + version: 0.0.6 + condition: clickhouse.enabled + values: + - ../values/clickhouse/values/operator.yaml.gotmpl + missingFileHandler: Info +- name: clickhouse-cluster + namespace: clickhouse + chart: clickhouse/clickhouse-cluster-helm + version: 0.0.6 + condition: clickhouse.enabled + needs: + - clickhouse/clickhouse-operator + values: + - ../values/clickhouse/values/cluster.yaml.gotmpl + postRenderer: ../bin/kustomizer + postRendererArgs: + - ../values/clickhouse/kustomize/{{ .Environment.Name }} + missingFileHandler: Info +- name: manifests + namespace: clickhouse + chart: manifests + condition: clickhouse.enabled + missingFileHandler: Info + values: + - ../values/env.yaml + - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml + - ../values/clickhouse/env.yaml.gotmpl + - ../values/clickhouse/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl + hooks: + - events: [ prepare, cleanup ] + showlogs: true + command: ../bin/helmify + args: + - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}' + - '{{`{{ .Release.Chart }}`}}' + - '{{`{{ .Environment.Name }}`}}' + - ../values/clickhouse/manifests + - manifests diff --git a/helmfile.d/opentelemetry-collector.yaml.gotmpl b/helmfile.d/opentelemetry-collector.yaml.gotmpl index 7c88f766..c9b22b0b 100644 --- a/helmfile.d/opentelemetry-collector.yaml.gotmpl +++ b/helmfile.d/opentelemetry-collector.yaml.gotmpl @@ -15,7 +15,7 @@ releases: version: 0.158.2 condition: otel.enabled values: - - ../values/opentelemetry-collector/values/values.yaml + - ../values/opentelemetry-collector/values/values.yaml.gotmpl - ../values/opentelemetry-collector/values/values-{{ .Environment.Name }}.yaml postRenderer: ../bin/kustomizer postRendererArgs: diff --git a/values/clickhouse/env-oceanbox.yaml.gotmpl b/values/clickhouse/env-oceanbox.yaml.gotmpl new file mode 100644 index 00000000..00e65b03 --- /dev/null +++ b/values/clickhouse/env-oceanbox.yaml.gotmpl @@ -0,0 +1,2 @@ +clickhouse: + enabled: true diff --git a/values/clickhouse/env.yaml.gotmpl b/values/clickhouse/env.yaml.gotmpl new file mode 100644 index 00000000..795af3b4 --- /dev/null +++ b/values/clickhouse/env.yaml.gotmpl @@ -0,0 +1,14 @@ +clickhouse: + enabled: false + autosync: true + clusterName: clickhouse + keeperName: clickhouse-keeper + storage: 50Gi + keeperStorage: 5Gi + database: otel + user: default + credentialsSecret: clickhouse-credentials + credentialsKey: password + ttl: + logs: 720h + traces: 336h diff --git a/values/clickhouse/manifests/clickhouse.yaml b/values/clickhouse/manifests/clickhouse.yaml new file mode 100644 index 00000000..56cfd393 --- /dev/null +++ b/values/clickhouse/manifests/clickhouse.yaml @@ -0,0 +1,42 @@ +{{- if .Values.clusterConfig.argo.enabled }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: clickhouse + namespace: argocd + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: clickhouse + server: 'https://kubernetes.default.svc' + sources: + - repoURL: {{ .Values.clusterConfig.manifests }} + targetRevision: HEAD + path: helmfile.d + plugin: + name: helmfile-cmp + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterConfig.cluster }} + - name: HELMFILE_ENVIRONMENT + value: default + - name: HELMFILE_FILE_PATH + value: clickhouse.yaml.gotmpl + project: sys + syncPolicy: + managedNamespaceMetadata: + labels: + component: sys + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - ServerSideApply=true + {{- if .Values.clickhouse.autosync }} + automated: + prune: true + # selfHeal: false + {{- end }} +{{- end }} diff --git a/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-grafana-read.yaml b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-grafana-read.yaml new file mode 100644 index 00000000..9a6acaf9 --- /dev/null +++ b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-grafana-read.yaml @@ -0,0 +1,25 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-prometheus + namespace: clickhouse +spec: + description: Allow Grafana queries and Prometheus metric scraping + endpointSelector: + matchLabels: {} + ingress: + - fromEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: prometheus + toPorts: + - ports: + - port: "9000" + protocol: TCP + - port: "8123" + protocol: TCP + - port: "8888" + protocol: TCP + - port: "9363" + protocol: TCP +{{- end }} diff --git a/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-operator-and-intra.yaml b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-operator-and-intra.yaml new file mode 100644 index 00000000..7d69de94 --- /dev/null +++ b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-operator-and-intra.yaml @@ -0,0 +1,15 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-intra + namespace: clickhouse +spec: + description: Allow all intra-namespace traffic (operator, ClickHouse servers, Keeper) + endpointSelector: + matchLabels: {} + ingress: + - fromEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: clickhouse +{{- end }} diff --git a/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-otel-write.yaml b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-otel-write.yaml new file mode 100644 index 00000000..49928cb7 --- /dev/null +++ b/values/clickhouse/manifests/network/CiliumNetworkPolicy-allow-otel-write.yaml @@ -0,0 +1,21 @@ +{{- if .Values.clusterConfig.cilium.enabled }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-otel-write + namespace: clickhouse +spec: + description: Allow the opentelemetry collector to write to ClickHouse + endpointSelector: + matchLabels: {} + ingress: + - fromEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: otel + toPorts: + - ports: + - port: "9000" + protocol: TCP + - port: "8123" + protocol: TCP +{{- end }} diff --git a/values/clickhouse/values/cluster.yaml.gotmpl b/values/clickhouse/values/cluster.yaml.gotmpl new file mode 100644 index 00000000..d4422775 --- /dev/null +++ b/values/clickhouse/values/cluster.yaml.gotmpl @@ -0,0 +1,32 @@ +imageTag: "24.8" + +clickhouse: + enabled: true + meta: + name: {{ .Values.clickhouse.clusterName }} + spec: + shards: 1 + replicas: 1 + defaultUserPassword: + secret: + name: {{ .Values.clickhouse.credentialsSecret }} + key: {{ .Values.clickhouse.credentialsKey }} + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.clickhouse.storage }} + +keeper: + enabled: true + meta: + name: {{ .Values.clickhouse.keeperName }} + spec: + replicas: 1 + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.clickhouse.keeperStorage }} diff --git a/values/clickhouse/values/operator.yaml.gotmpl b/values/clickhouse/values/operator.yaml.gotmpl new file mode 100644 index 00000000..ffbeeaeb --- /dev/null +++ b/values/clickhouse/values/operator.yaml.gotmpl @@ -0,0 +1 @@ +# clickhouse-operator-helm values (chart defaults) diff --git a/values/opentelemetry-collector/values/values.yaml b/values/opentelemetry-collector/values/values.yaml.gotmpl similarity index 69% rename from values/opentelemetry-collector/values/values.yaml rename to values/opentelemetry-collector/values/values.yaml.gotmpl index 02acfe8a..420c5e9c 100644 --- a/values/opentelemetry-collector/values/values.yaml +++ b/values/opentelemetry-collector/values/values.yaml.gotmpl @@ -4,6 +4,14 @@ image: service: type: LoadBalancer loadBalancerIP: 10.255.241.12 +{{- if .Values.clickhouse.enabled }} +extraEnvs: + - name: CH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.clickhouse.credentialsSecret }} + key: {{ .Values.clickhouse.credentialsKey }} +{{- end }} config: # receivers: # prometheus/collector: @@ -31,6 +39,20 @@ config: endpoint: http://loki-write-headless.loki:3100/otlp tls: insecure: true + {{- if .Values.clickhouse.enabled }} + clickhouse: + endpoint: tcp://{{ .Values.clickhouse.clusterName }}.clickhouse.svc:9000?dial_timeout=10s + database: {{ .Values.clickhouse.database }} + username: {{ .Values.clickhouse.user }} + password: ${env:CH_PASSWORD} + create_schema: true + logs_table_name: otel_logs + traces_table_name: otel_traces + ttl: {{ .Values.clickhouse.ttl.logs }} + timeout: 10s + retry_on_failure: + enabled: true + {{- end }} debug/metrics: verbosity: detailed debug/traces: @@ -45,8 +67,7 @@ config: traces: receivers: [otlp] # zipkin processors: [batch] - exporters: [otlp] - # exporters: [otlphttp/traces,debug/traces] + exporters: [otlp{{ if .Values.clickhouse.enabled }}, clickhouse{{ end }}] metrics: receivers: [otlp,prometheus] # prometheus/collector processors: [batch] @@ -55,8 +76,7 @@ config: logs: receivers: [otlp] processors: [batch] - exporters: [otlphttp/logs] - # exporters: [otlphttp/logs,debug/logs] + exporters: [otlphttp/logs{{ if .Values.clickhouse.enabled }}, clickhouse{{ end }}] ports: metrics: enabled: true diff --git a/values/prometheus/env-oceanbox.yaml.gotmpl b/values/prometheus/env-oceanbox.yaml.gotmpl index f6f7219f..d0a9ecfe 100644 --- a/values/prometheus/env-oceanbox.yaml.gotmpl +++ b/values/prometheus/env-oceanbox.yaml.gotmpl @@ -11,10 +11,7 @@ prometheus: persistence: true plugins: - volkovlabs-image-panel - - marcusolsson-static-datasource - - marcusolsson-calendar-panel - grafana-clock-panel - - redis-datasource thanos: enabled: true coredns: diff --git a/values/prometheus/values/prometheus.yaml.gotmpl b/values/prometheus/values/prometheus.yaml.gotmpl index e9504575..f1a36d74 100644 --- a/values/prometheus/values/prometheus.yaml.gotmpl +++ b/values/prometheus/values/prometheus.yaml.gotmpl @@ -108,11 +108,21 @@ grafana: defaultDashboardsEnabled: {{ .Values.prometheus.grafana.defaultDashboardsEnabled }} deploymentStrategy: type: Recreate - {{- if .Values.prometheus.grafana.plugins }} + {{- if or .Values.prometheus.grafana.plugins .Values.clickhouse.enabled }} plugins: {{- range .Values.prometheus.grafana.plugins }} - {{ . }} {{- end }} + {{- if .Values.clickhouse.enabled }} + - grafana-clickhouse-datasource + {{- end }} + {{- end }} + {{- if .Values.clickhouse.enabled }} + envValueFrom: + CLICKHOUSE_PASSWORD: + secretKeyRef: + name: {{ .Values.clickhouse.credentialsSecret }} + key: {{ .Values.clickhouse.credentialsKey }} {{- end }} grafana.ini: server: @@ -219,9 +229,49 @@ grafana: createPrometheusReplicasDatasources: false label: grafana_datasource {{ end }} - {{- if or .Values.loki.enabled .Values.prometheus.additionalDataSources }} + {{- if or .Values.loki.enabled .Values.clickhouse.enabled .Values.prometheus.additionalDataSources }} additionalDataSources: {{- end }} + {{- if .Values.clickhouse.enabled }} + - name: ClickHouse logs + type: grafana-clickhouse-datasource + uid: clickhouse-logs + access: proxy + editable: false + jsonData: + host: {{ .Values.clickhouse.clusterName }}.clickhouse.svc + port: 9000 + protocol: native + username: {{ .Values.clickhouse.user }} + defaultDatabase: {{ .Values.clickhouse.database }} + logs: + defaultDatabase: {{ .Values.clickhouse.database }} + defaultTable: otel_logs + otelEnabled: true + otelVersion: latest + secureJsonData: + password: ${CLICKHOUSE_PASSWORD} + - name: ClickHouse traces + type: grafana-clickhouse-datasource + uid: clickhouse-traces + access: proxy + editable: false + jsonData: + host: {{ .Values.clickhouse.clusterName }}.clickhouse.svc + port: 9000 + protocol: native + username: {{ .Values.clickhouse.user }} + defaultDatabase: {{ .Values.clickhouse.database }} + traces: + defaultDatabase: {{ .Values.clickhouse.database }} + defaultTable: otel_traces + otelEnabled: true + otelVersion: latest + tracesToLogs: + datasourceUid: clickhouse-logs + secureJsonData: + password: ${CLICKHOUSE_PASSWORD} + {{- end }} {{- if .Values.tempo.enabled }} - name: Tempo type: tempo diff --git a/values/proteus/manifests/workerdeployment.yaml b/values/proteus/manifests/workerdeployment.yaml index 30bdff94..cda766cc 100644 --- a/values/proteus/manifests/workerdeployment.yaml +++ b/values/proteus/manifests/workerdeployment.yaml @@ -1,7 +1,6 @@ # TEMPORAL_NAMESPACE : {env}-atlantis # ARCHIVE_PVC : the queue's archive PVC (prod+beta -> prod-queue, staging -> dev-queue) -# PROTEUS_EVENTS_QUEUE : {env}-proteus-job-events (queue the env's Atlantis binding reads) -# RABBITMQ_CONNSTRING secret : the broker that env's Atlantis consumes from (prod+beta share prod) +# Status/inbox/quota now flow via Temporal activities to the Atlantis notify worker; no RabbitMQ. # --- # apiVersion: temporal.io/v1alpha1 # kind: WorkerDeployment @@ -39,13 +38,6 @@ # value: prod # - name: ARCHIVE_PVC # value: prod-queue-ceph-archives -# - name: PROTEUS_EVENTS_QUEUE -# value: prod-proteus-job-events -# - name: RABBITMQ_CONNSTRING -# valueFrom: -# secretKeyRef: -# name: proteus-rabbitmq-prod -# key: connString # --- # apiVersion: temporal.io/v1alpha1 # kind: WorkerDeployment @@ -84,14 +76,6 @@ # # beta schedules into prod-queue (same as prod), so the same archive PVC. # - name: ARCHIVE_PVC # value: prod-queue-ceph-archives -# - name: PROTEUS_EVENTS_QUEUE -# value: beta-proteus-job-events -# # beta shares the prod broker (its Atlantis binding uses prod-atlantis-rabbitmq). -# - name: RABBITMQ_CONNSTRING -# valueFrom: -# secretKeyRef: -# name: proteus-rabbitmq-prod -# key: connString # --- apiVersion: temporal.io/v1alpha1 kind: WorkerDeployment @@ -129,10 +113,3 @@ spec: value: staging - name: ARCHIVE_PVC value: dev-queue-ceph-archives - - name: PROTEUS_EVENTS_QUEUE - value: staging-proteus-job-events - - name: RABBITMQ_CONNSTRING - valueFrom: - secretKeyRef: - name: proteus-rabbitmq-staging - key: connString