clickhouse: Add S3 cold tier and additional replicas

values/clickhouse/env-oceanbox.yaml.gotmpl

@@ -1,2 +1,4 @@
 clickhouse:
   enabled: true
+  s3:
+    enabled: true

values/clickhouse/env.yaml.gotmpl

@@ -3,10 +3,11 @@ clickhouse:
   autosync: true
   clusterName: clickhouse
   keeperName: clickhouse-keeper
-  # Headless client service the operator creates: <clusterName>-clickhouse-headless
   serviceName: clickhouse-clickhouse-headless
+  storageClass: ceph-rbd
   storage: 50Gi
   keeperStorage: 5Gi
+  keeperReplicas: 3
   resources:
     cpuRequest: "2"
     memoryRequest: 4Gi
@@ -15,11 +16,12 @@ clickhouse:
   user: default
   credentialsSecret: clickhouse-credentials
   credentialsKey: password
-  # Only used when S3-tiered storage is enabled in cluster.yaml.gotmpl (currently disabled).
   s3:
+    enabled: false
     endpoint: "http://10.255.241.30:30080"
     bucket: clickhouse
     secret: clickhouse-s3
+    moveFactor: 0.2
   ttl:
     logs: 720h
     traces: 336h

values/clickhouse/values/cluster.yaml.gotmpl

@@ -7,53 +7,59 @@ clickhouse:
   spec:
     shards: 1
     replicas: 1
+    podTemplate:
+      nodeHostnameKey: kubernetes.io/hostname
     settings:
       defaultUserPassword:
         secret:
           name: {{ .Values.clickhouse.credentialsSecret }}
           key: {{ .Values.clickhouse.credentialsKey }}
-      # S3-tiered storage (hot on local PVC, cold on TOS). Disabled for now — re-enable
-      # together with the containerTemplate.env block below and the s3 block in env.yaml.gotmpl.
-      # extraConfig:
-      #   storage_configuration:
-      #     disks:
-      #       s3:
-      #         type: s3
-      #         endpoint: {{ .Values.clickhouse.s3.endpoint }}/{{ .Values.clickhouse.s3.bucket }}/data/
-      #         use_environment_credentials: true
-      #     policies:
-      #       tiered:
-      #         volumes:
-      #           hot:
-      #             disk: default
-      #           cold:
-      #             disk: s3
-      #         move_factor: 0.2
-      #   merge_tree:
-      #     storage_policy: tiered
+      # Single replica: no cross-replica database sync needed. The operator's sync also
+      # breaks on the OTel-created (non-replicated) `otel` database. Revisit for multi-replica HA.
+      enableDatabaseSync: false
+      {{- if .Values.clickhouse.s3.enabled }}
+      extraConfig:
+        storage_configuration:
+          disks:
+            s3:
+              type: s3
+              endpoint: {{ .Values.clickhouse.s3.endpoint }}/{{ .Values.clickhouse.s3.bucket }}/data/
+              use_environment_credentials: true
+          policies:
+            tiered:
+              volumes:
+                hot:
+                  disk: default
+                cold:
+                  disk: s3
+              move_factor: {{ .Values.clickhouse.s3.moveFactor }}
+        merge_tree:
+          storage_policy: tiered
+      {{- end }}
     containerTemplate:
-      # Guarantee CPU via requests; no CPU limit so ClickHouse can burst across cores
-      # (a CPU limit throttles merges/queries).
       resources:
         requests:
           cpu: {{ .Values.clickhouse.resources.cpuRequest }}
           memory: {{ .Values.clickhouse.resources.memoryRequest }}
         limits:
           memory: {{ .Values.clickhouse.resources.memoryLimit }}
-      # env:  # S3 tiering credentials — re-enable with the storage_configuration block above.
-      #   - name: AWS_ACCESS_KEY_ID
-      #     valueFrom:
-      #       secretKeyRef:
-      #         name: {{ .Values.clickhouse.s3.secret }}
-      #         key: AWS_ACCESS_KEY_ID
-      #   - name: AWS_SECRET_ACCESS_KEY
-      #     valueFrom:
-      #       secretKeyRef:
-      #         name: {{ .Values.clickhouse.s3.secret }}
-      #         key: AWS_ACCESS_KEY_SECRET
+      {{- if .Values.clickhouse.s3.enabled }}
+      env:
+      - name: AWS_ACCESS_KEY_ID
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.clickhouse.s3.secret }}
+            key: AWS_ACCESS_KEY_ID
+      - name: AWS_SECRET_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.clickhouse.s3.secret }}
+            key: AWS_SECRET_ACCESS_KEY
+      {{- end }}
     dataVolumeClaimSpec:
       accessModes:
         - ReadWriteOnce
+      storageClassName: {{ .Values.clickhouse.storageClass }}
       resources:
         requests:
           storage: {{ .Values.clickhouse.storage }}
@@ -63,10 +69,14 @@ keeper:
   meta:
     name: {{ .Values.clickhouse.keeperName }}
   spec:
-    replicas: 1
+    replicas: {{ .Values.clickhouse.keeperReplicas }}
+    podTemplate:
+    podDisruptionBudget:
+      maxUnavailable: 1
     dataVolumeClaimSpec:
       accessModes:
         - ReadWriteOnce
+      storageClassName: {{ .Values.clickhouse.storageClass }}
       resources:
         requests:
           storage: {{ .Values.clickhouse.keeperStorage }}