ingest: Add Worker for CronJobs

helmfile.d/ingest.yaml.gotmpl

@@ -0,0 +1,27 @@
+bases:
+ - ../envs/environments.yaml.gotmpl
+
+commonLabels:
+  tier: system
+
+releases:
+- name: manifests
+  namespace: cron
+  chart: manifests
+  condition: ingest.enabled
+  missingFileHandler: Info
+  values:
+  - ../values/env.yaml
+  - ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
+  - ../values/ingest/env.yaml.gotmpl
+  - ../values/ingest/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
+  hooks:
+  - events: [ prepare, cleanup ]
+    showlogs: true
+    command: ../bin/helmify
+    args:
+    - '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
+    - '{{`{{ .Release.Chart }}`}}'
+    - '{{`{{ .Environment.Name }}`}}'
+    - ../values/ingest/manifests
+    - manifests

values/ingest/env-ekman.yaml.gotmpl

@@ -0,0 +1,2 @@
+ingest:
+  enabled: true

values/ingest/env.yaml.gotmpl

@@ -0,0 +1,3 @@
+ingest:
+  enabled: false
+  autosync: false

values/ingest/manifests/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingest
+  namespace: ingest
+  labels:
+    app.kubernetes.io/name: ingest
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingest
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingest
+    spec:
+      # NOTE: Run as the data uid so downloaded files are owned correctly
+      securityContext:
+        runAsUser: 5000
+        runAsGroup: 5000
+        fsGroup: 5000
+      containers:
+        - name: ingest
+          image: git.oceanbox.io/oceanbox/churn/ingest:0.1.0
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: TEMPORAL_ADDRESS
+              value: temporal-frontend.temporal:7233
+            - name: TEMPORAL_NAMESPACE
+              value: prod-ingest
+            - name: TEMPORAL_TASK_QUEUE
+              value: ingest
+            - name: DATA_ROOT
+              value: /data/hdd/data
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: ekman-data

values/ingest/manifests/ingest.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.clusterConfig.argo.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingest
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  destination:
+    namespace: ingest
+    server: 'https://kubernetes.default.svc'
+  sources:
+  - repoURL: {{ .Values.clusterConfig.manifests }}
+    targetRevision: HEAD
+    path: helmfile.d
+    plugin:
+      name: helmfile-cmp
+      env:
+      - name: CLUSTER_NAME
+        value: {{ .Values.clusterConfig.cluster }}
+      - name: HELMFILE_ENVIRONMENT
+        value: default
+      - name: HELMFILE_FILE_PATH
+        value: ingest.yaml.gotmpl
+  project: sys
+  syncPolicy:
+    managedNamespaceMetadata:
+      labels:
+        component: sys
+    syncOptions:
+      - CreateNamespace=true
+      - ApplyOutOfSyncOnly=true
+    {{- if .Values.ingest.autosync }}
+    automated:
+      prune: true
+    {{- end }}
+{{- end }}

values/ingest/manifests/pvc.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-ingest-data
+spec:
+  accessModes:
+  - ReadWriteMany
+  capacity:
+    storage: 1Gi
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: ekman-data
+    namespace: ingest
+  csi:
+    driver: rook-ceph.cephfs.csi.ceph.com
+    nodeStageSecretRef:
+      name: rook-csi-cephfs-node
+      namespace: rook-ceph
+    volumeAttributes:
+      clusterID: rook-ceph
+      fsName: data
+      rootPath: /
+      staticVolume: "true"
+    volumeHandle: pv-ingest-data
+  persistentVolumeReclaimPolicy: Retain
+  volumeMode: Filesystem
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ekman-data
+  namespace: ingest
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: ""
+  volumeMode: Filesystem
+  volumeName: pv-ingest-data

values/temporal/env-ekman.yaml.gotmpl

@@ -1,5 +1,4 @@
 temporal:
-  # NOTE: Re-enable for the fapr migration
-  enabled: false
+  enabled: true
   autosync: false
-  ingress: false
+  ingress: true