Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 mrtz
|
ae01e69fc2
|
@@ -1,3 +0,0 @@
|
||||
FROM busybox
|
||||
|
||||
COPY keycloak-themes/oceanbox /theme
|
||||
@@ -1,47 +0,0 @@
|
||||
/* Oceanbox Keycloak Login Theme */
|
||||
|
||||
/* Brand colours */
|
||||
:root {
|
||||
--pf-v5-global--primary-color--100: #0bb4aa;
|
||||
--pf-v5-global--primary-color--200: #099e95;
|
||||
--pf-v5-global--link--Color: #0bb4aa;
|
||||
--pf-v5-global--link--Color--hover: #031275;
|
||||
}
|
||||
|
||||
/* Background */
|
||||
.login-pf body {
|
||||
background: #f9fafd url("../img/oceanbox-bg.png") no-repeat center bottom fixed;
|
||||
background-size: cover;
|
||||
}
|
||||
|
||||
/* Logo */
|
||||
div.kc-logo-text {
|
||||
background-image: url('../img/oceanbox-logo-text.png');
|
||||
height: 80px;
|
||||
width: 360px;
|
||||
background-repeat: no-repeat;
|
||||
background-size: contain;
|
||||
background-position: center;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
div.kc-logo-text span {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Primary button */
|
||||
.pf-v5-c-button.pf-m-primary {
|
||||
--pf-v5-c-button--m-primary--BackgroundColor: #0bb4aa;
|
||||
--pf-v5-c-button--m-primary--hover--BackgroundColor: #099e95;
|
||||
--pf-v5-c-button--m-primary--active--BackgroundColor: #37746F;
|
||||
--pf-v5-c-button--m-primary--focus--BackgroundColor: #099e95;
|
||||
}
|
||||
|
||||
/* Links */
|
||||
a, .pf-v5-c-button.pf-m-link {
|
||||
color: #0bb4aa;
|
||||
}
|
||||
|
||||
a:hover, .pf-v5-c-button.pf-m-link:hover {
|
||||
color: #031275;
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 MiB |
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
@@ -1,5 +0,0 @@
|
||||
parent=keycloak.v2
|
||||
import=common/keycloak
|
||||
|
||||
stylesCommon=vendor/patternfly-v5/patternfly.min.css vendor/patternfly-v5/patternfly-addons.css
|
||||
styles=css/styles.css css/oceanbox.css
|
||||
@@ -4,10 +4,10 @@ description: Atlantis map and simulation service
|
||||
type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
version: v1.42.25
|
||||
version: v1.46.5
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application.
|
||||
appVersion: v1.42.25
|
||||
appVersion: v1.46.5
|
||||
dependencies:
|
||||
- name: diagrid-dashboard
|
||||
version: "0.1.0"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: git.oceanbox.io/oceanbox/poseidon/atlantis
|
||||
tag: v1.42.25
|
||||
tag: v1.46.5
|
||||
pullPolicy: IfNotPresent
|
||||
init:
|
||||
enabled: false
|
||||
|
||||
@@ -13,9 +13,9 @@ type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: v2.9.4
|
||||
version: v1.46.5
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "v2.9.4"
|
||||
appVersion: "v1.46.5"
|
||||
|
||||
@@ -10,7 +10,7 @@ image:
|
||||
# This sets the pull policy for images.
|
||||
pullPolicy: IfNotPresent
|
||||
# Overrides the image tag whose default is the chart appVersion.
|
||||
tag: v2.9.4
|
||||
tag: v1.46.5
|
||||
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||
imagePullSecrets:
|
||||
- name: gitlab-pull-secret
|
||||
|
||||
Vendored
+46
@@ -0,0 +1,46 @@
|
||||
{{- if .Values.httpRoute.enabled -}}
|
||||
{{- $fullName := include "docs.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
labels:
|
||||
{{- include "docs.labels" . | nindent 4 }}
|
||||
spec:
|
||||
parentRefs:
|
||||
{{- toYaml .Values.httpRoute.parentRefs | nindent 4 }}
|
||||
{{- with .Values.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.httpRoute.rules }}
|
||||
- {{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
backendRefs:
|
||||
- name: {{ $fullName }}
|
||||
port: {{ $svcPort }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: allow-gateway-to-{{ $fullName }}
|
||||
labels:
|
||||
{{- include "docs.labels" . | nindent 4 }}
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
{{- include "docs.selectorLabels" . | nindent 6 }}
|
||||
ingress:
|
||||
- fromCIDRSet:
|
||||
{{- range .Values.clusterConfig.ingress_whitelist }}
|
||||
- cidr: {{ . }}
|
||||
{{- end }}
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
@@ -46,8 +46,13 @@ service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
ingress:
|
||||
enabled: true
|
||||
className: haproxy
|
||||
enabled: false
|
||||
className: nginx
|
||||
httpRoute:
|
||||
enabled: false
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
rules: []
|
||||
persistence:
|
||||
enabled: false
|
||||
size: 1G
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: fapr
|
||||
description: A Helm chart for Fapr (F# Dapr workflow orchestrator)
|
||||
type: application
|
||||
version: v0.1.0
|
||||
appVersion: v0.1.0
|
||||
@@ -1,61 +0,0 @@
|
||||
{{/* vim: set filetype=mustache: */}}
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "fapr.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
*/}}
|
||||
{{- define "fapr.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "fapr.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "fapr.labels" -}}
|
||||
helm.sh/chart: {{ include "fapr.chart" . }}
|
||||
{{ include "fapr.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "fapr.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "fapr.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "fapr.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{ default (include "fapr.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccount.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -1,85 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "fapr.fullname" . }}
|
||||
labels:
|
||||
{{- include "fapr.labels" . | nindent 4 }}
|
||||
spec:
|
||||
{{- if not .Values.autoscaling.enabled }}
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fapr.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
{{- with .Values.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fapr.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "fapr.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.service.port }}
|
||||
protocol: TCP
|
||||
env:
|
||||
{{- toYaml .Values.env | nindent 12 }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: appsettings
|
||||
mountPath: /app/appsettings.json
|
||||
subPath: appsettings.json
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: data
|
||||
{{- if .Values.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.persistence.existingClaim | default (include "fapr.fullname" .) }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
- name: appsettings
|
||||
configMap:
|
||||
name: {{ include "fapr.fullname" . }}-appsettings
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
@@ -1,21 +0,0 @@
|
||||
{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ include "fapr.fullname" . }}
|
||||
labels:
|
||||
{{ include "fapr.labels" . | indent 4 }}
|
||||
spec:
|
||||
accessModes:
|
||||
- {{ .Values.persistence.accessMode | quote }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size | quote }}
|
||||
{{- if .Values.persistence.storageClass }}
|
||||
{{- if (eq "-" .Values.persistence.storageClass) }}
|
||||
storageClassName: ""
|
||||
{{- else }}
|
||||
storageClassName: "{{ .Values.persistence.storageClass }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,49 +0,0 @@
|
||||
{{- if .Values.redis.enabled -}}
|
||||
apiVersion: dragonflydb.io/v1alpha1
|
||||
kind: Dragonfly
|
||||
metadata:
|
||||
name: {{ include "fapr.fullname" . }}-redis
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
linkerd.io/inject: disabled
|
||||
labels:
|
||||
app.kubernetes.io/created-by: dragonfly-operator
|
||||
{{- include "fapr.labels" . | nindent 4 }}
|
||||
spec:
|
||||
args:
|
||||
- --dbfilename=dump
|
||||
- --maxmemory=$(MAX_MEMORY)Mi
|
||||
- --proactor_threads=1
|
||||
- --logtostderr
|
||||
- --save_schedule=
|
||||
env:
|
||||
- name: MAX_MEMORY
|
||||
valueFrom:
|
||||
resourceFieldRef:
|
||||
resource: limits.memory
|
||||
divisor: 1Mi
|
||||
replicas: {{ .Values.redis.replicas | default "1" }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.redis.resources.cpu | default "100m" }}
|
||||
memory: {{ .Values.redis.resources.memory | default "128Mi" }}
|
||||
limits:
|
||||
memory: {{ .Values.redis.resources.memory | default "128Mi" }}
|
||||
authentication:
|
||||
passwordFromSecret:
|
||||
name: {{ .Values.redis.secret.name | default (printf "%s-redis" (include "fapr.fullname" .)) | quote }}
|
||||
key: {{ .Values.redis.secret.key | quote }}
|
||||
{{- if .Values.redis.backup.enabled }}
|
||||
snapshot:
|
||||
dir: /data
|
||||
cron: "0 3 * * *"
|
||||
enableOnMasterOnly: false
|
||||
persistentVolumeClaimSpec:
|
||||
storageClassName: {{ .Values.redis.storageClass | default "managed-nfs-storage" }}
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.redis.size | default "1Gi" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,15 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "fapr.fullname" . }}
|
||||
labels:
|
||||
{{- include "fapr.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "fapr.selectorLabels" . | nindent 4 }}
|
||||
@@ -1,12 +0,0 @@
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "fapr.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "fapr.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,67 +0,0 @@
|
||||
# Default values for fapr.
|
||||
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: git.oceanbox.io/oceanbox/fapr/fapr
|
||||
tag: v0.1.0
|
||||
pullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: APP_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: APP_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
imagePullSecrets:
|
||||
- name: gitlab-pull-secret
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
name: ""
|
||||
podAnnotations: {}
|
||||
podSecurityContext:
|
||||
fsGroup: 0
|
||||
securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: false
|
||||
runAsNonRoot: false
|
||||
runAsUser: 0
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
persistence:
|
||||
enabled: true
|
||||
existingClaim: ""
|
||||
size: 1Gi
|
||||
storageClass: "ceph-rbd"
|
||||
accessMode: ReadWriteMany
|
||||
redis:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
backup:
|
||||
enabled: false
|
||||
size: 1Gi
|
||||
storageClass: "ceph-rbd"
|
||||
secret:
|
||||
name: ""
|
||||
key: "redis-password"
|
||||
resources:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
resources: {}
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 3
|
||||
targetCPUUtilizationPercentage: 80
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
@@ -0,0 +1,46 @@
|
||||
{{- if .Values.httpRoute.enabled -}}
|
||||
{{- $fullName := include "makai.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
labels:
|
||||
{{- include "makai.labels" . | nindent 4 }}
|
||||
spec:
|
||||
parentRefs:
|
||||
{{- toYaml .Values.httpRoute.parentRefs | nindent 4 }}
|
||||
{{- with .Values.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.httpRoute.rules }}
|
||||
- {{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
backendRefs:
|
||||
- name: {{ $fullName }}
|
||||
port: {{ $svcPort }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: allow-gateway-to-{{ $fullName }}
|
||||
labels:
|
||||
{{- include "makai.labels" . | nindent 4 }}
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
{{- include "makai.selectorLabels" . | nindent 6 }}
|
||||
ingress:
|
||||
- fromCIDRSet:
|
||||
{{- range .Values.clusterConfig.ingress_whitelist }}
|
||||
- cidr: {{ . }}
|
||||
{{- end }}
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
@@ -46,8 +46,13 @@ service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
ingress:
|
||||
enabled: true
|
||||
className: haproxy
|
||||
enabled: false
|
||||
className: nginx
|
||||
httpRoute:
|
||||
enabled: false
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
rules: []
|
||||
persistence:
|
||||
enabled: false
|
||||
size: 1G
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
dependencies:
|
||||
- name: diagrid-dashboard
|
||||
repository: file://../diagrid-dashboard
|
||||
version: 0.1.0
|
||||
digest: sha256:4fdb3148a2a6439223d7844a3083da2de324dd47e5cb3ac4a5d9c436e6e2c775
|
||||
generated: "2026-02-25T16:15:48.608231856+01:00"
|
||||
@@ -4,10 +4,10 @@ description: A Helm chart for Kubernetes
|
||||
type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
version: v1.7.1
|
||||
version: v1.6.13
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application.
|
||||
appVersion: v1.7.1
|
||||
appVersion: v1.6.13
|
||||
dependencies:
|
||||
- name: diagrid-dashboard
|
||||
version: "0.1.0"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: git.oceanbox.io/oceanbox/plume/plume
|
||||
tag: v1.7.1
|
||||
tag: v1.6.13
|
||||
pullPolicy: IfNotPresent
|
||||
init:
|
||||
enabled: false
|
||||
@@ -90,5 +90,6 @@ serviceMonitor:
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
||||
diagrid-dashboard:
|
||||
enabled: false
|
||||
|
||||
@@ -4,10 +4,10 @@ description: A Helm chart for Kubernetes
|
||||
type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
version: v1.42.25
|
||||
version: v1.46.5
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application.
|
||||
appVersion: v1.42.25
|
||||
appVersion: v1.46.5
|
||||
dependencies:
|
||||
- name: diagrid-dashboard
|
||||
version: "0.1.0"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: git.oceanbox.io/oceanbox/poseidon/sorcerer
|
||||
tag: v1.42.25
|
||||
tag: v1.46.5
|
||||
pullPolicy: IfNotPresent
|
||||
init:
|
||||
enabled: false
|
||||
|
||||
@@ -15,7 +15,7 @@ releases:
|
||||
- name: argocd
|
||||
namespace: argocd
|
||||
chart: argo/argo-cd
|
||||
version: 9.5.15
|
||||
version: 9.4.10
|
||||
condition: argo.enabled
|
||||
values:
|
||||
- ../values/argo/values/argocd.yaml.gotmpl
|
||||
@@ -27,7 +27,7 @@ releases:
|
||||
- name: argocd-apps
|
||||
namespace: argocd
|
||||
chart: argo/argocd-apps
|
||||
version: 2.0.5
|
||||
version: 2.0.4
|
||||
condition: argo.apps.enabled
|
||||
values:
|
||||
- ../values/argo/values/apps.yaml.gotmpl
|
||||
@@ -35,7 +35,7 @@ releases:
|
||||
- name: argo-rollouts
|
||||
namespace: argocd
|
||||
chart: argo/argo-rollouts
|
||||
version: 2.40.9
|
||||
version: 2.40.6
|
||||
condition: argo.rollouts.enabled
|
||||
values:
|
||||
- ../values/argo/values/rollouts.yaml.gotmpl
|
||||
@@ -43,7 +43,7 @@ releases:
|
||||
- name: argo-workflows
|
||||
namespace: argocd
|
||||
chart: argo/argo-workflows
|
||||
version: 1.0.14
|
||||
version: 0.47.5
|
||||
condition: argo.workflows.enabled
|
||||
missingFileHandler: Info
|
||||
- name: manifests
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
# yaml-language-server: $schema=https://www.schemastore.org/helmfile.json
|
||||
|
||||
bases:
|
||||
- ../envs/environments.yaml.gotmpl
|
||||
|
||||
repositories:
|
||||
- name: catalyst
|
||||
oci: true
|
||||
url: 'public.ecr.aws/diagrid'
|
||||
|
||||
commonLabels:
|
||||
tier: system
|
||||
|
||||
releases:
|
||||
- name: catalyst
|
||||
namespace: cra-agent
|
||||
chart: catalyst/catalyst
|
||||
version: 1.47.0
|
||||
condition: catalyst.enabled
|
||||
values:
|
||||
- ../values/catalyst/values/values.yaml
|
||||
- ../values/catalyst/values/values-{{ .Environment.Name }}.yaml
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
- ../values/catalyst/kustomize/{{ .Environment.Name }}
|
||||
missingFileHandler: Info
|
||||
- name: manifests
|
||||
namespace: cra-agent
|
||||
chart: manifests
|
||||
condition: catalyst.enabled
|
||||
missingFileHandler: Info
|
||||
values:
|
||||
- ../values/env.yaml
|
||||
- ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
|
||||
- ../values/catalyst/env.yaml.gotmpl
|
||||
- ../values/catalyst/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
|
||||
hooks:
|
||||
- events: [ prepare, cleanup ]
|
||||
showlogs: true
|
||||
command: ../bin/helmify
|
||||
args:
|
||||
- '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
|
||||
- '{{`{{ .Release.Chart }}`}}'
|
||||
- '{{`{{ .Environment.Name }}`}}'
|
||||
- ../values/catalyst/manifests
|
||||
- manifests
|
||||
@@ -13,7 +13,7 @@ releases:
|
||||
- name: cert-manager
|
||||
namespace: cert-manager
|
||||
chart: cert-manager/cert-manager
|
||||
version: v1.20.2
|
||||
version: v1.19.4
|
||||
condition: cert_manager.enabled
|
||||
values:
|
||||
- ../values/cert-manager/values/cert-manager.yaml.gotmpl
|
||||
|
||||
@@ -11,6 +11,7 @@ releases:
|
||||
condition: docs.enabled
|
||||
values:
|
||||
- ../values/docs/values/values.yaml
|
||||
- ../values/docs/values/values.yaml.gotmpl
|
||||
- ../values/docs/values/values-{{ .Environment.Name }}.yaml
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
|
||||
@@ -13,7 +13,7 @@ releases:
|
||||
- name: dragonfly
|
||||
namespace: dragonfly
|
||||
chart: dragonfly/dragonfly-operator
|
||||
version: v1.5.0
|
||||
version: v1.4.0
|
||||
condition: dragonfly.enabled
|
||||
values:
|
||||
- ../values/dragonfly/values/dragonfly.yaml.gotmpl
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
bases:
|
||||
- ../envs/environments.yaml.gotmpl
|
||||
|
||||
commonLabels:
|
||||
tier: oceanbox
|
||||
|
||||
releases:
|
||||
- name: {{ .Environment.Name }}-fapr
|
||||
namespace: {{ .Environment.Name }}-fapr
|
||||
chart: ../charts/fapr
|
||||
condition: fapr.enabled
|
||||
values:
|
||||
- ../values/fapr/values/values.yaml
|
||||
- ../values/fapr/values/values-{{ .Environment.Name }}.yaml
|
||||
- ../values/fapr/values/values-{{ .Environment.Name }}-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
- ../values/fapr/kustomize/{{ .Environment.Name }}-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}
|
||||
missingFileHandler: Info
|
||||
- name: manifests
|
||||
namespace: {{ .Environment.Name }}-fapr
|
||||
chart: manifests
|
||||
condition: fapr.enabled
|
||||
missingFileHandler: Info
|
||||
values:
|
||||
- ../values/env.yaml
|
||||
- ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
|
||||
- ../values/fapr/env.yaml.gotmpl
|
||||
- ../values/fapr/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
|
||||
hooks:
|
||||
- events: [ prepare, cleanup ]
|
||||
showlogs: true
|
||||
command: ../bin/helmify
|
||||
args:
|
||||
- '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
|
||||
- '{{`{{ .Release.Chart }}`}}'
|
||||
- '{{`{{ .Environment.Name }}`}}'
|
||||
- ../values/fapr/manifests
|
||||
- manifests
|
||||
@@ -13,10 +13,11 @@ releases:
|
||||
- name: gitea
|
||||
namespace: gitea
|
||||
chart: gitea/gitea
|
||||
version: 12.6.0
|
||||
version: 12.5.0
|
||||
condition: gitea.enabled
|
||||
values:
|
||||
- ../values/gitea/values/values.yaml
|
||||
- ../values/gitea/values/values.yaml.gotmpl
|
||||
- ../values/gitea/values/values-{{ .Environment.Name }}.yaml
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
bases:
|
||||
- ../envs/environments.yaml.gotmpl
|
||||
|
||||
repositories:
|
||||
- name: haproxytech
|
||||
oci: true
|
||||
url: 'ghcr.io/haproxytech/helm-charts'
|
||||
|
||||
commonLabels:
|
||||
tier: system
|
||||
|
||||
releases:
|
||||
- name: ingress-haproxy
|
||||
namespace: ingress-haproxy
|
||||
chart: haproxytech/kubernetes-ingress
|
||||
version: 1.42.0
|
||||
condition: haproxy.enabled
|
||||
values:
|
||||
- ../values/ingress-haproxy/values/ingress-haproxy.yaml.gotmpl
|
||||
- ../values/ingress-haproxy/values/ingress-haproxy-{{ .Environment.Name }}.yaml.gotmpl
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
- ../values/ingress-haproxy/kustomize/{{ .Environment.Name }}
|
||||
missingFileHandler: Info
|
||||
- name: manifests
|
||||
namespace: ingress-haproxy
|
||||
chart: manifests
|
||||
condition: haproxy.enabled
|
||||
missingFileHandler: Info
|
||||
values:
|
||||
- ../values/env.yaml
|
||||
- ../values/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml
|
||||
- ../values/ingress-haproxy/env.yaml.gotmpl
|
||||
- ../values/ingress-haproxy/env-{{ requiredEnv "ARGOCD_ENV_CLUSTER_NAME" }}.yaml.gotmpl
|
||||
hooks:
|
||||
- events: [ prepare, cleanup ]
|
||||
showlogs: true
|
||||
command: ../bin/helmify
|
||||
args:
|
||||
- '{{`{{ if eq .Event.Name "prepare" }}build{{ else }}clean{{ end }}`}}'
|
||||
- '{{`{{ .Release.Chart }}`}}'
|
||||
- '{{`{{ .Environment.Name }}`}}'
|
||||
- ../values/ingress-haproxy/manifests
|
||||
- manifests
|
||||
@@ -12,7 +12,7 @@ releases:
|
||||
- name: ingress-nginx
|
||||
namespace: ingress-nginx
|
||||
chart: ingress-nginx/ingress-nginx
|
||||
version: 4.15.1
|
||||
version: 4.14.3
|
||||
condition: nginx.enabled
|
||||
values:
|
||||
- ../values/ingress-nginx/values/ingress-nginx.yaml.gotmpl
|
||||
|
||||
@@ -12,7 +12,7 @@ releases:
|
||||
- name: {{ .Environment.Name }}-keycloak
|
||||
namespace: keycloak
|
||||
chart: bitnami/keycloak
|
||||
version: 25.2.0
|
||||
version: 24.9.0
|
||||
condition: keycloak.enabled
|
||||
values:
|
||||
- ../values/keycloak/values/values.yaml
|
||||
|
||||
@@ -8,7 +8,7 @@ releases:
|
||||
- name: kueue
|
||||
namespace: kueue-system
|
||||
chart: oci://registry.k8s.io/kueue/charts/kueue
|
||||
version: 0.17.1
|
||||
version: 0.15.0
|
||||
condition: kueue.enabled
|
||||
values:
|
||||
- ../values/kueue/values/values.yaml
|
||||
|
||||
@@ -15,7 +15,7 @@ releases:
|
||||
- name: kyverno
|
||||
namespace: kyverno
|
||||
chart: kyverno/kyverno
|
||||
version: 3.8.1
|
||||
version: 3.7.1
|
||||
condition: kyverno.enabled
|
||||
values:
|
||||
- ../values/kyverno/values/kyverno.yaml.gotmpl
|
||||
|
||||
@@ -12,7 +12,7 @@ releases:
|
||||
- name: loki
|
||||
namespace: loki
|
||||
chart: loki/loki
|
||||
version: 7.0.0
|
||||
version: 6.53.0
|
||||
condition: loki.enabled
|
||||
values:
|
||||
- ../values/loki/values/loki.yaml.gotmpl
|
||||
|
||||
@@ -11,6 +11,7 @@ releases:
|
||||
condition: makai.enabled
|
||||
values:
|
||||
- ../values/makai/values/values.yaml
|
||||
- ../values/makai/values/values.yaml.gotmpl
|
||||
- ../values/makai/values/values-{{ .Environment.Name }}.yaml
|
||||
postRenderer: ../bin/kustomizer
|
||||
postRendererArgs:
|
||||
|
||||
@@ -12,7 +12,7 @@ releases:
|
||||
- name: mariadb-operator
|
||||
namespace: mariadb-operator
|
||||
chart: mariadb-operator/mariadb-operator
|
||||
version: 26.3.0
|
||||
version: 25.10.4
|
||||
condition: mariadb_operator.enabled
|
||||
values:
|
||||
- ../values/mariadb-operator/values/mariadb-operator.yaml.gotmpl
|
||||
|
||||
@@ -16,7 +16,7 @@ releases:
|
||||
namespace: {{ .Environment.Name }}-openfga
|
||||
{{- end }}
|
||||
chart: openfga/openfga
|
||||
version: 0.3.5
|
||||
version: 0.2.55
|
||||
condition: openfga.enabled
|
||||
values:
|
||||
- ../values/openfga/values/values.yaml
|
||||
|
||||
@@ -12,7 +12,7 @@ releases:
|
||||
- name: opentelemetry-collector
|
||||
namespace: otel
|
||||
chart: open-telemetry/opentelemetry-collector
|
||||
version: 0.156.2
|
||||
version: 0.146.1
|
||||
condition: otel.enabled
|
||||
values:
|
||||
- ../values/opentelemetry-collector/values/values.yaml
|
||||
|
||||
@@ -15,7 +15,7 @@ releases:
|
||||
- name: postgres-operator
|
||||
namespace: cnpg
|
||||
chart: cloudnative-pg/cloudnative-pg
|
||||
version: 0.28.2
|
||||
version: 0.27.0
|
||||
condition: postgres_operator.enabled
|
||||
values:
|
||||
- ../values/postgres-operator/values/postgres-operator.yaml.gotmpl
|
||||
@@ -27,7 +27,7 @@ releases:
|
||||
- name: plugin-barman-cloud
|
||||
namespace: cnpg
|
||||
chart: cloudnative-pg/plugin-barman-cloud
|
||||
version: 0.6.0
|
||||
version: 0.5.0
|
||||
condition: postgres_operator.enabled
|
||||
values:
|
||||
- ../values/postgres-operator/values/plugin-barman-cloud.yaml.gotmpl
|
||||
|
||||
@@ -15,7 +15,7 @@ releases:
|
||||
- name: prometheus
|
||||
namespace: prometheus
|
||||
chart: prometheus/kube-prometheus-stack
|
||||
version: 85.3.0
|
||||
version: 82.10.1
|
||||
condition: prometheus.enabled
|
||||
values:
|
||||
- ../values/prometheus/values/prometheus.yaml.gotmpl
|
||||
|
||||
@@ -13,7 +13,7 @@ releases:
|
||||
- name: slurm-operator
|
||||
namespace: slinky
|
||||
chart: slurm-operator/slurm-operator
|
||||
version: 1.0.3
|
||||
version: 1.0.2
|
||||
condition: slurm_operator.enabled
|
||||
values:
|
||||
- ../values/slurm-operator/values/slurm-operator.yaml.gotmpl
|
||||
|
||||
@@ -13,7 +13,7 @@ releases:
|
||||
- name: slurm
|
||||
namespace: slurm
|
||||
chart: slurm/slurm
|
||||
version: 1.0.3
|
||||
version: 1.0.2
|
||||
condition: slurm.enabled
|
||||
values:
|
||||
- ../values/slurm/values/slurm.yaml.gotmpl
|
||||
|
||||
@@ -14,7 +14,7 @@ releases:
|
||||
- name: umami
|
||||
namespace: analytics
|
||||
chart: umami/umami
|
||||
version: 7.9.4
|
||||
version: 7.7.2
|
||||
condition: umami.enabled
|
||||
values:
|
||||
- ../values/umami/values/values.yaml
|
||||
|
||||
@@ -15,7 +15,7 @@ releases:
|
||||
- name: velero
|
||||
namespace: velero
|
||||
chart: velero/velero
|
||||
version: 12.0.1
|
||||
version: 11.4.0
|
||||
condition: velero.enabled
|
||||
values:
|
||||
- ../values/velero/values/velero.yaml.gotmpl
|
||||
|
||||
@@ -3,8 +3,7 @@ bases:
|
||||
|
||||
repositories:
|
||||
- name: x509-exporter
|
||||
oci: true
|
||||
url: 'quay.io/enix/charts'
|
||||
url: 'https://charts.enix.io'
|
||||
|
||||
commonLabels:
|
||||
tier: sys
|
||||
@@ -13,7 +12,7 @@ releases:
|
||||
- name: x509-exporter
|
||||
namespace: x509-exporter
|
||||
chart: x509-exporter/x509-certificate-exporter
|
||||
version: 4.1.0
|
||||
version: 3.19.1
|
||||
condition: x509_exporter.enabled
|
||||
values:
|
||||
- ../values/x509-exporter/values/x509-exporter.yaml.gotmpl
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
buildGoModule,
|
||||
fetchFromGitHub,
|
||||
}:
|
||||
buildGoModule rec {
|
||||
pname = "kueuectl";
|
||||
version = "0.16.3";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "kubernetes-sigs";
|
||||
repo = "kueue";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-JbU+ZoQ+YriaiIbbVCe45OTYycxYRanLhmQAdpE+xQ4=";
|
||||
};
|
||||
|
||||
vendorHash = null;
|
||||
|
||||
subPackages = [ "cmd/kueuectl" ];
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ treefmt.evalModule pkgs {
|
||||
# --- Nix formatting ---
|
||||
nixfmt = {
|
||||
enable = true;
|
||||
package = pkgs.nixfmt;
|
||||
package = pkgs.nixfmt-rfc-style;
|
||||
};
|
||||
statix.enable = true;
|
||||
deadnix.enable = true;
|
||||
|
||||
+18
-5
@@ -1,10 +1,23 @@
|
||||
{
|
||||
"pins": {
|
||||
"git-hooks": {
|
||||
"type": "Git",
|
||||
"repository": {
|
||||
"type": "GitHub",
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix"
|
||||
},
|
||||
"branch": "master",
|
||||
"submodules": false,
|
||||
"revision": "a1ef738813b15cf8ec759bdff5761b027e3e1d23",
|
||||
"url": "https://github.com/cachix/git-hooks.nix/archive/a1ef738813b15cf8ec759bdff5761b027e3e1d23.tar.gz",
|
||||
"hash": "sha256-Efs3VUPelRduf3PpfPP2ovEB4CXT7vHf8W+xc49RL/U="
|
||||
},
|
||||
"nixpkgs": {
|
||||
"type": "Channel",
|
||||
"name": "nixpkgs-unstable",
|
||||
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre981196.b86751bc4085/nixexprs.tar.xz",
|
||||
"hash": "sha256-mBqzkn7oJti2hqeO8iTbDxKw+1ifxpP53feQ0CEXies="
|
||||
"url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre930822.ed142ab1b3a0/nixexprs.tar.xz",
|
||||
"hash": "sha256-XH6awru9NnBc/m+2YhRNT8r1PAKEiPGF3gs//F3ods0="
|
||||
},
|
||||
"treefmt-nix": {
|
||||
"type": "Git",
|
||||
@@ -15,9 +28,9 @@
|
||||
},
|
||||
"branch": "main",
|
||||
"submodules": false,
|
||||
"revision": "790751ff7fd3801feeaf96d7dc416a8d581265ba",
|
||||
"url": "https://github.com/numtide/treefmt-nix/archive/790751ff7fd3801feeaf96d7dc416a8d581265ba.tar.gz",
|
||||
"hash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0="
|
||||
"revision": "337a4fe074be1042a35086f15481d763b8ddc0e7",
|
||||
"url": "https://github.com/numtide/treefmt-nix/archive/337a4fe074be1042a35086f15481d763b8ddc0e7.tar.gz",
|
||||
"hash": "sha256-wQ6NJSuFqAEmIg2VMnLdCnUc0b7vslUohqqGGD+Fyxk="
|
||||
}
|
||||
},
|
||||
"version": 7
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: arome-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.py: |
|
||||
import os
|
||||
import sys
|
||||
from netCDF4 import Dataset
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
fname ="https://thredds.met.no/thredds/dodsC/meps25epsarchive/YEAR/MONTH/DAY/meps_det_sfc_YEARMONTHDAYT00Z.ncml"
|
||||
outdir = "/data/hdd/data/AROME"
|
||||
|
||||
|
||||
def generate_thredds_names(start, stop):
|
||||
start_date = datetime(int(start.split("-")[0]),
|
||||
int(start.split("-")[1]),
|
||||
int(start.split("-")[2]))
|
||||
end_date = datetime(int(stop.split("-")[0]),
|
||||
int(stop.split("-")[1]),
|
||||
int(stop.split("-")[2]))
|
||||
|
||||
date_list = []
|
||||
while start_date <= end_date:
|
||||
date_list.append(start_date)
|
||||
start_date += timedelta(days=1)
|
||||
|
||||
fileList = []
|
||||
for date in date_list:
|
||||
y = str(date.year)
|
||||
m = (str(date.month)).zfill(2)
|
||||
d = (str(date.day)).zfill(2)
|
||||
|
||||
f = re.sub("YEAR", y, fname)
|
||||
f = re.sub("MONTH", m, f)
|
||||
f = re.sub("DAY", d, f)
|
||||
fileList.append(f)
|
||||
|
||||
return fileList
|
||||
|
||||
def copy_thredds_file(threddsFile, savename):
|
||||
dsin = Dataset(threddsFile)
|
||||
|
||||
dsout = Dataset(savename, "w")
|
||||
|
||||
for dname, the_dim in dsin.dimensions.items():
|
||||
dsout.createDimension(dname, len(the_dim) if not the_dim.isunlimited() else None)
|
||||
|
||||
aromeNames = ["time",
|
||||
"longitude",
|
||||
"latitude",
|
||||
"land_area_fraction",
|
||||
"air_temperature_2m",
|
||||
"precipitation_amount_acc",
|
||||
"water_evaporation_amount",
|
||||
"relative_humidity_2m",
|
||||
"integral_of_surface_downwelling_longwave_flux_in_air_wrt_time",
|
||||
"integral_of_surface_net_downward_shortwave_flux_wrt_time",
|
||||
"air_pressure_at_sea_level",
|
||||
"x_wind_10m",
|
||||
"y_wind_10m"]
|
||||
|
||||
for v_name, varin in dsin.variables.items():
|
||||
if v_name in aromeNames:
|
||||
fill_value = None
|
||||
|
||||
if hasattr(varin, "_FillValue"):
|
||||
fill_value = varin._FillValue
|
||||
|
||||
outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions, fill_value=fill_value)
|
||||
|
||||
outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs() if k not in ["_FillValue"]})
|
||||
|
||||
outVar[:] = varin[:]
|
||||
dsout.close()
|
||||
|
||||
|
||||
os.makedirs(outdir, exist_ok=True)
|
||||
|
||||
fList = generate_thredds_names("2026-04-24", datetime.today().strftime("%Y-%m-%d"))
|
||||
failed = False
|
||||
for fname in fList:
|
||||
savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc")
|
||||
if os.path.exists(savename):
|
||||
print(f"Skipping {savename}, already exists")
|
||||
continue
|
||||
print(savename)
|
||||
try:
|
||||
try:
|
||||
copy_thredds_file(fname, savename)
|
||||
except:
|
||||
alt_fname = re.sub("sfc", "2_5km", fname)
|
||||
alt_fname = re.sub("ncml", "nc", alt_fname)
|
||||
copy_thredds_file(alt_fname, savename)
|
||||
except Exception as e:
|
||||
print(f"File not found: {fname} ({e})")
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
sys.exit(1)
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: arome
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: 0 6 * * * # Everyday at 06:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid" # If only one at at time set to Allow else Forbid
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: juselius/busynix:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
if nix-shell -p 'python3.withPackages(ps: [ps.netcdf4])' --run 'python3 /scripts/download.py'; then
|
||||
chown -R 5000:5000 /data/hdd/data/AROME
|
||||
chmod -R g+w /data/hdd/data/AROME
|
||||
else
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
fi
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: arome-script
|
||||
defaultMode: 0755
|
||||
@@ -1,178 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: mur-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.py: |
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
parser = argparse.ArgumentParser(description="Download MUR SST files from NASA Earthdata")
|
||||
parser.add_argument("-sd", "--start_date", required=True, help="Start date (YYYY-MM-DD)")
|
||||
parser.add_argument("-ed", "--end_date", required=True, help="End date (YYYY-MM-DD)")
|
||||
parser.add_argument("-o", "--out_dir", default="MUR_SST_nc", help="Output directory")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def create_session():
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"User-Agent": "mur-sst-downloader",
|
||||
"Accept-Encoding": "identity"
|
||||
})
|
||||
return session
|
||||
|
||||
|
||||
def get_download_urls(startdate, enddate, verbose=False):
|
||||
url = (
|
||||
"https://cmr.earthdata.nasa.gov/search/granules.umm_json"
|
||||
f"?collection_concept_id=C1996881146-POCLOUD"
|
||||
f"&temporal={startdate}T00:00:00Z,{enddate}T00:00:00Z"
|
||||
"&pageSize=365"
|
||||
)
|
||||
|
||||
r = requests.get(url)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
urls = []
|
||||
for item in data["items"]:
|
||||
for link in item["umm"]["RelatedUrls"]:
|
||||
# Prefer direct HTTPS download links
|
||||
if link.get("Type") == "GET DATA":
|
||||
urls.append(link["URL"])
|
||||
|
||||
if verbose:
|
||||
print(f"Found {len(urls)} files")
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
def download_file(session, url, out_dir, verbose=False):
|
||||
filename = os.path.basename(url)
|
||||
local_path = os.path.join(out_dir, filename)
|
||||
|
||||
if os.path.exists(local_path):
|
||||
if verbose:
|
||||
print(f"Skipping existing: {filename}")
|
||||
return True
|
||||
|
||||
if verbose:
|
||||
print(f"Downloading: {filename}")
|
||||
|
||||
try:
|
||||
with session.get(url, stream=True, allow_redirects=True, timeout=60) as r:
|
||||
if r.status_code == 401:
|
||||
raise Exception("Unauthorized (check .netrc credentials)")
|
||||
|
||||
r.raise_for_status()
|
||||
|
||||
with open(local_path, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
if verbose:
|
||||
print(f"Saved: {filename}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Failed: {filename} -> {e}")
|
||||
return False
|
||||
|
||||
|
||||
def validate_dates(start, end):
|
||||
try:
|
||||
datetime.strptime(start, "%Y-%m-%d")
|
||||
datetime.strptime(end, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
print("Error: Dates must be in YYYY-MM-DD format")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
validate_dates(args.start_date, args.end_date)
|
||||
|
||||
# os.makedirs(args.out_dir, exist_ok=True)
|
||||
|
||||
session = create_session()
|
||||
|
||||
urls = get_download_urls(args.start_date, args.end_date, args.verbose)
|
||||
|
||||
failed = False
|
||||
for url in urls:
|
||||
if not download_file(session, url, args.out_dir, args.verbose):
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
sys.exit(1)
|
||||
print(f"\nDone. Downloaded files to: {args.out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: mur
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: "0 6 * * *" # Everyday at 06:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid"
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: juselius/busynix:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
nix-shell -p 'python3.withPackages(ps: [ps.requests])' coreutils --run '
|
||||
python3 /scripts/download.py \
|
||||
-sd $(date -d "3 days ago" +%Y-%m-%d) \
|
||||
-ed $(date +%Y-%m-%d) \
|
||||
-o /data/hdd/data/river-data/MUR/MUR_SST_nc \
|
||||
-v &&
|
||||
chown -R 5000:5000 /data/hdd/data/river-data/MUR/MUR_SST_nc &&
|
||||
chmod -R g+w /data/hdd/data/river-data/MUR/MUR_SST_nc
|
||||
' || {
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
}
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: netrc
|
||||
mountPath: /root/.netrc
|
||||
subPath: .netrc
|
||||
readOnly: true
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: mur-script
|
||||
defaultMode: 0755
|
||||
- name: netrc
|
||||
secret:
|
||||
secretName: mur-netrc
|
||||
@@ -1,245 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nemo-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.sh: |
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# this script downloads files from
|
||||
# https://data.marine.copernicus.eu/product/NWSHELF_ANALYSISFORECAST_PHY_004_013
|
||||
|
||||
set -euf -o pipefail
|
||||
|
||||
START_DATE="$1"
|
||||
END_DATE="$2"
|
||||
|
||||
current_date="$START_DATE"
|
||||
|
||||
while [[ "$current_date" < "$END_DATE" ]]; do
|
||||
next_date=$(date -I -d "$current_date + 1 day")
|
||||
echo "Running subset for $current_date to $next_date"
|
||||
|
||||
outfile="cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
|
||||
echo "Skipping salt (already exists)"
|
||||
else
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded salt"
|
||||
fi
|
||||
|
||||
outfile="cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
|
||||
echo "Skipping currents (already exists)"
|
||||
else
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded currents"
|
||||
fi
|
||||
|
||||
outfile="cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
|
||||
echo "Skipping temperature (already exists)"
|
||||
else
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded temperature"
|
||||
fi
|
||||
|
||||
outfile="cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i_${current_date}--${next_date}.nc"
|
||||
if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
|
||||
echo "Skipping ssh (already exists)"
|
||||
else
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded ssh"
|
||||
fi
|
||||
|
||||
current_date="$next_date"
|
||||
done
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nemo-nix
|
||||
namespace: cron
|
||||
data:
|
||||
shell.nix: |
|
||||
let
|
||||
nixpkgs = builtins.fetchTarball {
|
||||
url = "https://releases.nixos.org/nixos/25.11/nixos-25.11.9586.10e7ad5bbcb4/nixexprs.tar.xz";
|
||||
sha256 = "sha256-wjAIDqQxE+kWV2lbykQCcS+F0ArQwmN8iNw0kcj4iaA=";
|
||||
};
|
||||
pkgs = import nixpkgs { overlays = [ (import ./default.nix) ]; };
|
||||
in pkgs.mkShell {
|
||||
buildInputs = [
|
||||
(pkgs.python3.withPackages (ps: [ pkgs.copernicusmarine ]))
|
||||
pkgs.coreutils
|
||||
pkgs.bash
|
||||
];
|
||||
}
|
||||
default.nix: |
|
||||
final: prev: {
|
||||
arcosparse = prev.callPackage ./arcosparse.nix { };
|
||||
copernicusmarine = prev.callPackage ./copernicusmarine.nix {
|
||||
arcosparse = final.arcosparse;
|
||||
};
|
||||
}
|
||||
copernicusmarine.nix: |
|
||||
{
|
||||
fetchPypi,
|
||||
python3Packages,
|
||||
arcosparse,
|
||||
}:
|
||||
python3Packages.buildPythonPackage rec {
|
||||
pname = "copernicusmarine";
|
||||
version = "2.2.2";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit version;
|
||||
pname = "copernicusmarine";
|
||||
sha256 = "sha256-5T3iH4Hh08wIao2MMveb/bVnVz0pK0PoN4CRk811P0g=";
|
||||
};
|
||||
|
||||
pythonRelaxDeps = true;
|
||||
|
||||
nativeBuildInputs = [ python3Packages.poetry-core ];
|
||||
propagatedBuildInputs = with python3Packages; [
|
||||
boto3
|
||||
click
|
||||
dask
|
||||
h5netcdf
|
||||
arcosparse
|
||||
lxml
|
||||
numpy
|
||||
pydantic
|
||||
pystac
|
||||
requests
|
||||
semver
|
||||
setuptools
|
||||
tqdm
|
||||
xarray
|
||||
zarr
|
||||
];
|
||||
}
|
||||
arcosparse.nix: |
|
||||
{
|
||||
fetchPypi,
|
||||
python3Packages,
|
||||
}:
|
||||
python3Packages.buildPythonPackage rec {
|
||||
pname = "arcosparse";
|
||||
version = "0.4.2";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit version;
|
||||
pname = "arcosparse";
|
||||
sha256 = "sha256-Z8NW+dsC3uXk101kr8tzsgjAoFb4KNdGkxyFkJ5UhFA=";
|
||||
};
|
||||
|
||||
pythonRelaxDeps = true;
|
||||
|
||||
nativeBuildInputs = [ python3Packages.poetry-core ];
|
||||
propagatedBuildInputs = with python3Packages; [
|
||||
pyarrow
|
||||
pandas
|
||||
pystac
|
||||
tqdm
|
||||
requests
|
||||
];
|
||||
}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nemo
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: "0 13 * * *" # Everyday at 13:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid"
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: ghcr.io/lix-project/lix:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
nix-shell /nix-overlay/shell.nix \
|
||||
--keep COPERNICUSMARINE_SERVICE_USERNAME \
|
||||
--keep COPERNICUSMARINE_SERVICE_PASSWORD \
|
||||
--run '
|
||||
copernicusmarine login \
|
||||
--username "$COPERNICUSMARINE_SERVICE_USERNAME" \
|
||||
--password "$COPERNICUSMARINE_SERVICE_PASSWORD" \
|
||||
--force-overwrite &&
|
||||
bash /scripts/download.sh \
|
||||
$(date -d "2 days ago" +%Y-%m-%d) \
|
||||
$(date +%Y-%m-%d) &&
|
||||
chown -R 5000:5000 /data/hdd/data/NEMO &&
|
||||
chmod -R g+w /data/hdd/data/NEMO
|
||||
' || {
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
}
|
||||
env:
|
||||
- name: COPERNICUSMARINE_SERVICE_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nemo-credentials
|
||||
key: username
|
||||
- name: COPERNICUSMARINE_SERVICE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nemo-credentials
|
||||
key: password
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: nix
|
||||
mountPath: /nix-overlay
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: nemo-script
|
||||
defaultMode: 0755
|
||||
- name: nix
|
||||
configMap:
|
||||
name: nemo-nix
|
||||
defaultMode: 0644
|
||||
@@ -1,113 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: norkyst-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.sh: |
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# this script downloads files from:
|
||||
# https://thredds.met.no/thredds/catalog/fou-hi/new_norkyst800m/norkyst_v3_test/his/catalog.html
|
||||
|
||||
# safe bash settings
|
||||
set -euf -o pipefail
|
||||
|
||||
JOBS=8 # parallel downloads
|
||||
|
||||
# define start and end dates (YYYY-MM-DD)
|
||||
start_date=$(date -d "yesterday" +%Y-%m-%d)
|
||||
end_date=$(date -d "yesterday" +%Y-%m-%d)
|
||||
|
||||
# check if thredds is reachable before attempting any downloads
|
||||
if ! wget --spider --quiet "https://thredds.met.no/thredds/catalog/fou-hi/new_norkyst800m/norkyst_v3_test/his/catalog.html"; then
|
||||
echo "thredds.met.no is unreachable, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# function to print stuff in red
|
||||
red() {
|
||||
printf "\e[31m%s\e[0m" "$1"
|
||||
}
|
||||
|
||||
download_day() {
|
||||
local current_date="$1"
|
||||
local year month day file_name target_file_name url
|
||||
year=$(date -d "${current_date}" +%Y)
|
||||
month=$(date -d "${current_date}" +%m)
|
||||
day=$(date -d "${current_date}" +%d)
|
||||
|
||||
mkdir -p "/data/hdd/data/norkyst/${year}/${month}"
|
||||
|
||||
file_name="norkyst800_his_sdepth_${year}${month}${day}T00Z_m00_AN.nc"
|
||||
target_file_name="/data/hdd/data/norkyst/${year}/${month}/${file_name}"
|
||||
url="https://thredds.met.no/thredds/fileServer/fou-hi/new_norkyst800m/norkyst_v3_test/his/${year}/${month}/${day}/${file_name}"
|
||||
|
||||
if [[ ! -f "${target_file_name}" ]]; then
|
||||
if wget --spider --quiet "${url}"; then
|
||||
echo "downloading ${url}"
|
||||
wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}"
|
||||
else
|
||||
echo "${target_file_name} $(red 'not found on server')"
|
||||
fi
|
||||
else
|
||||
echo "${target_file_name} already exists locally"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f download_day red
|
||||
|
||||
current_date=$(date -d "${start_date}" +%Y-%m-%d)
|
||||
while [[ "${current_date}" < "${end_date}" || "${current_date}" == "${end_date}" ]]; do
|
||||
echo "${current_date}"
|
||||
current_date=$(date -d "${current_date} + 1 day" +%Y-%m-%d)
|
||||
done | parallel -j "${JOBS}" download_day
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: norkyst
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: 0 13 * * * # Everyday at 13:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Allow"
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: juselius/busynix:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
nix-env -iA nixpkgs.wget nixpkgs.coreutils nixpkgs.bash nixpkgs.parallel
|
||||
if bash /scripts/download.sh; then
|
||||
chown -R 10000:10000 /data/hdd/data/norkyst
|
||||
chmod -R g+w /data/hdd/data/norkyst
|
||||
else
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
fi
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: norkyst-script
|
||||
defaultMode: 0755
|
||||
@@ -1,153 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: norshelf-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.sh: |
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# this script downloads files from:
|
||||
# https://thredds.met.no/thredds/catalog/sea_norshelf_files/YYYY/catalog.html
|
||||
|
||||
# safe bash settings
|
||||
set -euf -o pipefail
|
||||
|
||||
# define start and end dates (YYYY-MM-DD)
|
||||
start_date="2026-03-01"
|
||||
end_date=$(date +%Y-%m-%d)
|
||||
|
||||
# check if thredds is reachable before attempting any downloads
|
||||
if ! wget --spider --quiet "https://thredds.met.no/thredds/catalog/sea_norshelf_files/catalog.html"; then
|
||||
echo "thredds.met.no is unreachable, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# function to print stuff in red
|
||||
red() {
|
||||
printf "\e[31m%s\e[0m" "$1"
|
||||
}
|
||||
|
||||
current_date=$(date -d "${start_date}" +%Y-%m-%d)
|
||||
while [[ "${current_date}" < "${end_date}" || "${current_date}" == "${end_date}" ]]; do
|
||||
year=$(date -d "${current_date}" +%Y)
|
||||
month=$(date -d "${current_date}" +%m)
|
||||
day=$(date -d "${current_date}" +%d)
|
||||
|
||||
mkdir -p "/data/hdd/data/norshelf/sea_norshelf_files/${year}/${month}"
|
||||
|
||||
file_name="norshelf_qck_an_${year}${month}${day}T00Z.nc"
|
||||
target_file_name="/data/hdd/data/norshelf/sea_norshelf_files/${year}/${month}/${file_name}"
|
||||
url="https://thredds.met.no/thredds/fileServer/sea_norshelf_files/${year}/${month}/${file_name}"
|
||||
|
||||
if [[ ! -f "${target_file_name}" ]]; then
|
||||
if wget --spider --quiet "${url}"; then
|
||||
echo "downloading ${url}"
|
||||
wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}"
|
||||
else
|
||||
echo "${target_file_name} $(red 'not found on server')"
|
||||
fi
|
||||
else
|
||||
echo "${target_file_name} already exists locally"
|
||||
fi
|
||||
|
||||
# move to next day
|
||||
current_date=$(date -d "${current_date} + 1 day" +%Y-%m-%d)
|
||||
done
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: norshelf
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: 0 13 * * * # Everyday at 13:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid" # If only one at at time set to Allow else Forbid
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: juselius/busynix:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
nix-env -iA nixpkgs.wget nixpkgs.coreutils nixpkgs.bash
|
||||
if bash /scripts/download.sh; then
|
||||
chown -R 5000:5000 /data/hdd/data/norshelf
|
||||
chmod -R g+w /data/hdd/data/norshelf
|
||||
else
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
fi
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: norshelf-script
|
||||
defaultMode: 0755
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ekman-data
|
||||
namespace: cron
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
storageClassName: ""
|
||||
volumeMode: Filesystem
|
||||
volumeName: pv-ekman-data
|
||||
status:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
capacity:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: pv-ekman-data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
claimRef:
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
name: ekman-data
|
||||
namespace: cron
|
||||
capacity:
|
||||
storage: 1Gi
|
||||
csi:
|
||||
driver: rook-ceph.cephfs.csi.ceph.com
|
||||
nodeStageSecretRef:
|
||||
name: rook-csi-cephfs-node
|
||||
namespace: rook-ceph
|
||||
volumeAttributes:
|
||||
clusterID: rook-ceph
|
||||
fsName: data
|
||||
rootPath: /
|
||||
staticVolume: "true"
|
||||
volumeHandle: pv-ekman-data
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
volumeMode: Filesystem
|
||||
@@ -1,60 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nve-config
|
||||
namespace: cron
|
||||
data:
|
||||
appsettings.json: |
|
||||
{
|
||||
"NveUrl": "https://chartserver.nve.no/ShowData.aspx?req=getchart&ver=1.0",
|
||||
"DataDir": "/data/hdd/data/river-data"
|
||||
}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nve
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: "0 8 * * *" # Everyday at 08:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid"
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "Never"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: git.oceanbox.io/oceanbox/churn/riverrun:24a8bbbc-debug
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
if riverrun data --download --ndays 5000; then
|
||||
chown -R 5000:5000 /data/hdd/data/river-data/Data
|
||||
chmod -R g+w /data/hdd/data/river-data/Data
|
||||
else
|
||||
echo "Job failed, sleeping 30 minutes before retry..."
|
||||
sleep 1800
|
||||
exit 1
|
||||
fi
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: config
|
||||
mountPath: /app/appsettings.json
|
||||
subPath: appsettings.json
|
||||
readOnly: true
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: config
|
||||
configMap:
|
||||
name: nve-config
|
||||
+1
-3
@@ -3,12 +3,10 @@
|
||||
"extends": [
|
||||
"config:recommended"
|
||||
],
|
||||
"minimumReleaseAge": "7 days",
|
||||
"dependencyDashboard": true,
|
||||
"semanticCommits": "disabled",
|
||||
"ignorePaths": [
|
||||
"**/bootstrap/**",
|
||||
"**/attic/**"
|
||||
"**/bootstrap/**"
|
||||
],
|
||||
"helmfile": {
|
||||
"managerFilePatterns": [
|
||||
|
||||
@@ -7,7 +7,6 @@ let
|
||||
overlays = [ ];
|
||||
};
|
||||
treefmt = import ./nix/treefmt.nix { };
|
||||
kueuectl = pkgs.callPackage ./nix/kueuectl.nix { };
|
||||
in
|
||||
pkgs.mkShellNoCC {
|
||||
packages = [
|
||||
@@ -28,7 +27,6 @@ pkgs.mkShellNoCC {
|
||||
pkgs.kubectl-rook-ceph
|
||||
|
||||
# other tools activate when needed
|
||||
kueuectl
|
||||
# pkgs.step-cli
|
||||
# pkgs.linkerd
|
||||
# pkgs.cmctl
|
||||
@@ -37,15 +35,12 @@ pkgs.mkShellNoCC {
|
||||
# pkgs.renovate
|
||||
# pkgs.graphviz
|
||||
# pkgs.hubble
|
||||
pkgs.cilium-cli
|
||||
pkgs.dapr-cli
|
||||
# pkgs.dapr-cli
|
||||
];
|
||||
|
||||
# Environment variables
|
||||
ARGOCD_ENV_CLUSTER_NAME = "ekman";
|
||||
HELM_GIT_ACCESS_TOKEN = "glpat-xxx";
|
||||
API_SERVER_IP = "localhost";
|
||||
API_SERVER_PORT = "7445";
|
||||
|
||||
# Alternative shells
|
||||
passthru = pkgs.lib.mapAttrs (name: value: pkgs.mkShellNoCC (value // { inherit name; })) {
|
||||
|
||||
@@ -28,6 +28,7 @@ spec:
|
||||
managedNamespaceMetadata:
|
||||
labels:
|
||||
component: sys
|
||||
shared-gateway-access: "true"
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
{{- if .Values.clusterConfig.gatewayAPI.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: argocd-server
|
||||
namespace: argocd
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared-gateway
|
||||
namespace: kube-system
|
||||
sectionName: https-internal
|
||||
hostnames:
|
||||
- argocd.{{ .Values.clusterConfig.domain }}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: "/"
|
||||
backendRefs:
|
||||
- name: argocd-server
|
||||
port: 80
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: allow-gateway-to-argocd
|
||||
namespace: argocd
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: argocd-server
|
||||
ingress:
|
||||
- fromCIDRSet:
|
||||
{{- range .Values.clusterConfig.ingress_whitelist }}
|
||||
- cidr: {{ . }}
|
||||
{{- end }}
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": argocd
|
||||
{{- end }}
|
||||
@@ -94,14 +94,6 @@ spec:
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: jobset-system
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: ingress-haproxy
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: dex
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: cra-agent
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: catalyst
|
||||
server: https://kubernetes.default.svc
|
||||
sourceRepos:
|
||||
- https://argoproj.github.io/argo-helm
|
||||
- https://kubernetes-sigs.github.io/metrics-server/
|
||||
@@ -141,14 +133,10 @@ spec:
|
||||
- ghcr.io/spegel-org/helm-charts
|
||||
- quay.io/cilium/charts
|
||||
- quay.io/jetstack/charts
|
||||
- quay.io/enix/charts
|
||||
- registry.k8s.io/jobset/charts/jobset
|
||||
- ghcr.io/dragonflydb/dragonfly-operator/helm/dragonfly-operator
|
||||
- docker.gitea.com
|
||||
- https://operator.mariadb.com/mariadb-enterprise-operator
|
||||
- https://ot-container-kit.github.io/helm-charts
|
||||
- https://operator.mariadb.com
|
||||
- https://ot-container-kit.github.io/helm-charts
|
||||
- https://twin.github.io/helm-charts
|
||||
- https://charts.dexidp.io
|
||||
- public.ecr.aws/diagrid/catalyst
|
||||
- ghcr.io/haproxytech/helm-charts
|
||||
|
||||
@@ -4,8 +4,11 @@ global:
|
||||
## Ref: https://github.com/argoproj/argo-cd
|
||||
##
|
||||
configs:
|
||||
{{- if .Values.argocd.anyNamespaces.enabled }}
|
||||
params:
|
||||
{{- if .Values.clusterConfig.gatewayAPI.enabled }}
|
||||
server.insecure: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.argocd.anyNamespaces.enabled }}
|
||||
applicationsetcontroller.namespaces: "{{ .Values.argocd.anyNamespaces.glob }}"
|
||||
# TODO(kai): anyapp will disable PR review apps. Look into anyapp settings to fix it
|
||||
applicationsetcontroller.enable.scm.providers: "false"
|
||||
@@ -238,6 +241,9 @@ server:
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
ingress:
|
||||
{{- if .Values.clusterConfig.gatewayAPI.enabled }}
|
||||
enabled: false
|
||||
{{- else }}
|
||||
enabled: true
|
||||
ingressClassName: nginx
|
||||
annotations:
|
||||
@@ -254,6 +260,7 @@ server:
|
||||
- secretName: argocd-tls
|
||||
hosts:
|
||||
- "argocd.{{ .Values.clusterConfig.domain }}"
|
||||
{{- end }}
|
||||
applicationSet:
|
||||
metrics:
|
||||
enabled: true
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: beta-atlantis-actor-config
|
||||
data:
|
||||
KUEUE_NAMESPACE: "prod-queue"
|
||||
XTRACT_IMAGE: "git.oceanbox.io/oceanbox/katamari/excavator:v1.2.14"
|
||||
XTRACT_QUEUE: "prod-queue"
|
||||
PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.2.14"
|
||||
PLUME_QUEUE: "prod-queue"
|
||||
@@ -76,7 +76,7 @@
|
||||
"https://maps.beta.oceanbox.io"
|
||||
],
|
||||
"appName": "atlantis",
|
||||
"appEnv": "preprod",
|
||||
"appEnv": "prod",
|
||||
"appNamespace": "atlantis",
|
||||
"appVersion": "2.95.1",
|
||||
"otelCollector": "http://opentelemetry-collector.otel.svc:4317",
|
||||
|
||||
@@ -8,8 +8,3 @@
|
||||
value:
|
||||
secretRef:
|
||||
name: prod-atlantis-env
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/envFrom/-
|
||||
value:
|
||||
configMapRef:
|
||||
name: beta-atlantis-actor-config
|
||||
@@ -14,7 +14,6 @@ patches:
|
||||
resources:
|
||||
- ../base
|
||||
- rbac.yaml
|
||||
- actor-config.yaml
|
||||
- tracing.yaml
|
||||
- bindings.yaml
|
||||
- pubsub.yaml
|
||||
|
||||
@@ -8,7 +8,6 @@ rules:
|
||||
- ""
|
||||
resourceNames:
|
||||
- beta-atlantis-appsettings
|
||||
- beta-atlantis-actor-config
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: prod-atlantis-actor-config
|
||||
data:
|
||||
KUEUE_NAMESPACE: "prod-queue"
|
||||
XTRACT_IMAGE: "git.oceanbox.io/oceanbox/katamari/excavator:v1.2.8"
|
||||
XTRACT_QUEUE: "prod-queue"
|
||||
PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.2.8"
|
||||
PLUME_QUEUE: "prod-queue"
|
||||
@@ -8,8 +8,3 @@
|
||||
value:
|
||||
secretRef:
|
||||
name: prod-atlantis-env
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/envFrom/-
|
||||
value:
|
||||
configMapRef:
|
||||
name: prod-atlantis-actor-config
|
||||
@@ -13,7 +13,6 @@ patches:
|
||||
resources:
|
||||
- ../base
|
||||
- secrets.yaml
|
||||
- actor-config.yaml
|
||||
- rbac.yaml
|
||||
- tracing.yaml
|
||||
- bindings.yaml
|
||||
|
||||
@@ -8,7 +8,6 @@ rules:
|
||||
- ""
|
||||
resourceNames:
|
||||
- prod-atlantis-appsettings
|
||||
- prod-atlantis-actor-config
|
||||
resources:
|
||||
- configmaps
|
||||
verbs:
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: staging-atlantis-actor-config
|
||||
data:
|
||||
XTRACT_IMAGE: "git.oceanbox.io/oceanbox/katamari/excavator:v1.4.0"
|
||||
XTRACT_QUEUE: "dev-queue"
|
||||
PLUME_IMAGE: "git.oceanbox.io/oceanbox/katamari/plume:v1.4.0"
|
||||
PLUME_QUEUE: "dev-queue"
|
||||
@@ -8,8 +8,3 @@
|
||||
value:
|
||||
secretRef:
|
||||
name: staging-atlantis-env
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/envFrom/-
|
||||
value:
|
||||
configMapRef:
|
||||
name: staging-atlantis-actor-config
|
||||
@@ -14,7 +14,6 @@ resources:
|
||||
- ../base
|
||||
- rbac.yaml
|
||||
- secrets.yaml
|
||||
- actor-config.yaml
|
||||
- tracing.yaml
|
||||
- bindings.yaml
|
||||
- pubsub.yaml
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user