feat(cron): Add arome and norshelf jobs

2026-04-24 15:23:10 +02:00
parent 4cd08aa272
commit 9779200961
2 changed files with 282 additions and 0 deletions
@@ -0,0 +1,136 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: arome-script
+  namespace: cron
+data:
+  download.py: |
+    import os
+    from netCDF4 import Dataset
+    import re
+    from datetime import datetime, timedelta
+
+    fname ="https://thredds.met.no/thredds/dodsC/meps25epsarchive/YEAR/MONTH/DAY/meps_det_sfc_YEARMONTHDAYT00Z.ncml"
+    outdir = "/data/hdd/data/AROME"
+
+
+    def generate_thredds_names(start, stop):
+        start_date = datetime(int(start.split("-")[0]),
+                              int(start.split("-")[1]),
+                              int(start.split("-")[2]))
+        end_date = datetime(int(stop.split("-")[0]),
+                            int(stop.split("-")[1]),
+                            int(stop.split("-")[2]))
+
+        date_list = []
+        while start_date <= end_date:
+            date_list.append(start_date)
+            start_date += timedelta(days=1)
+
+        fileList = []
+        for date in date_list:
+            y = str(date.year)
+            m = (str(date.month)).zfill(2)
+            d = (str(date.day)).zfill(2)
+
+            f = re.sub("YEAR", y, fname)
+            f = re.sub("MONTH", m, f)
+            f = re.sub("DAY", d, f)
+            fileList.append(f)
+
+        return fileList
+
+    def copy_thredds_file(threddsFile, savename):
+        dsin = Dataset(threddsFile)
+
+        dsout = Dataset(savename, "w")
+
+        for dname, the_dim in dsin.dimensions.items():
+            dsout.createDimension(dname, len(the_dim) if not the_dim.isunlimited() else None)
+
+        aromeNames = ["time",
+                      "longitude",
+                      "latitude",
+                      "land_area_fraction",
+                      "air_temperature_2m",
+                      "precipitation_amount_acc",
+                      "water_evaporation_amount",
+                      "relative_humidity_2m",
+                      "integral_of_surface_downwelling_longwave_flux_in_air_wrt_time",
+                      "integral_of_surface_net_downward_shortwave_flux_wrt_time",
+                      "air_pressure_at_sea_level",
+                      "x_wind_10m",
+                      "y_wind_10m"]
+
+        for v_name, varin in dsin.variables.items():
+            if v_name in aromeNames:
+                fill_value = None
+
+                if hasattr(varin, "_FillValue"):
+                    fill_value = varin._FillValue
+
+                outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions, fill_value=fill_value)
+
+                outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs() if k not in ["_FillValue"]})
+
+                outVar[:] = varin[:]
+        dsout.close()
+
+
+    os.makedirs(outdir, exist_ok=True)
+
+    fList = generate_thredds_names("2026-04-24", datetime.today().strftime("%Y-%m-%d"))
+    for fname in fList:
+        savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc")
+        print(savename)
+        try:
+            try:
+                copy_thredds_file(fname, savename)
+            except:
+                fname = re.sub("sfc", "2_5km", fname)
+                fname = re.sub("ncml", "nc", fname)
+                copy_thredds_file(fname, savename)
+        except:
+            print("File not found: " + fname)
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: arome
+  namespace: cron
+spec:
+  schedule: 0 6 * * * # Everyday at 06:00, use https://crontab.guru
+  concurrencyPolicy: "Forbid" # If only one at at time set to Allow else Forbid
+  successfulJobsHistoryLimit: 10
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 10
+      template:
+        spec:
+          restartPolicy: "OnFailure"
+          containers:
+            - name: cronpod
+              image: juselius/busynix:1.1
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  nix-shell -p 'python3.withPackages(ps: [ps.netcdf4])' --run 'python3 /scripts/download.py'
+                  chown -R kraken /data/hdd/data/AROME
+              resources: {}
+              volumeMounts:
+              - name: data
+                mountPath: /data
+              - name: script
+                mountPath: /scripts
+              securityContext: {}
+          volumes:
+          - name: data
+            persistentVolumeClaim:
+              claimName: ekman-data
+          - name: script
+            configMap:
+              name: arome-script
+              defaultMode: 0755