152 lines
4.8 KiB
YAML
152 lines
4.8 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: arome-script
|
|
namespace: cron
|
|
data:
|
|
download.py: |
|
|
import os
|
|
import sys
|
|
from netCDF4 import Dataset
|
|
import re
|
|
from datetime import datetime, timedelta
|
|
|
|
fname ="https://thredds.met.no/thredds/dodsC/meps25epsarchive/YEAR/MONTH/DAY/meps_det_sfc_YEARMONTHDAYT00Z.ncml"
|
|
outdir = "/data/hdd/data/AROME"
|
|
|
|
|
|
def generate_thredds_names(start, stop):
|
|
start_date = datetime(int(start.split("-")[0]),
|
|
int(start.split("-")[1]),
|
|
int(start.split("-")[2]))
|
|
end_date = datetime(int(stop.split("-")[0]),
|
|
int(stop.split("-")[1]),
|
|
int(stop.split("-")[2]))
|
|
|
|
date_list = []
|
|
while start_date <= end_date:
|
|
date_list.append(start_date)
|
|
start_date += timedelta(days=1)
|
|
|
|
fileList = []
|
|
for date in date_list:
|
|
y = str(date.year)
|
|
m = (str(date.month)).zfill(2)
|
|
d = (str(date.day)).zfill(2)
|
|
|
|
f = re.sub("YEAR", y, fname)
|
|
f = re.sub("MONTH", m, f)
|
|
f = re.sub("DAY", d, f)
|
|
fileList.append(f)
|
|
|
|
return fileList
|
|
|
|
def copy_thredds_file(threddsFile, savename):
|
|
dsin = Dataset(threddsFile)
|
|
|
|
dsout = Dataset(savename, "w")
|
|
|
|
for dname, the_dim in dsin.dimensions.items():
|
|
dsout.createDimension(dname, len(the_dim) if not the_dim.isunlimited() else None)
|
|
|
|
aromeNames = ["time",
|
|
"longitude",
|
|
"latitude",
|
|
"land_area_fraction",
|
|
"air_temperature_2m",
|
|
"precipitation_amount_acc",
|
|
"water_evaporation_amount",
|
|
"relative_humidity_2m",
|
|
"integral_of_surface_downwelling_longwave_flux_in_air_wrt_time",
|
|
"integral_of_surface_net_downward_shortwave_flux_wrt_time",
|
|
"air_pressure_at_sea_level",
|
|
"x_wind_10m",
|
|
"y_wind_10m"]
|
|
|
|
for v_name, varin in dsin.variables.items():
|
|
if v_name in aromeNames:
|
|
fill_value = None
|
|
|
|
if hasattr(varin, "_FillValue"):
|
|
fill_value = varin._FillValue
|
|
|
|
outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions, fill_value=fill_value)
|
|
|
|
outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs() if k not in ["_FillValue"]})
|
|
|
|
outVar[:] = varin[:]
|
|
dsout.close()
|
|
|
|
|
|
os.makedirs(outdir, exist_ok=True)
|
|
|
|
fList = generate_thredds_names("2026-04-24", datetime.today().strftime("%Y-%m-%d"))
|
|
failed = False
|
|
for fname in fList:
|
|
savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc")
|
|
if os.path.exists(savename):
|
|
print(f"Skipping {savename}, already exists")
|
|
continue
|
|
print(savename)
|
|
try:
|
|
try:
|
|
copy_thredds_file(fname, savename)
|
|
except:
|
|
alt_fname = re.sub("sfc", "2_5km", fname)
|
|
alt_fname = re.sub("ncml", "nc", alt_fname)
|
|
copy_thredds_file(alt_fname, savename)
|
|
except Exception as e:
|
|
print(f"File not found: {fname} ({e})")
|
|
failed = True
|
|
|
|
if failed:
|
|
sys.exit(1)
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: arome
|
|
namespace: cron
|
|
spec:
|
|
schedule: 0 6 * * * # Everyday at 06:00, use https://crontab.guru
|
|
concurrencyPolicy: "Forbid" # If only one at at time set to Allow else Forbid
|
|
successfulJobsHistoryLimit: 10
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 3
|
|
template:
|
|
spec:
|
|
restartPolicy: "Never"
|
|
containers:
|
|
- name: cronpod
|
|
image: juselius/busynix:1.1
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
if nix-shell -p 'python3.withPackages(ps: [ps.netcdf4])' --run 'python3 /scripts/download.py'; then
|
|
chown -R 5000:5000 /data/hdd/data/AROME
|
|
chmod -R g+w /data/hdd/data/AROME
|
|
else
|
|
echo "Job failed, sleeping 30 minutes before retry..."
|
|
sleep 1800
|
|
exit 1
|
|
fi
|
|
resources: {}
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
- name: script
|
|
mountPath: /scripts
|
|
securityContext: {}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: ekman-data
|
|
- name: script
|
|
configMap:
|
|
name: arome-script
|
|
defaultMode: 0755
|