feat(sorcerer): Add raid-data pvc and cronjobs
This commit is contained in:
@@ -0,0 +1,225 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nemo-script
|
||||
namespace: cron
|
||||
data:
|
||||
download.sh: |
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# this script downloads files from
|
||||
# https://data.marine.copernicus.eu/product/NWSHELF_ANALYSISFORECAST_PHY_004_013
|
||||
|
||||
set -euf -o pipefail
|
||||
|
||||
START_DATE="$1"
|
||||
END_DATE="$2"
|
||||
|
||||
current_date="$START_DATE"
|
||||
|
||||
while [[ "$current_date" < "$END_DATE" ]]; do
|
||||
next_date=$(date -I -d "$current_date + 1 day")
|
||||
echo "Running subset for $current_date to $next_date"
|
||||
|
||||
outfile="cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded salt"
|
||||
|
||||
outfile="cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded currents"
|
||||
|
||||
outfile="cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded temperature"
|
||||
|
||||
outfile="cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i_${current_date}--${next_date}.nc"
|
||||
copernicusmarine subset \
|
||||
--dataset-id cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i \
|
||||
-t "$current_date" \
|
||||
-T "$next_date" \
|
||||
-f "$outfile" \
|
||||
-o /data/hdd/data/NEMO/
|
||||
echo "Downloaded ssh"
|
||||
|
||||
current_date="$next_date"
|
||||
done
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nemo-nix
|
||||
namespace: cron
|
||||
data:
|
||||
shell.nix: |
|
||||
let
|
||||
nixpkgs = builtins.fetchTarball {
|
||||
url = "https://releases.nixos.org/nixos/25.11/nixos-25.11.9586.10e7ad5bbcb4/nixexprs.tar.xz";
|
||||
sha256 = "sha256-wjAIDqQxE+kWV2lbykQCcS+F0ArQwmN8iNw0kcj4iaA=";
|
||||
};
|
||||
pkgs = import nixpkgs { overlays = [ (import ./default.nix) ]; };
|
||||
in pkgs.mkShell {
|
||||
buildInputs = [
|
||||
(pkgs.python3.withPackages (ps: [ pkgs.copernicusmarine ]))
|
||||
pkgs.coreutils
|
||||
pkgs.bash
|
||||
];
|
||||
}
|
||||
default.nix: |
|
||||
final: prev: {
|
||||
arcosparse = prev.callPackage ./arcosparse.nix { };
|
||||
copernicusmarine = prev.callPackage ./copernicusmarine.nix {
|
||||
arcosparse = final.arcosparse;
|
||||
};
|
||||
}
|
||||
copernicusmarine.nix: |
|
||||
{
|
||||
fetchPypi,
|
||||
python3Packages,
|
||||
arcosparse,
|
||||
}:
|
||||
python3Packages.buildPythonPackage rec {
|
||||
pname = "copernicusmarine";
|
||||
version = "2.2.2";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit version;
|
||||
pname = "copernicusmarine";
|
||||
sha256 = "sha256-5T3iH4Hh08wIao2MMveb/bVnVz0pK0PoN4CRk811P0g=";
|
||||
};
|
||||
|
||||
pythonRelaxDeps = true;
|
||||
|
||||
nativeBuildInputs = [ python3Packages.poetry-core ];
|
||||
propagatedBuildInputs = with python3Packages; [
|
||||
boto3
|
||||
click
|
||||
dask
|
||||
h5netcdf
|
||||
arcosparse
|
||||
lxml
|
||||
numpy
|
||||
pydantic
|
||||
pystac
|
||||
requests
|
||||
semver
|
||||
setuptools
|
||||
tqdm
|
||||
xarray
|
||||
zarr
|
||||
];
|
||||
}
|
||||
arcosparse.nix: |
|
||||
{
|
||||
fetchPypi,
|
||||
python3Packages,
|
||||
}:
|
||||
python3Packages.buildPythonPackage rec {
|
||||
pname = "arcosparse";
|
||||
version = "0.4.2";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit version;
|
||||
pname = "arcosparse";
|
||||
sha256 = "sha256-Z8NW+dsC3uXk101kr8tzsgjAoFb4KNdGkxyFkJ5UhFA=";
|
||||
};
|
||||
|
||||
pythonRelaxDeps = true;
|
||||
|
||||
nativeBuildInputs = [ python3Packages.poetry-core ];
|
||||
propagatedBuildInputs = with python3Packages; [
|
||||
pyarrow
|
||||
pandas
|
||||
pystac
|
||||
tqdm
|
||||
requests
|
||||
];
|
||||
}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nemo
|
||||
namespace: cron
|
||||
spec:
|
||||
schedule: "0 13 * * *" # Everyday at 13:00, use https://crontab.guru
|
||||
concurrencyPolicy: "Forbid"
|
||||
successfulJobsHistoryLimit: 10
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 10
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: "OnFailure"
|
||||
containers:
|
||||
- name: cronpod
|
||||
image: ghcr.io/lix-project/lix:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
nix-shell /nix-overlay/shell.nix \
|
||||
--keep COPERNICUSMARINE_SERVICE_USERNAME \
|
||||
--keep COPERNICUSMARINE_SERVICE_PASSWORD \
|
||||
--run '
|
||||
copernicusmarine login \
|
||||
--username "$COPERNICUSMARINE_SERVICE_USERNAME" \
|
||||
--password "$COPERNICUSMARINE_SERVICE_PASSWORD" \
|
||||
--force-overwrite
|
||||
bash /scripts/download.sh \
|
||||
$(date -d "2 days ago" +%Y-%m-%d) \
|
||||
$(date +%Y-%m-%d)
|
||||
chown -R 5000:5000 /data/hdd/data/NEMO
|
||||
chmod -R g+w /data/hdd/data/NEMO
|
||||
'
|
||||
env:
|
||||
- name: COPERNICUSMARINE_SERVICE_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nemo-credentials
|
||||
key: username
|
||||
- name: COPERNICUSMARINE_SERVICE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: nemo-credentials
|
||||
key: password
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: nix
|
||||
mountPath: /nix-overlay
|
||||
securityContext: {}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ekman-data
|
||||
- name: script
|
||||
configMap:
|
||||
name: nemo-script
|
||||
defaultMode: 0755
|
||||
- name: nix
|
||||
configMap:
|
||||
name: nemo-nix
|
||||
defaultMode: 0644
|
||||
Reference in New Issue
Block a user