manifests/raw/ekman/cronjobs/nemo/cron.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: nemo-script
  namespace: cron
data:
  download.sh: |
    #!/usr/bin/env bash

    # this script downloads files from
    # https://data.marine.copernicus.eu/product/NWSHELF_ANALYSISFORECAST_PHY_004_013

    set -euf -o pipefail

    START_DATE="$1"
    END_DATE="$2"

    current_date="$START_DATE"

    while [[ "$current_date" < "$END_DATE" ]]; do
      next_date=$(date -I -d "$current_date + 1 day")
      echo "Running subset for $current_date to $next_date"

      outfile="cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
      if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
        echo "Skipping salt (already exists)"
      else
        copernicusmarine subset \
          --dataset-id cmems_mod_nws_phy-sal_anfc_1.5km-3D_PT1H-i \
          -t "$current_date" \
          -T "$next_date" \
          -f "$outfile" \
          -o /data/hdd/data/NEMO/
        echo "Downloaded salt"
      fi

      outfile="cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
      if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
        echo "Skipping currents (already exists)"
      else
        copernicusmarine subset \
          --dataset-id cmems_mod_nws_phy-cur_anfc_1.5km-3D_PT1H-i \
          -t "$current_date" \
          -T "$next_date" \
          -f "$outfile" \
          -o /data/hdd/data/NEMO/
        echo "Downloaded currents"
      fi

      outfile="cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i_${current_date}--${next_date}.nc"
      if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
        echo "Skipping temperature (already exists)"
      else
        copernicusmarine subset \
          --dataset-id cmems_mod_nws_phy-tem_anfc_1.5km-3D_PT1H-i \
          -t "$current_date" \
          -T "$next_date" \
          -f "$outfile" \
          -o /data/hdd/data/NEMO/
        echo "Downloaded temperature"
      fi

      outfile="cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i_${current_date}--${next_date}.nc"
      if [[ -f "/data/hdd/data/NEMO/$outfile" ]]; then
        echo "Skipping ssh (already exists)"
      else
        copernicusmarine subset \
          --dataset-id cmems_mod_nws_phy-ssh_anfc_1.5km-2D_PT15M-i \
          -t "$current_date" \
          -T "$next_date" \
          -f "$outfile" \
          -o /data/hdd/data/NEMO/
        echo "Downloaded ssh"
      fi

      current_date="$next_date"
    done
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nemo-nix
  namespace: cron
data:
  shell.nix: |
    let
      nixpkgs = builtins.fetchTarball {
        url = "https://releases.nixos.org/nixos/25.11/nixos-25.11.9586.10e7ad5bbcb4/nixexprs.tar.xz";
        sha256 = "sha256-wjAIDqQxE+kWV2lbykQCcS+F0ArQwmN8iNw0kcj4iaA=";
      };
      pkgs = import nixpkgs { overlays = [ (import ./default.nix) ]; };
    in pkgs.mkShell {
      buildInputs = [
        (pkgs.python3.withPackages (ps: [ pkgs.copernicusmarine ]))
        pkgs.coreutils
        pkgs.bash
      ];
    }
  default.nix: |
    final: prev: {
      arcosparse = prev.callPackage ./arcosparse.nix { };
      copernicusmarine = prev.callPackage ./copernicusmarine.nix {
        arcosparse = final.arcosparse;
      };
    }
  copernicusmarine.nix: |
    {
      fetchPypi,
      python3Packages,
      arcosparse,
    }:
    python3Packages.buildPythonPackage rec {
      pname = "copernicusmarine";
      version = "2.2.2";
      format = "pyproject";

      src = fetchPypi {
        inherit version;
        pname = "copernicusmarine";
        sha256 = "sha256-5T3iH4Hh08wIao2MMveb/bVnVz0pK0PoN4CRk811P0g=";
      };

      pythonRelaxDeps = true;

      nativeBuildInputs = [ python3Packages.poetry-core ];
      propagatedBuildInputs = with python3Packages; [
        boto3
        click
        dask
        h5netcdf
        arcosparse
        lxml
        numpy
        pydantic
        pystac
        requests
        semver
        setuptools
        tqdm
        xarray
        zarr
      ];
    }
  arcosparse.nix: |
    {
      fetchPypi,
      python3Packages,
    }:
    python3Packages.buildPythonPackage rec {
      pname = "arcosparse";
      version = "0.4.2";
      format = "pyproject";

      src = fetchPypi {
        inherit version;
        pname = "arcosparse";
        sha256 = "sha256-Z8NW+dsC3uXk101kr8tzsgjAoFb4KNdGkxyFkJ5UhFA=";
      };

      pythonRelaxDeps = true;

      nativeBuildInputs = [ python3Packages.poetry-core ];
      propagatedBuildInputs = with python3Packages; [
        pyarrow
        pandas
        pystac
        tqdm
        requests
      ];
    }
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: nemo
  namespace: cron
spec:
  schedule: "0 13 * * *" # Everyday at 13:00, use https://crontab.guru
  concurrencyPolicy: "Forbid"
  successfulJobsHistoryLimit: 10
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 10
      template:
        spec:
          restartPolicy: "OnFailure"
          containers:
            - name: cronpod
              image: ghcr.io/lix-project/lix:latest
              imagePullPolicy: IfNotPresent
              command:
                - /bin/sh
                - -c
                - |
                  nix-shell /nix-overlay/shell.nix \
                    --keep COPERNICUSMARINE_SERVICE_USERNAME \
                    --keep COPERNICUSMARINE_SERVICE_PASSWORD \
                    --run '
                    copernicusmarine login \
                      --username "$COPERNICUSMARINE_SERVICE_USERNAME" \
                      --password "$COPERNICUSMARINE_SERVICE_PASSWORD" \
                      --force-overwrite
                    bash /scripts/download.sh \
                      $(date -d "2 days ago" +%Y-%m-%d) \
                      $(date +%Y-%m-%d)
                    chown -R 5000:5000 /data/hdd/data/NEMO
                    chmod -R g+w /data/hdd/data/NEMO
                  '
              env:
                - name: COPERNICUSMARINE_SERVICE_USERNAME
                  valueFrom:
                    secretKeyRef:
                      name: nemo-credentials
                      key: username
                - name: COPERNICUSMARINE_SERVICE_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: nemo-credentials
                      key: password
              resources: {}
              volumeMounts:
                - name: data
                  mountPath: /data
                - name: script
                  mountPath: /scripts
                - name: nix
                  mountPath: /nix-overlay
              securityContext: {}
          volumes:
            - name: data
              persistentVolumeClaim:
                claimName: ekman-data
            - name: script
              configMap:
                name: nemo-script
                defaultMode: 0755
            - name: nix
              configMap:
                name: nemo-nix
                defaultMode: 0644