diff --git a/raw/ekman/cronjobs/arome/cron.yaml b/raw/ekman/cronjobs/arome/cron.yaml index e4b75ac6..2cc0cd36 100644 --- a/raw/ekman/cronjobs/arome/cron.yaml +++ b/raw/ekman/cronjobs/arome/cron.yaml @@ -6,6 +6,7 @@ metadata: data: download.py: | import os + import time from netCDF4 import Dataset import re from datetime import datetime, timedelta @@ -83,15 +84,23 @@ data: for fname in fList: savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc") print(savename) - try: + retries = 4 + for attempt in range(retries): try: - copy_thredds_file(fname, savename) - except: - fname = re.sub("sfc", "2_5km", fname) - fname = re.sub("ncml", "nc", fname) - copy_thredds_file(fname, savename) - except: - print("File not found: " + fname) + try: + copy_thredds_file(fname, savename) + except: + alt_fname = re.sub("sfc", "2_5km", fname) + alt_fname = re.sub("ncml", "nc", alt_fname) + copy_thredds_file(alt_fname, savename) + break + except Exception as e: + if attempt == retries - 1: + print("File not found: " + fname) + else: + backoff = min(2 ** attempt * 5, 60) + print(f"Retrying in {backoff}s... ({e})") + time.sleep(backoff) --- apiVersion: batch/v1 kind: CronJob diff --git a/raw/ekman/cronjobs/mur/cron.yaml b/raw/ekman/cronjobs/mur/cron.yaml index 5087c462..4b5750fd 100644 --- a/raw/ekman/cronjobs/mur/cron.yaml +++ b/raw/ekman/cronjobs/mur/cron.yaml @@ -8,6 +8,7 @@ data: import argparse import os import sys + import time import requests from datetime import datetime @@ -86,8 +87,10 @@ data: if attempt == retries - 1: print(f"Failed: {filename} -> {e}") else: + backoff = min(2 ** attempt * 5, 60) if verbose: - print(f"Retrying {filename}... ({e})") + print(f"Retrying {filename} in {backoff}s... ({e})") + time.sleep(backoff) def validate_dates(start, end): diff --git a/raw/ekman/cronjobs/norkyst/cron.yaml b/raw/ekman/cronjobs/norkyst/cron.yaml index 9027c867..83313cca 100644 --- a/raw/ekman/cronjobs/norkyst/cron.yaml +++ b/raw/ekman/cronjobs/norkyst/cron.yaml @@ -46,7 +46,7 @@ data: if [[ ! -f "${target_file_name}" ]]; then if wget --spider --quiet "${url}"; then echo "downloading ${url}" - wget -O "${target_file_name}" "${url}" + wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}" else echo "${target_file_name} $(red 'not found on server')" fi @@ -70,7 +70,7 @@ metadata: namespace: cron spec: schedule: 0 13 * * * # Everyday at 13:00, use https://crontab.guru - concurrencyPolicy: "Forbid" + concurrencyPolicy: "Allow" successfulJobsHistoryLimit: 10 failedJobsHistoryLimit: 3 jobTemplate: @@ -89,7 +89,7 @@ spec: - | nix-env -iA nixpkgs.wget nixpkgs.coreutils nixpkgs.bash nixpkgs.parallel bash /scripts/download.sh - chown -R 5000:5000 /data/hdd/data/norkyst + chown -R 10000:10000 /data/hdd/data/norkyst chmod -R g+w /data/hdd/data/norkyst resources: {} volumeMounts: diff --git a/raw/ekman/cronjobs/norshelf/cron.yaml b/raw/ekman/cronjobs/norshelf/cron.yaml index 23a82c6d..94298f74 100644 --- a/raw/ekman/cronjobs/norshelf/cron.yaml +++ b/raw/ekman/cronjobs/norshelf/cron.yaml @@ -43,7 +43,7 @@ data: if [[ ! -f "${target_file_name}" ]]; then if wget --spider --quiet "${url}"; then echo "downloading ${url}" - wget -O "${target_file_name}" "${url}" + wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}" else echo "${target_file_name} $(red 'not found on server')" fi