fix(cron): Add backoff

raw/ekman/cronjobs/arome/cron.yaml

@@ -6,6 +6,7 @@ metadata:
 data:
   download.py: |
     import os
+    import time
     from netCDF4 import Dataset
     import re
     from datetime import datetime, timedelta
@@ -83,15 +84,23 @@ data:
     for fname in fList:
         savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc")
         print(savename)
-        try:
+        retries = 4
+        for attempt in range(retries):
             try:
-                copy_thredds_file(fname, savename)
-            except:
-                fname = re.sub("sfc", "2_5km", fname)
-                fname = re.sub("ncml", "nc", fname)
-                copy_thredds_file(fname, savename)
-        except:
-            print("File not found: " + fname)
+                try:
+                    copy_thredds_file(fname, savename)
+                except:
+                    alt_fname = re.sub("sfc", "2_5km", fname)
+                    alt_fname = re.sub("ncml", "nc", alt_fname)
+                    copy_thredds_file(alt_fname, savename)
+                break
+            except Exception as e:
+                if attempt == retries - 1:
+                    print("File not found: " + fname)
+                else:
+                    backoff = min(2 ** attempt * 5, 60)
+                    print(f"Retrying in {backoff}s... ({e})")
+                    time.sleep(backoff)
 ---
 apiVersion: batch/v1
 kind: CronJob

raw/ekman/cronjobs/mur/cron.yaml

@@ -8,6 +8,7 @@ data:
     import argparse
     import os
     import sys
+    import time
     import requests
     from datetime import datetime
 
@@ -86,8 +87,10 @@ data:
                 if attempt == retries - 1:
                     print(f"Failed: {filename} -> {e}")
                 else:
+                    backoff = min(2 ** attempt * 5, 60)
                     if verbose:
-                        print(f"Retrying {filename}... ({e})")
+                        print(f"Retrying {filename} in {backoff}s... ({e})")
+                    time.sleep(backoff)
 
 
     def validate_dates(start, end):

raw/ekman/cronjobs/norkyst/cron.yaml

@@ -46,7 +46,7 @@ data:
         if [[ ! -f "${target_file_name}" ]]; then
             if wget --spider --quiet "${url}"; then
                 echo "downloading ${url}"
-                wget -O "${target_file_name}" "${url}"
+                wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}"
             else
                 echo "${target_file_name} $(red 'not found on server')"
             fi
@@ -70,7 +70,7 @@ metadata:
   namespace: cron
 spec:
   schedule: 0 13 * * * # Everyday at 13:00, use https://crontab.guru
-  concurrencyPolicy: "Forbid"
+  concurrencyPolicy: "Allow"
   successfulJobsHistoryLimit: 10
   failedJobsHistoryLimit: 3
   jobTemplate:
@@ -89,7 +89,7 @@ spec:
                 - |
                   nix-env -iA nixpkgs.wget nixpkgs.coreutils nixpkgs.bash nixpkgs.parallel
                   bash /scripts/download.sh
-                  chown -R 5000:5000 /data/hdd/data/norkyst
+                  chown -R 10000:10000 /data/hdd/data/norkyst
                   chmod -R g+w /data/hdd/data/norkyst
               resources: {}
               volumeMounts:

raw/ekman/cronjobs/norshelf/cron.yaml

@@ -43,7 +43,7 @@ data:
         if [[ ! -f "${target_file_name}" ]]; then
             if wget --spider --quiet "${url}"; then
                 echo "downloading ${url}"
-                wget -O "${target_file_name}" "${url}"
+                wget --tries=5 --waitretry=60 -O "${target_file_name}" "${url}"
             else
                 echo "${target_file_name} $(red 'not found on server')"
             fi