fix(cron): Add backoff

raw/ekman/cronjobs/arome/cron.yaml

@@ -6,6 +6,7 @@ metadata:
 data:
   download.py: |
     import os
+    import time
     from netCDF4 import Dataset
     import re
     from datetime import datetime, timedelta
@@ -83,15 +84,23 @@ data:
     for fname in fList:
         savename = os.path.join(outdir, fname.split("/")[-1].split(".")[0] + ".nc")
         print(savename)
-        try:
+        retries = 4
+        for attempt in range(retries):
             try:
-                copy_thredds_file(fname, savename)
-            except:
-                fname = re.sub("sfc", "2_5km", fname)
-                fname = re.sub("ncml", "nc", fname)
-                copy_thredds_file(fname, savename)
-        except:
-            print("File not found: " + fname)
+                try:
+                    copy_thredds_file(fname, savename)
+                except:
+                    alt_fname = re.sub("sfc", "2_5km", fname)
+                    alt_fname = re.sub("ncml", "nc", alt_fname)
+                    copy_thredds_file(alt_fname, savename)
+                break
+            except Exception as e:
+                if attempt == retries - 1:
+                    print("File not found: " + fname)
+                else:
+                    backoff = min(2 ** attempt * 5, 60)
+                    print(f"Retrying in {backoff}s... ({e})")
+                    time.sleep(backoff)
 ---
 apiVersion: batch/v1
 kind: CronJob