fix: retry failed cartopy retrievals (#1455)

* fix: retry failed cartopy retrievals

* fix

* fix

scripts/_helpers.py

@@ -9,12 +9,15 @@
 import logging
 import os
 import re
+import time
 import urllib
-from functools import partial
+from functools import partial, wraps
 from os.path import exists
 from pathlib import Path
 from shutil import copyfile
+from typing import Callable
 
+import fiona
 import pandas as pd
 import pytz
 import requests
@@ -415,6 +418,47 @@ def progress_retrieve(url, file, disable=False):
                     t.update(len(data))
 
 
+def retry(func: Callable) -> Callable:
+    """
+    Retry decorator to run retry function on specific exceptions, before raising them.
+
+    Can for example be used for debugging issues which are hard to replicate or
+    for for handling retrieval errors.
+
+    Currently catches:
+    - fiona.errors.DriverError
+
+    Parameters
+    ----------
+    retries : int
+        Number of retries before raising the exception.
+    delay : int
+        Delay between retries in seconds.
+
+    Returns
+    -------
+    callable
+        A decorator function that can be used to wrap the function to be retried.
+    """
+    retries = 3
+    delay = 5
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        for attempt in range(retries):
+            try:
+                return func(*args, **kwargs)
+            except fiona.errors.DriverError as e:
+                logger.warning(
+                    f"Attempt {attempt + 1} failed: {type(e).__name__} - {e}. "
+                    f"Retrying..."
+                )
+                time.sleep(delay)
+        raise Exception("Retrieval retries exhausted.")
+
+    return wrapper
+
+
 def mock_snakemake(
     rulename,
     root_dir=None,

scripts/plot_gas_network.py

@@ -13,13 +13,14 @@
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, set_scenario_config
+from _helpers import configure_logging, retry, set_scenario_config
 from plot_power_network import assign_location, load_projection
 from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches
 
 logger = logging.getLogger(__name__)
 
 
+@retry
 def plot_ch4_map(n):
     # if "gas pipeline" not in n.links.carrier.unique():
     #     return

scripts/plot_hydrogen_network.py

@@ -13,7 +13,7 @@
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, set_scenario_config
+from _helpers import configure_logging, retry, set_scenario_config
 from plot_power_network import assign_location, load_projection
 from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches
 
@@ -43,6 +43,7 @@ def group_pipes(df, drop_direction=False):
     )
 
 
+@retry
 def plot_h2_map(n, regions):
     # if "H2 pipeline" not in n.links.carrier.unique():
     #     return

scripts/plot_power_network.py

@@ -14,7 +14,7 @@
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, rename_techs, set_scenario_config
+from _helpers import configure_logging, rename_techs, retry, set_scenario_config
 from plot_summary import preferred_order
 from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches
 
@@ -62,6 +62,7 @@ def load_projection(plotting_params):
     return proj_func(**proj_kwargs)
 
 
+@retry
 def plot_map(
     n,
     components=["links", "stores", "storage_units", "generators"],

scripts/plot_power_network_perfect.py

@@ -13,14 +13,15 @@
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, set_scenario_config
+from _helpers import configure_logging, retry, set_scenario_config
 from plot_power_network import assign_location, load_projection, rename_techs_tyndp
 from plot_summary import preferred_order
 from pypsa.plot import add_legend_circles, add_legend_lines
 
 logger = logging.getLogger(__name__)
 
 
+@retry
 def plot_map_perfect(
     n,
     components=["Link", "Store", "StorageUnit", "Generator"],