diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index aeff5f12da..406db3f5ab 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -14,6 +14,7 @@ from copy import deepcopy from glob import glob from logging import INFO +import time from tempfile import mkdtemp @@ -605,11 +606,28 @@ def _get_inputs(self): output_value = outputs.dictcopy()[output_name] logger.debug("output: %s", output_name) - try: - self.set_input(key, deepcopy(output_value)) - except traits.TraitError as e: + traits_err = None + # input_tries = 1 # Default no reties + # input_retry_delay = 5 + # input_retry_exponential_backoff_factor = 1 + input_tries = int(self.config["execution"]["input_tries"]) + input_retry_delay = int(self.config["execution"]["input_retry_delay"]) + input_retry_exp_backoff_factor = int(self.config["execution"]["input_retry_exp_backoff_factor"]) + for try_n in range(input_tries): + try: + self.set_input(key, deepcopy(output_value)) + break + except traits.TraitError as e: + traits_err = e + if input_tries != 1: + sleep_time = input_retry_delay*try_n*input_retry_exp_backoff_factor + logger.warning("Input set failed for : {0} -> {1}. Retrying in {2} secs. ".format(key, + results_fname, + sleep_time)) + time.sleep(sleep_time) + if traits_err is not None: msg = ( - e.args[0], + traits_err.args[0], "", "Error setting node input:", "Node: %s" % self.name, @@ -617,7 +635,7 @@ def _get_inputs(self): "results_file: %s" % results_fname, "value: %s" % str(output_value), ) - e.args = ("\n".join(msg),) + traits_err.args = ("\n".join(msg),) raise # Successfully set inputs diff --git a/nipype/utils/config.py b/nipype/utils/config.py index e7020eb30d..1508541bcc 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -62,6 +62,9 @@ poll_sleep_duration = 2 xvfb_max_wait = 10 check_version = true +input_tries = 1 +input_retry_delay = 5 +input_retry_exp_backoff_factor = 1 [monitoring] enabled = false