From c1007723dc78d2bb15afb9439514fd2bf0a213d7 Mon Sep 17 00:00:00 2001 From: Daniel Geisler Date: Tue, 3 Mar 2020 18:06:36 +0100 Subject: [PATCH 01/13] FIX: check if result has an attribute outputs before accessing it --- nipype/pipeline/engine/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 78d9417f9f..d8d2f1f9e8 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -291,7 +291,7 @@ def load_resultfile(results_file, resolve=True): raise FileNotFoundError(results_file) result = loadpkl(results_file) - if resolve and result.outputs: + if resolve and hasattr(result,"outputs") and result.outputs: try: outputs = result.outputs.get() except TypeError: # This is a Bunch From 75ce23df7495ba520ef5d5529e81fb92caeb0a97 Mon Sep 17 00:00:00 2001 From: Daniel Ge Date: Wed, 4 Mar 2020 16:21:42 +0100 Subject: [PATCH 02/13] Update nipype/pipeline/engine/utils.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/engine/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index d8d2f1f9e8..d7a65b74de 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -291,7 +291,7 @@ def load_resultfile(results_file, resolve=True): raise FileNotFoundError(results_file) result = loadpkl(results_file) - if resolve and hasattr(result,"outputs") and result.outputs: + if resolve and getattr(result, "outputs", None): try: outputs = result.outputs.get() except TypeError: # This is a Bunch From 7b1a2f44ac8c206106ce1d03d476d4a0b971323c Mon Sep 17 00:00:00 2001 From: Daniel Geisler Date: Sat, 7 Mar 2020 23:33:48 +0100 Subject: [PATCH 03/13] First attempt of a SGE test --- nipype/pipeline/plugins/tests/test_sgelike.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 nipype/pipeline/plugins/tests/test_sgelike.py diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py new file mode 100644 index 0000000000..4b1a9e10bd --- /dev/null +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -0,0 +1,42 @@ +from nipype.pipeline.plugins.base import SGELikeBatchManagerBase +from nipype.interfaces.utility import Function +import nipype.pipeline.engine as pe +from os.path import join +import os +from glob import glob +import pytest +from mock import patch +from tempfile import TemporaryDirectory +import subprocess + +def crasher(): + raise ValueError() + + +def submit_batchtask(self, scriptfile, node): + self._pending[1] = node.output_dir() + subprocess.call(["bash", scriptfile]) + return 1 + +def is_pending(taskid): + return False + + +@patch.object(SGELikeBatchManagerBase, '_submit_batchtask', new=submit_batchtask) +@patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) +def test_crashfile_creation(): + cur_dir = os.getcwd() + with TemporaryDirectory(prefix="test_engine_", dir=cur_dir) as tmpdirname: + pipe = pe.Workflow(name="pipe", base_dir=tmpdirname) + pipe.config["execution"]["crashdump_dir"] = tmpdirname + pipe.add_nodes([pe.Node(interface=Function(function=crasher), + name="crasher")]) + sgelike_plugin = SGELikeBatchManagerBase("") + with pytest.raises(RuntimeError) as e: + assert pipe.run(plugin=sgelike_plugin) + assert (str(e.value) == + "Workflow did not execute cleanly. Check log for details") + + crashfiles = glob(join(tmpdirname,"crash*crasher*.pklz")) + assert len(crashfiles) == 1 + os.chdir(cur_dir) From 9c8f322f381ae51f19b5f34c75c52ba944362d49 Mon Sep 17 00:00:00 2001 From: Daniel Geisler Date: Sun, 8 Mar 2020 11:14:30 +0100 Subject: [PATCH 04/13] FIX: add missing argument to is_pending mock function --- nipype/pipeline/plugins/tests/test_sgelike.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 4b1a9e10bd..40649c9af9 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -9,6 +9,7 @@ from tempfile import TemporaryDirectory import subprocess + def crasher(): raise ValueError() @@ -18,7 +19,8 @@ def submit_batchtask(self, scriptfile, node): subprocess.call(["bash", scriptfile]) return 1 -def is_pending(taskid): + +def is_pending(self, taskid): return False @@ -40,3 +42,4 @@ def test_crashfile_creation(): crashfiles = glob(join(tmpdirname,"crash*crasher*.pklz")) assert len(crashfiles) == 1 os.chdir(cur_dir) + From 57ab1080f69d24acbed9cbdf55e98e8f5d9b5a5a Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:25:29 +0100 Subject: [PATCH 05/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 40649c9af9..b973b1e841 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -5,7 +5,7 @@ import os from glob import glob import pytest -from mock import patch +from unittest.mock import patch from tempfile import TemporaryDirectory import subprocess @@ -42,4 +42,3 @@ def test_crashfile_creation(): crashfiles = glob(join(tmpdirname,"crash*crasher*.pklz")) assert len(crashfiles) == 1 os.chdir(cur_dir) - From 269ac56c21b170dce1af6ebead51dbf49302b48a Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:25:39 +0100 Subject: [PATCH 06/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index b973b1e841..27cd733b39 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -26,7 +26,7 @@ def is_pending(self, taskid): @patch.object(SGELikeBatchManagerBase, '_submit_batchtask', new=submit_batchtask) @patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) -def test_crashfile_creation(): +def test_crashfile_creation(tmp_path): cur_dir = os.getcwd() with TemporaryDirectory(prefix="test_engine_", dir=cur_dir) as tmpdirname: pipe = pe.Workflow(name="pipe", base_dir=tmpdirname) From 98aeca13a312e0326355a4a0f9ab5a9a30f02601 Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:25:50 +0100 Subject: [PATCH 07/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 27cd733b39..10c332bea1 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -27,8 +27,6 @@ def is_pending(self, taskid): @patch.object(SGELikeBatchManagerBase, '_submit_batchtask', new=submit_batchtask) @patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) def test_crashfile_creation(tmp_path): - cur_dir = os.getcwd() - with TemporaryDirectory(prefix="test_engine_", dir=cur_dir) as tmpdirname: pipe = pe.Workflow(name="pipe", base_dir=tmpdirname) pipe.config["execution"]["crashdump_dir"] = tmpdirname pipe.add_nodes([pe.Node(interface=Function(function=crasher), From 33befb1a960f3da4388005efdf5245b46ee27d8d Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:25:59 +0100 Subject: [PATCH 08/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 10c332bea1..a29d0528c7 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -39,4 +39,3 @@ def test_crashfile_creation(tmp_path): crashfiles = glob(join(tmpdirname,"crash*crasher*.pklz")) assert len(crashfiles) == 1 - os.chdir(cur_dir) From ce22e364266d814fdeba5110860248f6d96fa534 Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:26:14 +0100 Subject: [PATCH 09/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index a29d0528c7..143df8ddcc 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -27,7 +27,7 @@ def is_pending(self, taskid): @patch.object(SGELikeBatchManagerBase, '_submit_batchtask', new=submit_batchtask) @patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) def test_crashfile_creation(tmp_path): - pipe = pe.Workflow(name="pipe", base_dir=tmpdirname) + pipe = pe.Workflow(name="pipe", base_dir=str(tmp_path)) pipe.config["execution"]["crashdump_dir"] = tmpdirname pipe.add_nodes([pe.Node(interface=Function(function=crasher), name="crasher")]) From ef739f047615758e2552f793669846267cc2aa5b Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:26:23 +0100 Subject: [PATCH 10/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 143df8ddcc..025bba1133 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -28,7 +28,7 @@ def is_pending(self, taskid): @patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) def test_crashfile_creation(tmp_path): pipe = pe.Workflow(name="pipe", base_dir=str(tmp_path)) - pipe.config["execution"]["crashdump_dir"] = tmpdirname + pipe.config["execution"]["crashdump_dir"] = str(tmp_path) pipe.add_nodes([pe.Node(interface=Function(function=crasher), name="crasher")]) sgelike_plugin = SGELikeBatchManagerBase("") From 27a2472b16de5861b95238d414978033dcee77f3 Mon Sep 17 00:00:00 2001 From: Daniel Ge <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 14:26:33 +0100 Subject: [PATCH 11/13] Update nipype/pipeline/plugins/tests/test_sgelike.py Co-Authored-By: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_sgelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 025bba1133..80debae87c 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -37,5 +37,5 @@ def test_crashfile_creation(tmp_path): assert (str(e.value) == "Workflow did not execute cleanly. Check log for details") - crashfiles = glob(join(tmpdirname,"crash*crasher*.pklz")) + crashfiles = tmp_path.glob("crash*crasher*.pklz") assert len(crashfiles) == 1 From 02991da67458b879d7c6360aa6457eb3c1bd5a07 Mon Sep 17 00:00:00 2001 From: Daniel <3453485+daniel-ge@users.noreply.github.com> Date: Sun, 8 Mar 2020 15:05:28 +0100 Subject: [PATCH 12/13] FIX: get length of generator + STY: Black --- nipype/pipeline/plugins/tests/test_sgelike.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 80debae87c..02fdf263d1 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -13,8 +13,8 @@ def crasher(): raise ValueError() - -def submit_batchtask(self, scriptfile, node): + +def submit_batchtask(self, scriptfile, node): self._pending[1] = node.output_dir() subprocess.call(["bash", scriptfile]) return 1 @@ -24,18 +24,16 @@ def is_pending(self, taskid): return False -@patch.object(SGELikeBatchManagerBase, '_submit_batchtask', new=submit_batchtask) -@patch.object(SGELikeBatchManagerBase, '_is_pending', new=is_pending) +@patch.object(SGELikeBatchManagerBase, "_submit_batchtask", new=submit_batchtask) +@patch.object(SGELikeBatchManagerBase, "_is_pending", new=is_pending) def test_crashfile_creation(tmp_path): - pipe = pe.Workflow(name="pipe", base_dir=str(tmp_path)) - pipe.config["execution"]["crashdump_dir"] = str(tmp_path) - pipe.add_nodes([pe.Node(interface=Function(function=crasher), - name="crasher")]) - sgelike_plugin = SGELikeBatchManagerBase("") - with pytest.raises(RuntimeError) as e: - assert pipe.run(plugin=sgelike_plugin) - assert (str(e.value) == - "Workflow did not execute cleanly. Check log for details") - - crashfiles = tmp_path.glob("crash*crasher*.pklz") - assert len(crashfiles) == 1 + pipe = pe.Workflow(name="pipe", base_dir=str(tmp_path)) + pipe.config["execution"]["crashdump_dir"] = str(tmp_path) + pipe.add_nodes([pe.Node(interface=Function(function=crasher), name="crasher")]) + sgelike_plugin = SGELikeBatchManagerBase("") + with pytest.raises(RuntimeError) as e: + assert pipe.run(plugin=sgelike_plugin) + assert str(e.value) == "Workflow did not execute cleanly. Check log for details" + + crashfiles = tmp_path.glob("crash*crasher*.pklz") + assert len(list(crashfiles)) == 1 From 4a6d6b88a8aea55ffa3d318e4090d77556afb3cf Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 9 Mar 2020 09:51:12 -0400 Subject: [PATCH 13/13] TEST: Cleanup imports --- nipype/pipeline/plugins/tests/test_sgelike.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_sgelike.py b/nipype/pipeline/plugins/tests/test_sgelike.py index 02fdf263d1..9c7cdc1412 100644 --- a/nipype/pipeline/plugins/tests/test_sgelike.py +++ b/nipype/pipeline/plugins/tests/test_sgelike.py @@ -1,12 +1,8 @@ from nipype.pipeline.plugins.base import SGELikeBatchManagerBase from nipype.interfaces.utility import Function import nipype.pipeline.engine as pe -from os.path import join -import os -from glob import glob import pytest from unittest.mock import patch -from tempfile import TemporaryDirectory import subprocess