Merge pull request #3434 from heplesser/easier_other

Simplify handling of OTHER labels in new-style mpitests

Author: heplesser

testsuite/pytests/sli2py_mpi/mpi_test_wrapper.py

@@ -28,7 +28,7 @@
 be fixed while the number of MPI processes is varied, but this is not required.
 
 - The process is managed by subclasses of the `MPITestWrapper` base class
-- Each test file must contain exactly one test function
+- Each test file must contain **exactly one** test function
     - The test function must be decorated with a subclass of `MPITestWrapper`
     - The wrapper will write a modified version of the test file as `runner.py`
       to a temporary directory and mpirun it from there; results are collected
@@ -99,9 +99,14 @@ class MPITestWrapper:
     """
 
     RUNNER = "runner.py"
+
+    # Formats for output file names to be written by NEST and by the evaluation function.
+    # The first placeholder will be filled with the number of processes used.
+    # SPIKE and MULTI labels are passed to spike recorder/multimeter, which add "-{Rank}.dat" automatically.
+    # For OTHER, the test function needs to provide the Rank explicitly.
     SPIKE_LABEL = "spike-{}"
     MULTI_LABEL = "multi-{}"
-    OTHER_LABEL = "other-{}"
+    OTHER_LABEL = "other-{}-{}.dat"
 
     RUNNER_TEMPLATE = textwrap.dedent(
         """\
@@ -195,13 +200,18 @@ def wrapper(*args, **kwargs):
         return wrapper
 
     def _collect_result_by_label(self, tmpdirpath, label):
+        # Build complete patterns here including the rank part and ending
+        if not label.endswith("-{}-{}.dat"):
+            assert label.endswith("-{}")
+            label += "-{}.dat"
+
         try:
-            next(tmpdirpath.glob(f"{label.format('*')}.dat"))
+            next(tmpdirpath.glob(label.format("*", "*")))
         except StopIteration:
             return None  # no data for this label
 
         return {
-            n_procs: [pd.read_csv(f, sep="\t", comment="#") for f in tmpdirpath.glob(f"{label.format(n_procs)}-*.dat")]
+            n_procs: [pd.read_csv(f, sep="\t", comment="#") for f in tmpdirpath.glob(label.format(n_procs, "*"))]
             for n_procs in self._procs_lst
         }
 

testsuite/pytests/sli2py_mpi/test_all_to_all.py

@@ -25,7 +25,7 @@
 from mpi_test_wrapper import MPITestAssertEqual
 
 
-# Parametrization over the number of nodes here only so show hat it works
+# Parametrization over the number of nodes here only to show hat it works
 @pytest.mark.parametrize("N", [4, 7])
 @MPITestAssertEqual([1, 4], debug=False)
 def test_all_to_all(N):
@@ -42,4 +42,4 @@ def test_all_to_all(N):
     nest.Connect(nrns, nrns, "all_to_all")
 
     conns = nest.GetConnections().get(output="pandas").drop(labels=["target_thread", "port"], axis=1)
-    conns.to_csv(OTHER_LABEL.format(nest.num_processes) + f"-{nest.Rank()}.dat", index=False)  # noqa: F821
+    conns.to_csv(OTHER_LABEL.format(nest.num_processes, nest.Rank()), index=False)  # noqa: F821

testsuite/pytests/sli2py_mpi/test_issue_1957.py

@@ -44,7 +44,7 @@ def test_issue_1957():
     if pre_conns:
         # need to do this here, Disconnect invalidates pre_conns
         df = pd.DataFrame.from_dict(pre_conns.get()).drop(labels="target_thread", axis=1)
-        df.to_csv(OTHER_LABEL.format(nest.num_processes) + f"-{nest.Rank()}.dat", index=False)  # noqa: F821
+        df.to_csv(OTHER_LABEL.format(nest.num_processes, nest.Rank()), index=False)  # noqa: F821
 
     nest.Disconnect(nrn, nrn)
     nest.Disconnect(nrn, nrn)

testsuite/pytests/sli2py_mpi/test_mini_brunel_ps.py

@@ -100,4 +100,4 @@ def test_mini_brunel_ps():
     nest.Simulate(400)
 
     # Uncomment next line to provoke test failure
-    # nest.Simulate(200 if -nest.num_processes == 1 else 400)
+    # nest.Simulate(200 if nest.num_processes == 1 else 400)