support -launcher=fork for mpi (#392)

Author: rscohn2

.github/workflows/ci.yml

@@ -105,7 +105,7 @@ jobs:
       run: srun -p pvc-shared scripts/devcloud-build.sh
     - name: Test
       # mpi does not like the way SLURM sets environment variables
-      run: srun -p pvc-shared /usr/bin/env -i scripts/devcloud-run.sh
+      run: srun -p pvc-shared scripts/devcloud-run.sh
 
   publish:
     needs: [checks, clang, gcc, icpx]

CMakeLists.txt

@@ -20,6 +20,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   option(ENABLE_SYCL "Build sycl shp examples" OFF)
   option(ENABLE_CUDA "Build for cuda" OFF)
   option(ENABLE_FORMAT "Build with format library" ON)
+  option(ENABLE_MPIFORK "Use MPI fork launcher" OFF)
   option(GCC_TOOLCHAIN, "GCC toolchain to be used by clang-based compilers" OFF)
 
   #
@@ -28,6 +29,10 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   set(CMAKE_CXX_EXTENSIONS OFF)
   set(CMAKE_CXX_STANDARD 20)
 
+  if(ENABLE_MPIFORK)
+    set(MPIEXEC_LAUNCHFLAGS "-launcher=fork")
+  endif()
+
   if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     add_compile_options(-fconcepts-diagnostics-depth=10)
   endif()
@@ -142,9 +147,12 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(mdspan)
 
 function(add_mpi_test test_name name processes)
-  add_test(NAME ${test_name}
-           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${processes}
-                   ${MPIEXEC_PREFLAGS} ./${name} ${ARGN} COMMAND_EXPAND_LISTS)
+  add_test(
+    NAME ${test_name}
+    COMMAND
+      ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${processes}
+      ${MPIEXEC_PREFLAGS} ${MPIEXEC_LAUNCHFLAGS} ./${name} ${ARGN}
+      COMMAND_EXPAND_LISTS)
 endfunction()
 
 install(DIRECTORY include DESTINATION ${CMAKE_INSTALL_PREFIX})

scripts/devcloud-build.sh

@@ -7,5 +7,6 @@
 set -e
 hostname
 source /opt/intel/oneapi/setvars.sh
-cmake -B build -DENABLE_SYCL=on
+# devcloud requires --launcher=fork for mpi
+cmake -B build -DENABLE_SYCL=on -DENABLE_MPIFORK=on
 make -C build -j

scripts/devcloud-run.sh

@@ -8,9 +8,6 @@ set -e
 hostname
 source /opt/intel/oneapi/setvars.sh
 
-# we clear the environment to make mpi work, add back error output
-export CTEST_OUTPUT_ON_FAILURE=1
-
 # default sycl device will be GPU, if available. Only use 1 GPU:
 # workaround for devcloud multi-card not working
 ONEAPI_DEVICE_SELECTOR=level_zero:0 make -C build test