[coll-comm] disable neighbor comm if open-mpi version < 4.1.0

Signed-off-by: Marcel Koch <[email protected]>

Author: MarcelKoch

CMakeLists.txt

@@ -360,7 +360,7 @@ if(GINKGO_BUILD_HWLOC AND (MSVC OR WIN32 OR CYGWIN OR APPLE))
 endif()
 
 set(GINKGO_HAVE_GPU_AWARE_MPI OFF)
-set(GINKGO_FORCE_SPMV_BLOCKING_COMM OFF)
+set(GINKGO_HAVE_OPENMPI_PRE_4_1_X OFF)
 if(GINKGO_BUILD_MPI)
     find_package(MPI 3.1 COMPONENTS CXX REQUIRED)
     if(GINKGO_FORCE_GPU_AWARE_MPI)
@@ -388,13 +388,15 @@ if(GINKGO_BUILD_MPI)
         if(NOT valid_openmpi_version)
             message(
                 WARNING
-                "OpenMPI v4.0.x has a bug that forces us to use blocking communication in our distributed "
+                "OpenMPI v4.0.x has several bugs that forces us to use non-optimal communication in our distributed "
                 "matrix class. To enable faster, non-blocking communication, consider updating your OpenMPI version or "
                 "switch to a different vendor."
             )
-            set(GINKGO_FORCE_SPMV_BLOCKING_COMM ON)
+            set(GINKGO_HAVE_OPENMPI_PRE_4_1_X ON)
         endif()
+        unset(valid_openmpi_version)
     endif()
+    unset(uses_openmpi)
 endif()
 
 # Try to find the third party packages before using our subdirectories

core/CMakeLists.txt

@@ -150,10 +150,15 @@ if(GINKGO_BUILD_MPI)
             distributed/dense_communicator.cpp
             distributed/matrix.cpp
             distributed/partition_helpers.cpp
-            distributed/neighborhood_communicator.cpp
             distributed/vector.cpp
             distributed/preconditioner/schwarz.cpp
     )
+    if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X)
+        target_sources(
+            ${ginkgo_core}
+            PRIVATE distributed/neighborhood_communicator.cpp
+        )
+    endif()
 endif()
 
 # MSVC or CLANG/msys2 with shared: make ginkgo be the major library

core/distributed/dense_communicator.cpp

@@ -80,7 +80,7 @@ request DenseCommunicator::i_all_to_all_v(std::shared_ptr<const Executor> exec,
                                           void* recv_buffer,
                                           MPI_Datatype recv_type) const
 {
-#ifdef GINKGO_FORCE_SPMV_BLOCKING_COMM
+#ifdef GINKGO_HAVE_OPENMPI_PRE_4_1_X
     comm_.all_to_all_v(exec, send_buffer, send_sizes_.data(),
                        send_offsets_.data(), send_type, recv_buffer,
                        recv_sizes_.data(), recv_offsets_.data(), recv_type);

core/distributed/matrix.cpp

@@ -478,7 +478,7 @@ mpi::request Matrix<ValueType, LocalIndexType, GlobalIndexType>::communicate(
     auto recv_ptr = use_host_buffer ? host_recv_buffer_->get_values()
                                     : recv_buffer_->get_values();
     exec->synchronize();
-#ifdef GINKGO_FORCE_SPMV_BLOCKING_COMM
+#ifdef GINKGO_HAVE_OPENMPI_PRE_4_1_X
     comm.all_to_all_v(use_host_buffer ? exec->get_master() : exec, send_ptr,
                       send_sizes_.data(), send_offsets_.data(), type.get(),
                       recv_ptr, recv_sizes_.data(), recv_offsets_.data(),

core/test/mpi/distributed/CMakeLists.txt

@@ -1,8 +1,11 @@
 ginkgo_create_test(helpers MPI_SIZE 1)
 ginkgo_create_test(matrix MPI_SIZE 1)
 ginkgo_create_test(dense_communicator MPI_SIZE 6)
-ginkgo_create_test(neighborhood_communicator MPI_SIZE 6)
 ginkgo_create_test(vector_cache MPI_SIZE 3)
 
+if(NOT GINKGO_HAVE_OPENMPI_PRE_4_1_X)
+    ginkgo_create_test(neighborhood_communicator MPI_SIZE 6)
+endif()
+
 add_subdirectory(preconditioner)
 add_subdirectory(solver)

include/ginkgo/config.hpp.in

@@ -128,7 +128,7 @@
 
 /* Do we need to use blocking communication in our SpMV? */
 // clang-format off
-#cmakedefine GINKGO_FORCE_SPMV_BLOCKING_COMM
+#cmakedefine01 GINKGO_HAVE_OPENMPI_PRE_4_1_X
 // clang-format on
 
 

include/ginkgo/core/distributed/neighborhood_communicator.hpp

@@ -9,7 +9,7 @@
 #include <ginkgo/config.hpp>
 
 
-#if GINKGO_BUILD_MPI
+#if GINKGO_BUILD_MPI && !GINKGO_HAVE_OPENMPI_PRE_4_1_X
 
 #include <ginkgo/core/base/mpi.hpp>
 #include <ginkgo/core/distributed/collective_communicator.hpp>