bug fix in freeing MPI communicator. MDI is now able to exit cleanly after solving the DFT problem

Author: dsambit

include/dftfeWrapper.h

@@ -39,7 +39,7 @@ namespace dftfe
      * after calling MPI_Init
      */
     static void
-    globalHandlesInitialize();
+    globalHandlesInitialize(const MPI_Comm &mpi_comm_world);
 
     /**
      * @brief must be called only once at end of program from all processors

src/dftfeWrapper.cc

@@ -130,9 +130,9 @@ namespace dftfe
   } // namespace internalWrapper
 
   void
-  dftfeWrapper::globalHandlesInitialize()
+  dftfeWrapper::globalHandlesInitialize(const MPI_Comm &mpi_comm_world)
   {
-    sc_init(MPI_COMM_WORLD, 0, 0, nullptr, SC_LP_SILENT);
+    sc_init(mpi_comm_world, 0, 0, nullptr, SC_LP_SILENT);
     p4est_init(nullptr, SC_LP_SILENT);
 
 #ifdef USE_PETSC
@@ -358,7 +358,13 @@ namespace dftfe
   {
     clear();
     if (mpi_comm_parent != MPI_COMM_NULL)
-      MPI_Comm_dup(mpi_comm_parent, &d_mpi_comm_parent);
+      {
+        int ierr = MPI_Comm_dup(mpi_comm_parent, &d_mpi_comm_parent);
+        if (ierr != 0)
+          {
+            throw std::runtime_error("MPI_Comm_dup failed.");
+          }
+      }
 
     createScratchFolder();
 
@@ -863,6 +869,9 @@ namespace dftfe
           delete d_dftfeParamsPtr;
         MPI_Comm_free(&d_mpi_comm_parent);
       }
+    d_dftfeBasePtr    = nullptr;
+    d_dftfeParamsPtr  = nullptr;
+    d_mpi_comm_parent = MPI_COMM_NULL;
   }
 
   void

src/main.cc

@@ -45,14 +45,9 @@ main(int argc, char *argv[])
   //
   MPI_Init(&argc, &argv);
 
-  dftfe::dftfeWrapper::globalHandlesInitialize();
-
 #if defined(DFTFE_WITH_MDI)
-
   MPI_Comm mpi_world_comm;
-
-
-  int ret;
+  int      ret;
   // Initialize MDI
   ret = MDI_Init(&argc, &argv);
   if (ret != 0)
@@ -83,8 +78,12 @@ main(int argc, char *argv[])
       throw std::runtime_error("MDI_MPI_get_world_comm failed.");
     }
 
+  dftfe::dftfeWrapper::globalHandlesInitialize(mpi_world_comm);
   dftfe::MDIEngine mdiEngine(mpi_world_comm, argc, argv);
+  dftfe::dftfeWrapper::globalHandlesFinalize();
+  MPI_Barrier(mpi_world_comm);
 #else
+  dftfe::dftfeWrapper::globalHandlesInitialize(MPI_COMM_WORLD);
   const double start = MPI_Wtime();
   int          world_rank;
   MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
@@ -205,10 +204,10 @@ main(int argc, char *argv[])
         << "============================================================================================="
         << std::endl;
     }
-#endif
 
   dftfe::dftfeWrapper::globalHandlesFinalize();
-
+  MPI_Barrier(MPI_COMM_WORLD);
+#endif
   MPI_Finalize();
   return 0;
 }

src/mdi/MDIEngine.cpp

@@ -65,6 +65,7 @@ namespace dftfe
     // root = 1 for proc 0, otherwise 0
     int rank;
     MPI_Comm_rank(d_dftfeMPIComm, &rank);
+    std::cout << "rank: " << rank << std::endl;
     d_root = (rank == 0) ? 1 : 0;
 
     // MDI setup
@@ -278,7 +279,7 @@ namespace dftfe
     else if (strcmp(command, "EXIT") == 0)
       {
         d_exit_command = true;
-
+        d_dftfeWrapper.clear();
         // -------------------------------------------------------
         // unknown command
         // -------------------------------------------------------

src/mdi/libraryMDI.cpp

@@ -75,7 +75,7 @@ MDI_Plugin_init_dftfe()
     MPI_Abort(MPI_COMM_WORLD, 1);
 
   // open DFT-FE
-  dftfe::dftfeWrapper::globalHandlesInitialize();
+  // dftfe::dftfeWrapper::globalHandlesInitialize();
 
   // launch MDI engine in endless loop
   dftfe::MDIEngine mdiEngine(mpi_world_comm, mdi_argc, mdi_argv);