Merged in vselfChanges (pull request #710)

VselfChanges

Approved-by: Sambit Das
Approved-by: Phani Motamarri

Author: knikhil1995

CMakeLists.txt

@@ -129,6 +129,7 @@ set_property(TARGET "dftfe_exe" PROPERTY OUTPUT_NAME "dftfe")
 SET(TARGET_SRC
   ./src/dftfeWrapper.cc
   ./src/dft/dft.cc
+  ./src/dft/analyticSmearedLoadManager.cc
   ./src/dft/vselfBinsManager.cc
   ./src/dft/energyCalculator.cc
   ./src/dft/energyDensity.cc

helpers/NERSCPerlmutter/jobscripts/complexCPUtests.slurm

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH -A m5214
+#SBATCH -C cpu
+#SBATCH -q regular
+#SBATCH --job-name complexCPUtests
+#SBATCH -t 04:00:00
+#SBATCH -N 1
+
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+
+BUILD_DIR=/global/homes/r/rezgar/dftfe/build/releaseCPU/complex
+
+cd $BUILD_DIR
+
+ctest -E "TiAl_hubbard|bccFe_01_a|bccFe_01_b|bccFe_afm" --output-on-failure 2>&1 | tee ctest-complex-perlmutter-CPU.out

include/dftfe/FEBasisOperations.h

@@ -332,6 +332,12 @@ namespace dftfe
                         const bool isResizeTempStorageForCellMatrices,
                         const bool isResizeTempStorageForIntegralEvaluations);
 
+      /**
+       * @brief Clears the internal temporary storage.
+       */
+      void
+      clearTempStorage();
+
       /**
        * @brief Number of quadrature points per cell for the quadratureID set in reinit.
        */

include/dftfe/MPICommunicatorP2P.h

@@ -69,6 +69,21 @@ namespace dftfe
           std::shared_ptr<const MPIPatternP2P<memorySpace>> mpiPatternP2P,
           const dftfe::uInt                                 blockSize);
 
+        /*
+         * @Brief Completes any non-blocking communication that was started
+         * (through one of the *Begin() calls) but never finished (through the
+         * matching *End() call), and reports it on std::cerr. Such an operation
+         * would otherwise leak its MPI_Request objects, which are drawn from a
+         * finite pool inside the MPI implementation.
+         *
+         * @note This reports rather than throws. A destructor is implicitly
+         * noexcept, and more importantly it is routinely run while another
+         * exception is unwinding the stack, where a second exception would
+         * call std::terminate and discard the original error. The unmatched
+         * Begin()/End() itself is caught by the checks in those functions.
+         */
+        ~MPICommunicatorP2P();
+
         void
         updateGhostValues(MemoryStorage<ValueType, memorySpace> &dataArray,
                           const dftfe::uInt communicationChannel = 0);
@@ -124,6 +139,20 @@ namespace dftfe
         void
         setCompressBitsPerValue(dftfe::uInt bpv);
 
+      private:
+        /*
+         * @Brief Helper used by the destructor. If \p inFlight is true, waits
+         * on \p requests so that the MPI_Request objects are returned to the
+         * MPI implementation, and reports the unmatched Begin() on \p opName
+         * to std::cerr.
+         */
+        void
+        reclaimPendingRequests(bool                     &inFlight,
+                               std::vector<MPI_Request> &requests,
+                               const std::string        &opName);
+
+
+
       private:
         std::shared_ptr<const MPIPatternP2P<memorySpace>> d_mpiPatternP2P;
 
@@ -207,6 +236,17 @@ namespace dftfe
         std::vector<MPI_Request> d_requestsAccumulateInsertLocallyOwned;
         MPI_Comm                 d_mpiCommunicator;
 
+        /*
+         * Each of the three operations owns a single set of MPI_Request
+         * handles, which is reused across communication channels. Starting an
+         * operation while a previous one is still in flight would silently
+         * overwrite (and thereby leak) the outstanding handles, so the flags
+         * below track the Begin()/End() pairing and are asserted on.
+         */
+        bool d_updateGhostValuesInFlight;
+        bool d_accumulateAddLocallyOwnedInFlight;
+        bool d_accumulateInsertLocallyOwnedInFlight;
+
         communicationProtocol  d_commProtocol;
         communicationPrecision d_commPrecision;
       };