Merge branch 'publicGithubDevelop' of https://bitbucket.org/dftfedevelopers/dftfe into BFGSPreconOpt

Author: knikhil1995

CMakeLists.txt

@@ -548,7 +548,7 @@ ENDIF()
 #
 # ELPA. Pass path to -DCMAKE_PREFIX_PATH
 #
-find_package(ELPA 2022 REQUIRED)
+find_package(ELPA 2024 REQUIRED)
 MESSAGE(STATUS "Will Link With ELPA")
 target_link_libraries(${TARGETLIB} PUBLIC ELPA::ELPA)
 

README.md

@@ -15,7 +15,7 @@ DFT-FE code builds on top of the deal.II library for everything that has to do w
 The steps to install the necessary dependencies and DFT-FE itself are described in the *Installation* section of the DFT-FE manual (compile doc/manual/manual.tex or download the development version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-develop.pdf)). 
 
 We have created several shell based installation scripts for the development version of DFT-FE (`publicGithubDevelop` branch) on various machines:
-  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScript)
+  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScriptROCM6)
   - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterScript)
   - [ALCF Polaris](https://github.com/dftfeDevelopers/install_DFTFE/tree/polarisScript)
   - [UMICH Greatlakes](https://github.com/dftfeDevelopers/install_DFTFE/tree/greatlakesScript)

cmake/FindELPA.cmake

@@ -18,7 +18,7 @@ find_package(PkgConfig)
 # elpa-2020.05.001.pc
 set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH TRUE)
 foreach(pkg elpa_openmp elpa) # prioritize elpa_openmp
-    foreach(ver 2022.11.001)
+    foreach(ver 2024.03.001)
         pkg_search_module(PC_ELPA ${pkg})
         if(PC_ELPA_FOUND)
             break()

doc/manual/installation.tex

@@ -59,7 +59,7 @@ \subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, Sc
 \item {\bf ScaLAPACK}: ScaLAPACK library is used by DFT-FE via ELPA for its parallel linear algebra routines involving dense matrices, as well being a dependency for ELPA. \textcolor{red}{\bf If Intel MKL math library is available, please skip this step, as the ScaLAPACK libraries therein can be used directly.} If Intel MKL math library is not available, Netlib ScaLAPACK \url{http://www.netlib.org/scalapack/} needs to be installed using the instructions below. Download the current release version (2.2.0) from \url{http://www.netlib.org/scalapack/#\_software}, and build a shared library (use \verb|BUILD_SHARED_LIBS=ON|, \verb|BUILD_STATIC_LIBS=OFF| and \verb|BUILD_TESTING=OFF|) installation of ScaLAPACK using cmake. We recommend using the ccmake gui interface for the installation.  Further, use the appropriate compilers for \verb|CMAKE_C_COMPILER| and \verb|CMAKE_FORTRAN_COMPILER|, and also use \verb|-fPIC| flag for \verb|CMAKE_C_FLAGS| and \verb|-fPIC -fallow-argument-mismatch| for \verb|CMAKE_Fortran_FLAGS|. For best performance, ScaLAPACK must be linked to optimized BLAS-LAPACK libraries by using\\ \verb|USE_OPTIMIZED_LAPACK_BLAS=ON|, and providing external paths to BLAS-LAPACK libraries (MKL, OpenBlas, ESSL etc.) during the cmake configuration. 
 %Alternatively one can also use the python based installer~\url{http://www.netlib.org/scalapack/scalapack_installer.tgz} for Linux.
 
-\item {\bf ELPA}: ELPA library is used by DFT-FE for its parallel linear algebra routines involving dense matrices. ELPA requires the ScaLAPACK library (see above) as a dependency. Download the latest version elpa-2022.11.001 from \url{https://elpa.mpcdf.mpg.de/software/} and follow the installation instructions in there. Example of ELPA installation on UMICH Greatlakes supercomputer with GNU compiler, Open MPI library, and Intel MKL math library:
+\item {\bf ELPA}: ELPA library is used by DFT-FE for its parallel linear algebra routines involving dense matrices. ELPA requires the ScaLAPACK library (see above) as a dependency. Download the latest version elpa-2024.03.001 from \url{https://elpa.mpcdf.mpg.de/software/} and follow the installation instructions in there. Example of ELPA installation on UMICH Greatlakes supercomputer with GNU compiler, Open MPI library, and Intel MKL math library:
 \begin{verbatim}
 $ cd elpaDir
 $ mkdir build

pseudoConverters/pseudoPotentialToDftfeConverter.cc

@@ -606,14 +606,12 @@ namespace dftfe
     }
     void
     xmltoOrbitalFile(std::string file_path_in, std::string file_path_out)
-    {
-      // Extracting radial coordinates
+    { // Extracting radial coordinates
       std::vector<double>      radial_coord;
       std::vector<std::string> radial_tag;
       radial_tag.push_back("PP_MESH");
       radial_tag.push_back("PP_R");
       radial_coord = XmlTagReaderMain(radial_tag, file_path_in);
-
       std::vector<std::string> pswfc_tag;
       pswfc_tag.push_back("PP_PSWFC");
       for (int i = 1; i <= xmlNodeChildCount(pswfc_tag, file_path_in); i++)
@@ -640,13 +638,32 @@ namespace dftfe
             {
               index = std::distance(attr_type.begin(), it);
             }
-          std::string orbital_string = attr_value[index];
-          for (auto &w : orbital_string)
+          std::string orbital_string_nl = attr_value[index];
+          for (auto &w : orbital_string_nl)
             {
               w = tolower(w);
             }
+          char n = orbital_string_nl[0];
+          char l;
+          if (orbital_string_nl[1] == 's')
+            {
+              l = '0';
+            }
+          if (orbital_string_nl[1] == 'p')
+            {
+              l = '1';
+            }
+          if (orbital_string_nl[1] == 'd')
+            {
+              l = '2';
+            }
+          if (orbital_string_nl[1] == 'f')
+            {
+              l = '3';
+            }
+          std::string  orbital_string = "psi";
           std::fstream file;
-          file.open(file_path_out + "/" + orbital_string + ".dat",
+          file.open(file_path_out + "/" + orbital_string + n + l + ".inp",
                     std::ios::out);
           file << std::setprecision(12);
           if (file.is_open())

setupDevelopPetsc.sh

@@ -24,7 +24,7 @@ libxcDir="/home/vikramg/DFT-softwares-gcc/libxc/libxc-5.2.3/install_libxc5.2.3"
 spglibDir="/home/vikramg/DFT-softwares-gcc/spglib/install"
 xmlIncludeDir="/usr/include/libxml2"
 xmlLibDir="/usr/lib64"
-ELPA_PATH="/home/vikramg/DFT-softwares-gcc/elpa/install202211"
+ELPA_PATH="/home/vikramg/DFT-softwares-gcc/elpa/install2024"
 
 
 #Paths for optional external libraries

src/dft/dft.cc

@@ -743,7 +743,29 @@ namespace dftfe
                   << std::endl;
           }
       }
+    // convert pseudopotential files in upf format to dftfe format
+    if (d_dftParamsPtr->verbosity >= 1)
+      {
+        pcout
+          << std::endl
+          << "Reading Pseudo-potential data for each atom from the list given in : "
+          << d_dftParamsPtr->pseudoPotentialFile << std::endl;
+      }
+
+    int              nlccFlag = 0;
+    std::vector<int> pspFlags(2, 0);
+    if (dealii::Utilities::MPI::this_mpi_process(d_mpiCommParent) == 0 &&
+        d_dftParamsPtr->isPseudopotential == true)
+      pspFlags = pseudoUtils::convert(d_dftParamsPtr->pseudoPotentialFile,
+                                      d_dftfeScratchFolderName,
+                                      d_dftParamsPtr->verbosity,
+                                      d_dftParamsPtr->natomTypes,
+                                      d_dftParamsPtr->pseudoTestsFlag);
 
+    nlccFlag = pspFlags[0];
+    nlccFlag = dealii::Utilities::MPI::sum(nlccFlag, d_mpiCommParent);
+    if (nlccFlag > 0 && d_dftParamsPtr->isPseudopotential == true)
+      d_dftParamsPtr->nonLinearCoreCorrection = true;
     // estimate total number of wave functions from atomic orbital filling
     if (d_dftParamsPtr->startingWFCType == "ATOMIC")
       determineOrbitalFilling();
@@ -803,29 +825,7 @@ namespace dftfe
                                           d_numEigenValuesRR);
       }
 
-    // convert pseudopotential files in upf format to dftfe format
-    if (d_dftParamsPtr->verbosity >= 1)
-      {
-        pcout
-          << std::endl
-          << "Reading Pseudo-potential data for each atom from the list given in : "
-          << d_dftParamsPtr->pseudoPotentialFile << std::endl;
-      }
-
-    int              nlccFlag = 0;
-    std::vector<int> pspFlags(2, 0);
-    if (dealii::Utilities::MPI::this_mpi_process(d_mpiCommParent) == 0 &&
-        d_dftParamsPtr->isPseudopotential == true)
-      pspFlags = pseudoUtils::convert(d_dftParamsPtr->pseudoPotentialFile,
-                                      d_dftfeScratchFolderName,
-                                      d_dftParamsPtr->verbosity,
-                                      d_dftParamsPtr->natomTypes,
-                                      d_dftParamsPtr->pseudoTestsFlag);
 
-    nlccFlag = pspFlags[0];
-    nlccFlag = dealii::Utilities::MPI::sum(nlccFlag, d_mpiCommParent);
-    if (nlccFlag > 0 && d_dftParamsPtr->isPseudopotential == true)
-      d_dftParamsPtr->nonLinearCoreCorrection = true;
     if (d_dftParamsPtr->isPseudopotential == true)
       {
         // pcout<<"dft.cc 827 ONCV Number of cells DEBUG:

src/dft/initElectronicFields.cc

@@ -92,26 +92,28 @@ namespace dftfe
         mpi_communicator, "Overloaded constraint matrices initialized");
 
     //
-    // initialize density and PSI/ interpolate from previous ground state
-    // solution
+    // initialize PSI and density
     //
-    for (unsigned int kPoint = 0;
-         kPoint < (1 + d_dftParamsPtr->spinPolarized) * d_kPointWeights.size();
-         ++kPoint)
+
+    AssertThrow(
+      (1 + d_dftParamsPtr->spinPolarized) * d_kPointWeights.size() *
+          d_numEigenValues <
+        INT_MAX / matrix_free_data.get_vector_partitioner()->local_size(),
+      dealii::ExcMessage(
+        "DFT-FE error: size of local wavefunctions storage exceeds integer bounds. Please increase number of MPI tasks"));
+
+    d_eigenVectorsFlattenedHost.resize(
+      (d_numEigenValues *
+       matrix_free_data.get_vector_partitioner()->local_size()) *
+        (1 + d_dftParamsPtr->spinPolarized) * d_kPointWeights.size(),
+      dataTypes::number(0.0));
+    if (d_numEigenValuesRR != d_numEigenValues)
       {
-        d_eigenVectorsFlattenedHost.resize(
-          (d_numEigenValues *
-           matrix_free_data.get_vector_partitioner()->local_size()) *
+        d_eigenVectorsRotFracDensityFlattenedHost.resize(
+          d_numEigenValuesRR *
+            matrix_free_data.get_vector_partitioner()->local_size() *
             (1 + d_dftParamsPtr->spinPolarized) * d_kPointWeights.size(),
           dataTypes::number(0.0));
-        if (d_numEigenValuesRR != d_numEigenValues)
-          {
-            d_eigenVectorsRotFracDensityFlattenedHost.resize(
-              d_numEigenValuesRR *
-                matrix_free_data.get_vector_partitioner()->local_size() *
-                (1 + d_dftParamsPtr->spinPolarized) * d_kPointWeights.size(),
-              dataTypes::number(0.0));
-          }
       }
 
     pcout << std::endl

src/dft/psiInitialGuess.cc

@@ -53,7 +53,7 @@ namespace dftfe
     //
     char psiFile[256];
 
-    if (d_dftParamsPtr->isPseudopotential)
+    if (d_dftParamsPtr->isPseudopotential && d_dftParamsPtr->pseudoTestsFlag)
       // if(d_dftParamsPtr->pseudoProjector==2)
       // sprintf(psiFile,
       // "%s/data/electronicStructure/pseudoPotential/z%u/oncv/singleAtomData/psi%u%u.inp",
@@ -65,7 +65,12 @@ namespace dftfe
         Z,
         n,
         l);
-
+    else if (d_dftParamsPtr->isPseudopotential &&
+             !d_dftParamsPtr->pseudoTestsFlag)
+      strcpy(psiFile,
+             (d_dftfeScratchFolderName + "/z" + std::to_string(Z) + "/" +
+              "psi" + std::to_string(n) + std::to_string(l) + ".inp")
+               .c_str());
     else
       sprintf(
         psiFile,

utils/dftParameters.cc

@@ -1260,7 +1260,7 @@ namespace dftfe
     useMixedPrecXTHXSpectrumSplit                  = false;
     useMixedPrecSubspaceRotRR                      = false;
     useMixedPrecCommunOnlyXTHXCGSO                 = false;
-    spectrumSplitStartingScfIter                   = 1;
+    spectrumSplitStartingScfIter                   = 0;
     useELPA                                        = false;
     constraintsParallelCheck                       = true;
     createConstraintsFromSerialDofhandler          = true;
@@ -2013,6 +2013,9 @@ namespace dftfe
       {
         spinMixingEnhancementFactor = 1.0;
       }
+
+    if (numCoreWfcRR == 0)
+      spectrumSplitStartingScfIter = 10000;
   }