Skip to content

Commit 6165bca

Browse files
dsambitdsambit
committed
cleanups
1 parent 9570b54 commit 6165bca

File tree

5 files changed

+83
-168
lines changed

5 files changed

+83
-168
lines changed

‎helpers/UMGreatLakes/setupDevelopPetscMDIInstall.sh

+6-6
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ SRC=`dirname $0` # location of source directory
1818
# and optimization flag
1919

2020
#Paths for required external libraries
21-
dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/dealiiDevCustomized/install_gcc8.2.0_openmpi4.0.6_real_cpu"
22-
dealiiPetscComplexDir="/home/vikramg/DFT-FE-softwares/dealiiDevCustomized/install_gcc8.2.0_openmpi4.0.6_complex_cpu"
23-
alglibDir="/home/vikramg/DFT-FE-softwares/alglibGCC/cpp/src"
24-
libxcDir="/home/vikramg/DFT-FE-softwares/libxc/install_libxc5.1.5"
25-
spglibDir="/home/vikramg/DFT-FE-softwares/spglibGCC"
21+
dealiiPetscRealDir="/home/vikramg/DFT-softwares-gcc/dealii/install_real_cpu"
22+
dealiiPetscComplexDir="/home/vikramg/DFT-softwares-gcc/dealii/install_complex_cpu"
23+
alglibDir="/home/vikramg/DFT-softwares-gcc/alglib/alglib-cpp/src"
24+
libxcDir="/home/vikramg/DFT-softwares-gcc/libxc/install_libxc5.2.3"
25+
spglibDir="/home/vikramg/DFT-softwares-gcc/spglib/install"
2626
xmlIncludeDir="/usr/include/libxml2"
2727
xmlLibDir="/usr/lib64"
28-
ELPA_PATH="/home/vikramg/DFT-FE-softwares/elpa/elpa2021.05.002_gcc8.2.0_openmpi4.0.6_install"
28+
ELPA_PATH="/home/vikramg/DFT-softwares-gcc/elpa/install"
2929

3030
#Paths for optional external libraries
3131
NCCL_PATH=""

‎setupDevelopPetsc.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ ELPA_PATH="/home/vikramg/DFT-softwares-gcc/elpa/install"
2929

3030
#Paths for optional external libraries
3131
NCCL_PATH=""
32+
mdiPath="/home/dsambit/exaaltInterfaceRelated/MDI_Library/install"
3233

3334

3435
#Toggle GPU compilation
@@ -69,6 +70,7 @@ function cmake_real() {
6970
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
7071
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
7172
-DXML_INCLUDE_DIR=$xmlIncludeDir \
73+
-DMDI_PATH=$mdiPath \
7274
-DWITH_NCCL=$withNCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$NCCL_PATH"\
7375
-DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DCMAKE_CUDA_FLAGS="$cuda_flags"\
7476
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile \
@@ -84,7 +86,8 @@ function cmake_cplx() {
8486
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
8587
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
8688
-DXML_INCLUDE_DIR=$xmlIncludeDir \
87-
-DCMAKE_PREFIX_PATH="$ELPA_PATH" \
89+
-DMDI_PATH=$mdiPath\
90+
-DWITH_NCCL=$withNCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$NCCL_PATH" \
8891
-DWITH_COMPLEX=ON \
8992
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile \
9093
-DHIGHERQUAD_PSP=$withHigherQuadPSP\

‎setupDevelopPetscMDIInstall.sh

-117
This file was deleted.

‎setupUserPetsc.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ ELPA_PATH="/home/vikramg/DFT-softwares-gcc/elpa/install"
2929

3030
#Paths for optional external libraries
3131
NCCL_PATH=""
32-
32+
mdiPath=""
3333

3434
#Toggle GPU compilation
3535
withGPU=OFF
@@ -69,6 +69,7 @@ function cmake_real() {
6969
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
7070
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
7171
-DXML_INCLUDE_DIR=$xmlIncludeDir \
72+
-DMDI_PATH=$mdiPath \
7273
-DWITH_NCCL=$withNCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$NCCL_PATH"\
7374
-DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DCMAKE_CUDA_FLAGS="$cuda_flags"\
7475
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile \
@@ -84,7 +85,8 @@ function cmake_cplx() {
8485
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
8586
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
8687
-DXML_INCLUDE_DIR=$xmlIncludeDir \
87-
-DCMAKE_PREFIX_PATH="$ELPA_PATH" \
88+
-DMDI_PATH=$mdiPath\
89+
-DWITH_NCCL=$withNCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH" \
8890
-DWITH_COMPLEX=ON \
8991
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile \
9092
-DHIGHERQUAD_PSP=$withHigherQuadPSP\

‎src/mdi/MDIEngine.cpp

+69-42
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ namespace dftfe
6565
// root = 1 for proc 0, otherwise 0
6666
int rank;
6767
MPI_Comm_rank(d_dftfeMPIComm, &rank);
68-
std::cout << "rank: " << rank << std::endl;
6968
d_root = (rank == 0) ? 1 : 0;
7069

7170
// MDI setup
@@ -463,13 +462,17 @@ namespace dftfe
463462
{
464463
d_actionflag = 0;
465464
d_flag_cell = 1;
466-
int ierr = MDI_Recv(d_sys_cell, 9, MDI_DOUBLE, d_mdicomm);
467-
if (ierr)
468-
AssertThrow(false, dealii::ExcMessage("MDI: >CELL data"));
469-
// error->all(FLERR, "MDI: >CELL data");
465+
466+
if (d_root == 1)
467+
{
468+
int ierr = MDI_Recv(d_sys_cell, 9, MDI_DOUBLE, d_mdicomm);
469+
if (ierr)
470+
AssertThrow(false, dealii::ExcMessage("MDI: >CELL data"));
471+
}
472+
470473
MPI_Bcast(d_sys_cell, 9, MPI_DOUBLE, 0, d_dftfeMPIComm);
471-
for (int i = 0; i < 9; i++)
472-
std::cout << "cell: " << d_sys_cell[i] << std::endl;
474+
// for (int i = 0; i < 9; i++)
475+
// std::cout << "cell: " << d_sys_cell[i] << std::endl;
473476
}
474477

475478
/* ----------------------------------------------------------------------
@@ -485,10 +488,14 @@ namespace dftfe
485488
{
486489
d_actionflag = 0;
487490
d_flag_cell_displ = 1;
488-
int ierr = MDI_Recv(d_sys_cell_displ, 3, MDI_DOUBLE, d_mdicomm);
489-
if (ierr)
490-
AssertThrow(false, dealii::ExcMessage("MDI: >CELL_DISPLS data"));
491-
// error->all(FLERR, "MDI: >CELL_DISPLS data");
491+
492+
if (d_root == 1)
493+
{
494+
int ierr = MDI_Recv(d_sys_cell_displ, 3, MDI_DOUBLE, d_mdicomm);
495+
if (ierr)
496+
AssertThrow(false, dealii::ExcMessage("MDI: >CELL_DISPLS data"));
497+
}
498+
492499
MPI_Bcast(d_sys_cell_displ, 3, MPI_DOUBLE, 0, d_dftfeMPIComm);
493500
}
494501

@@ -503,14 +510,18 @@ namespace dftfe
503510
d_flag_coords = 1;
504511
int n = 3 * d_sys_natoms;
505512
d_sys_coords.resize(n);
506-
int ierr = MDI_Recv(&d_sys_coords[0], n, MDI_DOUBLE, d_mdicomm);
507-
if (ierr)
508-
AssertThrow(false, dealii::ExcMessage("MDI: >COORDS data"));
509-
// error->all(FLERR, "MDI: >COORDS data");
513+
514+
if (d_root == 1)
515+
{
516+
int ierr = MDI_Recv(&d_sys_coords[0], n, MDI_DOUBLE, d_mdicomm);
517+
if (ierr)
518+
AssertThrow(false, dealii::ExcMessage("MDI: >COORDS data"));
519+
}
520+
510521
MPI_Bcast(&d_sys_coords[0], n, MPI_DOUBLE, 0, d_dftfeMPIComm);
511522

512-
for (int i = 0; i < n; i++)
513-
std::cout << "coord: " << d_sys_coords[i] << std::endl;
523+
// for (int i = 0; i < n; i++)
524+
// std::cout << "coord: " << d_sys_coords[i] << std::endl;
514525
}
515526

516527
/* ----------------------------------------------------------------------
@@ -523,12 +534,15 @@ namespace dftfe
523534
{
524535
d_actionflag = 0;
525536
d_flag_natoms = 1;
526-
int ierr = MDI_Recv(&d_sys_natoms, 1, MDI_INT, d_mdicomm);
527-
if (ierr)
528-
AssertThrow(false, dealii::ExcMessage("MDI: >NATOMS data"));
529-
// error->all(FLERR, "MDI: >NATOMS data");
537+
if (d_root == 1)
538+
{
539+
int ierr = MDI_Recv(&d_sys_natoms, 1, MDI_INT, d_mdicomm);
540+
if (ierr)
541+
AssertThrow(false, dealii::ExcMessage("MDI: >NATOMS data"));
542+
}
543+
530544
MPI_Bcast(&d_sys_natoms, 1, MPI_INT, 0, d_dftfeMPIComm);
531-
std::cout << "n atoms: " << d_sys_natoms << std::endl;
545+
// std::cout << "n atoms: " << d_sys_natoms << std::endl;
532546
}
533547

534548
/* ----------------------------------------------------------------------
@@ -542,15 +556,18 @@ namespace dftfe
542556
d_actionflag = 0;
543557
d_flag_elements = 1;
544558
d_sys_elements.resize(d_sys_natoms);
545-
int ierr = MDI_Recv(&d_sys_elements[0], d_sys_natoms, MDI_INT, d_mdicomm);
546-
if (ierr)
547-
AssertThrow(false, dealii::ExcMessage("MDI: >ELEMENTS data"));
548-
// error->all(FLERR, "MDI: >ELEMENTS data");
549-
// FIXME: check if the correct communicator is being used
559+
if (d_root == 1)
560+
{
561+
int ierr =
562+
MDI_Recv(&d_sys_elements[0], d_sys_natoms, MDI_INT, d_mdicomm);
563+
if (ierr)
564+
AssertThrow(false, dealii::ExcMessage("MDI: >ELEMENTS data"));
565+
}
566+
550567
MPI_Bcast(&d_sys_elements[0], d_sys_natoms, MPI_INT, 0, d_dftfeMPIComm);
551568

552-
for (int i = 0; i < d_sys_natoms; i++)
553-
std::cout << "element: " << d_sys_elements[i] << std::endl;
569+
// for (int i = 0; i < d_sys_natoms; i++)
570+
// std::cout << "element: " << d_sys_elements[i] << std::endl;
554571
}
555572

556573
// ----------------------------------------------------------------------
@@ -574,10 +591,13 @@ namespace dftfe
574591
{
575592
// NOTE: energy should be QM energy
576593
const double energy = d_dftfeWrapper.getDFTFreeEnergy();
577-
int ierr = MDI_Send(&energy, 1, MDI_DOUBLE, d_mdicomm);
578-
if (ierr)
579-
AssertThrow(false, dealii::ExcMessage("MDI: <ENERGY data"));
580-
// error->all(FLERR, "MDI: <ENERGY data");
594+
if (d_root == 1)
595+
{
596+
int ierr = MDI_Send(&energy, 1, MDI_DOUBLE, d_mdicomm);
597+
if (ierr)
598+
AssertThrow(false, dealii::ExcMessage("MDI: <ENERGY data"));
599+
}
600+
MPI_Barrier(d_dftfeMPIComm);
581601
}
582602

583603
/* ----------------------------------------------------------------------
@@ -600,15 +620,18 @@ namespace dftfe
600620
forces[3 * i + 1] = ionicForces[i][1];
601621
forces[3 * i + 2] = ionicForces[i][2];
602622
}
603-
int ierr = MDI_Send(&forces[0], 3 * d_sys_natoms, MDI_DOUBLE, d_mdicomm);
604-
if (ierr)
605-
AssertThrow(false, dealii::ExcMessage("MDI: <FORCES data"));
606-
// error->all(FLERR, "MDI: <FORCES data");
623+
if (d_root == 1)
624+
{
625+
int ierr =
626+
MDI_Send(&forces[0], 3 * d_sys_natoms, MDI_DOUBLE, d_mdicomm);
627+
if (ierr)
628+
AssertThrow(false, dealii::ExcMessage("MDI: <FORCES data"));
629+
}
630+
MPI_Barrier(d_dftfeMPIComm);
607631
}
608632

609633
/* ----------------------------------------------------------------------
610634
<STRESS command
611-
send 6-component stress tensor (no kinetic energy term)
612635
---------------------------------------------------------------------- */
613636

614637
void
@@ -629,10 +652,14 @@ namespace dftfe
629652
stressTensorFlattened[7] = stressTensor[2][1];
630653
stressTensorFlattened[8] = stressTensor[2][2];
631654

632-
int ierr = MDI_Send(&stressTensorFlattened[0], 9, MDI_DOUBLE, d_mdicomm);
633-
if (ierr)
634-
AssertThrow(false, dealii::ExcMessage("MDI: <STRESS data"));
635-
// error->all(FLERR, "MDI: <STRESS data");
655+
if (d_root == 1)
656+
{
657+
int ierr =
658+
MDI_Send(&stressTensorFlattened[0], 9, MDI_DOUBLE, d_mdicomm);
659+
if (ierr)
660+
AssertThrow(false, dealii::ExcMessage("MDI: <STRESS data"));
661+
}
662+
MPI_Barrier(d_dftfeMPIComm);
636663
}
637664
} // namespace dftfe
638665
#endif

0 commit comments

Comments
 (0)