Skip to content

Commit e81c513

Browse files
knikhil1995knikhil1995
committed
Cleanups
1 parent 934a52b commit e81c513

11 files changed

+1053
-1543
lines changed

Diff for: ‎CMakeLists.txt

+17-30
Original file line numberDiff line numberDiff line change
@@ -208,19 +208,6 @@ SET(TARGET_SRC
208208
./src/pseudo/oncv/atomCenteredPostProcessing.cc)
209209

210210
IF ("${GPU_LANG}" STREQUAL "cuda")
211-
SET(DEVICE_HOST_SRC
212-
./src/linAlg/linearAlgebraOperationsDevice.cc
213-
./utils/constraintMatrixInfoDevice.cc
214-
./src/dft/solveVselfInBinsDevice.cc
215-
./src/linAlg/pseudoGSDevice.cc
216-
./src/linAlg/rayleighRitzDevice.cc
217-
./src/poisson/poissonSolverProblemDevice.cc
218-
./src/helmholtz/kerkerSolverProblemDevice.cc
219-
./src/solvers/eigenSolvers/chebyshevOrthogonalizedSubspaceIterationSolverDevice.cc
220-
./src/solvers/linearSolverDevice.cc
221-
./src/solvers/linearSolverProblemDevice.cc
222-
./src/solvers/linearSolverCGDevice.cc
223-
)
224211
SET(DEVICE_SRC
225212
./utils/MemoryTransferKernelsDevice.cc
226213
./utils/DeviceKernelsGeneric.cc
@@ -236,24 +223,11 @@ SET(DEVICE_SRC
236223
./utils/BLASWrapperDevice.cu.cc
237224
./utils/MPICommunicatorP2PKernelsDevice.cc
238225
./src/solvers/linearSolverCGDeviceKernels.cc
239-
./src/poisson/poissonSolverProblemDeviceKernels.cc
226+
./utils/matrixFreeDeviceKernels.cc
240227
./utils/FEBasisOperationsKernelsInternalDevice.cc
241228
./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
242229
)
243230
ELSEIF ("${GPU_LANG}" STREQUAL "hip")
244-
SET(DEVICE_HOST_SRC
245-
./src/linAlg/linearAlgebraOperationsDevice.cc
246-
./utils/constraintMatrixInfoDevice.cc
247-
./src/dft/solveVselfInBinsDevice.cc
248-
./src/linAlg/pseudoGSDevice.cc
249-
./src/linAlg/rayleighRitzDevice.cc
250-
./src/poisson/poissonSolverProblemDevice.cc
251-
./src/helmholtz/kerkerSolverProblemDevice.cc
252-
./src/solvers/eigenSolvers/chebyshevOrthogonalizedSubspaceIterationSolverDevice.cc
253-
./src/solvers/linearSolverDevice.cc
254-
./src/solvers/linearSolverProblemDevice.cc
255-
./src/solvers/linearSolverCGDevice.cc
256-
)
257231
SET(DEVICE_SRC
258232
./utils/MemoryTransferKernelsDevice.cc
259233
./utils/DeviceKernelsGeneric.cc
@@ -269,7 +243,7 @@ SET(DEVICE_SRC
269243
./utils/BLASWrapperDevice.hip.cc
270244
./utils/MPICommunicatorP2PKernelsDevice.cc
271245
./src/solvers/linearSolverCGDeviceKernels.cc
272-
./src/poisson/poissonSolverProblemDeviceKernelss.cc
246+
./utils/matrixFreeDeviceKernels.cc
273247
./utils/FEBasisOperationsKernelsInternalDevice.cc
274248
./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
275249
)
@@ -290,11 +264,24 @@ IF (WITH_GPU)
290264
ENDIF()
291265
ENDIF()
292266
IF (WITH_GPU)
293-
LIST(APPEND TARGET_SRC ${DEVICE_HOST_SRC} ${DEVICE_SRC})
267+
LIST(APPEND TARGET_SRC
268+
./src/linAlg/linearAlgebraOperationsDevice.cc
269+
./utils/constraintMatrixInfoDevice.cc
270+
./src/dft/solveVselfInBinsDevice.cc
271+
./src/linAlg/pseudoGSDevice.cc
272+
./src/linAlg/rayleighRitzDevice.cc
273+
./src/poisson/poissonSolverProblemDevice.cc
274+
./src/helmholtz/kerkerSolverProblemDevice.cc
275+
./src/solvers/eigenSolvers/chebyshevOrthogonalizedSubspaceIterationSolverDevice.cc
276+
./src/solvers/linearSolverDevice.cc
277+
./src/solvers/linearSolverProblemDevice.cc
278+
./src/solvers/linearSolverCGDevice.cc
279+
)
280+
LIST(APPEND TARGET_SRC ${DEVICE_SRC})
294281
ENDIF()
295282

296283

297-
ADD_LIBRARY(${TARGETLIB} SHARED ${TARGET_SRC})
284+
ADD_LIBRARY(${TARGETLIB} ${TARGET_SRC})
298285

299286
#IF("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
300287
#

Diff for: ‎helpers/MATRIXcluster/setupDevelopPetsc_MATRIX.sh

+28-25
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,23 @@ SRC=`dirname $0` # location of source directory
1717
# and optimization flag
1818

1919
#Paths for required external libraries
20-
dealiiPetscRealDir="/shared/dftfesoftwares2021/dealii/installgcc9.2cudaawarempiWithPetscSlpec"
21-
dealiiPetscComplexDir="/shared/dftfesoftwares2021/dealii/installgcc9.2cudaawarempiWithPetscSlpecComplex"
22-
alglibDir="/shared/dftfesoftwares2021/alglib/cpp/src"
23-
libxcDir="/shared/dftfesoftwares2021/libxc/gcc9.2_libxc_5.1.5"
24-
spglibDir="/shared/dftfesoftwares2021/spglib/gcc9.2_spglib"
20+
dealiiPetscRealDir="/storage/dftfeDependencies/dealii/installReal"
21+
dealiiPetscComplexDir="/storage/dftfeDependencies/dealii/installComplex"
22+
alglibDir="/storage/dftfeDependencies/alglib/install"
23+
libxcDir="/storage/dftfeDependencies/libxc/install"
24+
spglibDir="/storage/dftfeDependencies/spglib/install"
2525
xmlIncludeDir="/usr/include/libxml2"
26-
xmlLibDir="/usr/lib64"
27-
ELPA_PATH="/shared/dftfesoftwares2021/elpa2022/installgcc9.2elpa2022withcudaawarempi"
28-
numdiffdir="/shared/dftfesoftwares2021/numdiff/build"
26+
xmlLibDir="/usr/lib/x86_64-linux-gnu"
27+
ELPA_PATH="/storage/dftfeDependencies/elpa/install"
28+
dftdpath="/storage/dftfeDependencies/dftd/install"
29+
numdiffdir="/storage/dftfeDependencies/numdiff/install"
30+
2931

3032
#Paths for optional external libraries
3133
# path for NCCL/RCCL libraries
32-
DCCL_PATH=""
34+
DCCL_PATH="/apps/softwares/spack/opt/spack/linux-ubuntu24.04-cascadelake/gcc-13.3.0/nccl-2.23.4-1-xyspmp23glxb4slgne4xpemahjrkuyrj"
3335
mdiPath=""
36+
torchDir=""
3437

3538
#Toggle GPU compilation
3639
withGPU=ON
@@ -45,13 +48,13 @@ withGPUAwareMPI=OFF #Please use this option with care
4548
withDCCL=OFF
4649
withMDI=OFF
4750
withTorch=OFF
48-
withCustomizedDealii=ON
51+
withCustomizedDealii=OFF
4952

5053
#Compiler options and flags
51-
cxx_compiler=mpic++ #sets DCMAKE_CXX_COMPILER
52-
cxx_flags="-march=native -fPIC" #sets DCMAKE_CXX_FLAGS
54+
cxx_compiler=mpicxx #sets DCMAKE_CXX_COMPILER
55+
cxx_flags="-std=c++17 -march=native -fopenmp -fPIC" #sets DCMAKE_CXX_FLAGS
5356
cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE
54-
device_flags="-arch=sm_70" # set DCMAKE_CXX_CUDA_FLAGS
57+
device_flags="-arch=sm_70 -ccbin=mpicxx" # set DCMAKE_CXX_CUDA_FLAGS
5558
#(only applicable for withGPU=ON)
5659
device_architectures="70" # set DCMAKE_CXX_CUDA_ARCHITECTURES
5760
#(only applicable for withGPU=ON)
@@ -79,47 +82,47 @@ out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'`
7982

8083
function cmake_configure() {
8184
if [ "$gpuLang" = "cuda" ]; then
82-
cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
85+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE -DCMAKE_CXX_STANDARD=17 -DCMAKE_CXX_COMPILER=$cxx_compiler\
8386
-DCMAKE_CXX_FLAGS="$cxx_flags"\
8487
-DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
8588
-DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
8689
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
8790
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
8891
-DXML_INCLUDE_DIR=$xmlIncludeDir\
89-
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
92+
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch -DTORCH_DIR=$torchDir\
9093
-DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
91-
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
94+
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$dftdpath;$numdiffdir"\
9295
-DWITH_COMPLEX=$withComplex -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
9396
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
94-
-DHIGHERQUAD_PSP=$withHigherQuadPSP $1
97+
-DHIGHERQUAD_PSP=$withHigherQuadPSP -DBUILD_SHARED_LIBS=ON $1
9598
elif [ "$gpuLang" = "hip" ]; then
96-
cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
99+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE -DCMAKE_CXX_STANDARD=17 -DCMAKE_CXX_COMPILER=$cxx_compiler\
97100
-DCMAKE_CXX_FLAGS="$cxx_flags"\
98101
-DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
99102
-DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
100103
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
101104
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
102105
-DXML_INCLUDE_DIR=$xmlIncludeDir\
103-
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
106+
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch -DTORCH_DIR=$torchDir\
104107
-DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
105-
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
108+
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$dftdpath;$numdiffdir"\
106109
-DWITH_COMPLEX=$withComplex -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
107110
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
108-
-DHIGHERQUAD_PSP=$withHigherQuadPSP $1
111+
-DHIGHERQUAD_PSP=$withHigherQuadPSP -DBUILD_SHARED_LIBS=ON $1
109112
else
110-
cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
113+
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE -DCMAKE_CXX_STANDARD=17 -DCMAKE_CXX_COMPILER=$cxx_compiler\
111114
-DCMAKE_CXX_FLAGS="$cxx_flags"\
112115
-DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
113116
-DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
114117
-DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
115118
-DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
116119
-DXML_INCLUDE_DIR=$xmlIncludeDir\
117-
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
120+
-DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch -DTORCH_DIR=$torchDir\
118121
-DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
119-
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
122+
-DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$dftdpath;$numdiffdir"\
120123
-DWITH_COMPLEX=$withComplex \
121124
-DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
122-
-DHIGHERQUAD_PSP=$withHigherQuadPSP $1
125+
-DHIGHERQUAD_PSP=$withHigherQuadPSP -DBUILD_SHARED_LIBS=ON $1
123126
fi
124127
}
125128

Diff for: ‎include/matrixFreeDeviceKernels.h

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#ifndef poissonSolverProblemDeviceKernels_H
2+
#define poissonSolverProblemDeviceKernels_H
3+
#include <DeviceAPICalls.h>
4+
#include <DeviceDataTypeOverloads.h>
5+
#include <DeviceKernelLauncherConstants.h>
6+
7+
8+
namespace dftfe
9+
{
10+
template <typename Type, int M, int N, int K, int dim>
11+
struct matrixFreeDeviceKernels
12+
{
13+
static void
14+
computeAXDevicePoisson(const int blocks,
15+
const int threads,
16+
const int smem,
17+
Type * V,
18+
const Type *U,
19+
const Type *P,
20+
const Type *J,
21+
const int * map);
22+
23+
static void
24+
computeAXDeviceHelmholtz(const int blocks,
25+
const int threads,
26+
const int smem,
27+
Type * V,
28+
const Type *U,
29+
const Type *P,
30+
const Type *J,
31+
const int * map,
32+
const Type coeffHelmholtz);
33+
34+
static void
35+
computeAXDevicePoissonSetAttributes(const int smem);
36+
37+
static void
38+
computeAXDeviceHelmholtzSetAttributes(const int smem);
39+
};
40+
41+
} // namespace dftfe
42+
#endif

Diff for: ‎include/poissonSolverProblemDeviceKernels.h

-34
This file was deleted.

0 commit comments

Comments
 (0)