From df45b424c867544331cb51af89c2d50ec8bb1cad Mon Sep 17 00:00:00 2001 From: lukasm91 Date: Fri, 18 Oct 2024 17:00:01 +0200 Subject: [PATCH 1/4] Dumping whole layers (#163) * use gather for dumping --- src/programs/ectrans-benchmark.F90 | 47 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/programs/ectrans-benchmark.F90 b/src/programs/ectrans-benchmark.F90 index f9a12a8b3..5c00e6dbf 100644 --- a/src/programs/ectrans-benchmark.F90 +++ b/src/programs/ectrans-benchmark.F90 @@ -210,6 +210,8 @@ program ectrans_benchmark integer(kind=jpim) :: ierr +real(kind=jprb), allocatable :: global_field(:,:) + !=================================================================================================== #include "setup_trans0.h" @@ -217,6 +219,7 @@ program ectrans_benchmark #include "inv_trans.h" #include "dir_trans.h" #include "trans_inq.h" +#include "gath_grid.h" #include "specnorm.h" #include "abor1.intfb.h" #include "gstats_setup.intfb.h" @@ -652,12 +655,17 @@ program ectrans_benchmark ! While in grid point space, dump the values to disk, for debugging only !================================================================================================= - if (ldump_values) then - ! dump a field to a binary file - call dump_gridpoint_field(jstep, myproc, nproma, ngpblks, zgp2(:,1,:), 'S', noutdump) - call dump_gridpoint_field(jstep, myproc, nproma, ngpblks, zgpuv(:,nflevg,1,:), 'U', noutdump) - call dump_gridpoint_field(jstep, myproc, nproma, ngpblks, zgpuv(:,nflevg,2,:), 'V', noutdump) - call dump_gridpoint_field(jstep, myproc, nproma, ngpblks, zgp3a(:,nflevg,1,:), 'T', noutdump) + if (ldump_values .and. mod(jstep,10) == 1) then + if (myproc == 1) then + allocate(global_field(ngptotg,1)) + endif + call dump_gridpoint_field(jstep, myproc, nproma, global_field, zgp2(:,1:1,:), 's', noutdump) + call dump_gridpoint_field(jstep, myproc, nproma, global_field, zgpuv(:,nflevg:nflevg,1,:), 'u', noutdump) + call dump_gridpoint_field(jstep, myproc, nproma, global_field, zgpuv(:,nflevg:nflevg,2,:), 'v', noutdump) + call dump_gridpoint_field(jstep, myproc, nproma, global_field, zgp3a(:,nflevg:nflevg,1,:), 't', noutdump) + if (myproc == 1) then + deallocate(global_field) + endif endif !================================================================================================= @@ -1349,27 +1357,32 @@ end subroutine initialize_2d_spectral_field !=================================================================================================== -subroutine dump_gridpoint_field(jstep, myproc, nproma, ngpblks, fld, fldchar, noutdump) +subroutine dump_gridpoint_field(jstep, myproc, nproma, gfld, fld, fldchar, noutdump) ! Dump a 2d field to a binary file. integer(kind=jpim), intent(in) :: jstep ! Time step, used for naming file integer(kind=jpim), intent(in) :: myproc ! MPI rank, used for naming file integer(kind=jpim), intent(in) :: nproma ! Size of nproma - integer(kind=jpim), intent(in) :: ngpblks ! Number of nproma blocks - real(kind=jprb) , intent(in) :: fld(nproma,ngpblks) ! 2D field + real(kind=jprb) , intent(inout) :: gfld(:,:) ! 2d global field + real(kind=jprb) , intent(in) :: fld(:,:,:) ! 3d local field character , intent(in) :: fldchar ! Single character field identifier integer(kind=jpim), intent(in) :: noutdump ! Tnit number for output file - character(len=14) :: filename = "x.xxx.xxxx.dat" - - write(filename(1:1),'(a1)') fldchar - write(filename(3:5),'(i3.3)') jstep - write(filename(7:10),'(i4.4)') myproc + character(len=10) :: filename = "x.xxxx.dat" - open(noutdump, file=filename, form="unformatted") - write(noutdump) reshape(fld, (/ nproma*ngpblks /)) - close(noutdump) + if (myproc == 1) then + write(filename(1:1),'(a1)') fldchar + write(filename(3:6),'(i4.4)') jstep + open(noutdump,file=filename,form='unformatted') + endif + do ilev=1,size(fld,2) + call gath_grid(gfld(:,:),nproma,1,(/1/),1,fld(:,ilev:ilev,:)) + if (myproc == 1) write(unit=noutdump) gfld(:,1) + enddo + if (myproc == 1) then + close(noutdump) + endif end subroutine dump_gridpoint_field From fcc022cab161d9f43cbeb9c0112292afa6b3301c Mon Sep 17 00:00:00 2001 From: Lukas Mosimann Date: Thu, 17 Oct 2024 09:01:31 -0700 Subject: [PATCH 2/4] add missing check --- src/trans/gpu/algor/hicblas_gemm.hip.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trans/gpu/algor/hicblas_gemm.hip.cpp b/src/trans/gpu/algor/hicblas_gemm.hip.cpp index 9d6178bed..9e4b67c42 100644 --- a/src/trans/gpu/algor/hicblas_gemm.hip.cpp +++ b/src/trans/gpu/algor/hicblas_gemm.hip.cpp @@ -252,7 +252,7 @@ void hipblas_dgemm_wrapper (char transa, char transb, op_t2=HIPBLAS_OP_T; if (!hip_alreadyAllocated_dgemm_handle){ - hipblasCreate(&handle_hip_dgemm); + HICBLAS_CHECK(hipblasCreate(&handle_hip_dgemm)); hip_alreadyAllocated_dgemm_handle=true; } hipblasHandle_t handle = detail::get_hipblas_handle(); @@ -281,7 +281,7 @@ void hipblas_sgemm_wrapper (char transa, char transb, op_t2=HIPBLAS_OP_T; if (!hip_alreadyAllocated_sgemm_handle){ - hipblasCreate(&handle_hip_sgemm); + HICBLAS_CHECK(hipblasCreate(&handle_hip_sgemm)); hip_alreadyAllocated_sgemm_handle=true; } HICBLAS_CHECK(hipblasSgemmStridedBatched(handle_hip_sgemm,op_t1,op_t2,m,n,k, From a4521212eaf6ded7fb5aa58e4931f59e1292afff Mon Sep 17 00:00:00 2001 From: lukasm91 Date: Tue, 22 Oct 2024 08:32:05 +0200 Subject: [PATCH 3/4] Fix define (#166) ACCGPU is not defined because it is PRIVATE. _OPENACC is the right define to use here because it is enabled whenever the code is compiled with OpenACC support. --- src/programs/ectrans-benchmark.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/programs/ectrans-benchmark.F90 b/src/programs/ectrans-benchmark.F90 index 5c00e6dbf..675e6a1e4 100644 --- a/src/programs/ectrans-benchmark.F90 +++ b/src/programs/ectrans-benchmark.F90 @@ -1095,7 +1095,7 @@ subroutine get_command_line_arguments(nsmax, cgrid, iters, iters_warmup, nfld, n character(len=128) :: carg ! Storage variable for command line arguments integer :: iarg = 1 ! Argument index -#ifdef ACCGPU +#ifdef _OPENACC !$acc init #endif From 374e0c08340468b2ff1935f74a09f542165eede1 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 13 Sep 2024 08:05:07 +0000 Subject: [PATCH 4/4] cosmetic --- src/programs/ectrans-benchmark.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/programs/ectrans-benchmark.F90 b/src/programs/ectrans-benchmark.F90 index 675e6a1e4..a939e9aeb 100644 --- a/src/programs/ectrans-benchmark.F90 +++ b/src/programs/ectrans-benchmark.F90 @@ -579,7 +579,7 @@ program ectrans_benchmark if (verbosity >= 0 .and. myproc == 1) then write(nout,'(" ")') write(nout,'(a,i0,a,f9.2,a)') "ectrans_benchmark initialisation, on ",nproc,& - & " tasks, took",ztinit," sec" + & " tasks, took ",ztinit," sec" write(nout,'(" ")') endif