From 7b84bbf7329680c994635df44b93906064d61f09 Mon Sep 17 00:00:00 2001 From: Clara Draper <33430543+ClaraDraper-NOAA@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:26:08 -0500 Subject: [PATCH 1/2] Add writing out of ensemble mean for soil increments. (#802) --- src/enkf/controlvec.f90 | 8 ++++++++ src/enkf/gridio_gfs.f90 | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/enkf/controlvec.f90 b/src/enkf/controlvec.f90 index 0961549634..2195b3bd04 100644 --- a/src/enkf/controlvec.f90 +++ b/src/enkf/controlvec.f90 @@ -261,6 +261,8 @@ subroutine write_control(no_inflate_flag) real(r_single), allocatable, dimension(:,:) :: grdin_mean_tmp real(r_single), allocatable, dimension(:,:,:,:) :: grdin_mean +character(len=max_varname_length), dimension(nc3d) :: no_vars3d + if (nproc <= ntasks_io-1) then ! scale q by ensemble qsat, prior to averaging @@ -335,6 +337,9 @@ subroutine write_control(no_inflate_flag) else call writegriddata(0,0,cvars3d,cvars2d,nc3d,nc2d,clevels,ncdim,grdin_mean,no_inflate_flag) end if + elseif (nc2d>0) then ! always write sfc mean increment for land analysis + no_vars3d='' + call writeincrement(0,0,no_vars3d,cvars2d,nc3d,nc2d,clevels,ncdim,grdin_mean,no_inflate_flag) endif deallocate(grdin_mean) t2 = mpi_wtime() @@ -358,6 +363,9 @@ subroutine write_control(no_inflate_flag) else call writegriddata(0,0,cvars3d,cvars2d,nc3d,nc2d,clevels,ncdim,grdin_mean,no_inflate_flag) end if + elseif (nc2d>0) then ! always write sfc mean increment + no_vars3d='' + call writeincrement(0,0,no_vars3d,cvars2d,nc3d,nc2d,clevels,ncdim,grdin_mean,no_inflate_flag) endif deallocate(grdin_mean) t2 = mpi_wtime() diff --git a/src/enkf/gridio_gfs.f90 b/src/enkf/gridio_gfs.f90 index 6e68f1a347..e2ce16e77a 100644 --- a/src/enkf/gridio_gfs.f90 +++ b/src/enkf/gridio_gfs.f90 @@ -4114,7 +4114,7 @@ subroutine writeincrement(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,grdin, write(charnanal,'(i3.3)') nanal sfcbackgroundloop: do nb=1,nbackgrounds - if (nanal == 0 .and. write_ensmean) then + if (nanal == 0 ) then filenamein = trim(adjustl(datapath))//trim(adjustl(fgsfcfileprefixes(nb)))//"ensmean" filenameout = trim(adjustl(datapath))//trim(adjustl(incsfcfileprefixes(nb)))//"ensmean" else From 2136b872cc44ccebc92b2f29b7ee45f43b12cafa Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:27:14 -0500 Subject: [PATCH 2/2] Add/fix build capability for Gaea-C5, Gaea-C6, and container (#800) --- modulefiles/gsi_container.intel.lua | 28 +++ ...si_gaea.intel.lua => gsi_gaeac5.intel.lua} | 10 +- modulefiles/gsi_gaeac6.intel.lua | 29 +++ regression/regression_param.sh | 61 +++++-- regression/regression_var.sh | 18 +- src/gsi/CMakeLists.txt | 6 +- ush/detect_machine.sh | 21 ++- ush/module-setup.sh | 7 + ush/{sub_gaea => sub_gaeac5} | 6 +- ush/sub_gaeac6 | 170 ++++++++++++++++++ 10 files changed, 322 insertions(+), 34 deletions(-) create mode 100644 modulefiles/gsi_container.intel.lua rename modulefiles/{gsi_gaea.intel.lua => gsi_gaeac5.intel.lua} (74%) create mode 100644 modulefiles/gsi_gaeac6.intel.lua rename ush/{sub_gaea => sub_gaeac5} (97%) create mode 100755 ush/sub_gaeac6 diff --git a/modulefiles/gsi_container.intel.lua b/modulefiles/gsi_container.intel.lua new file mode 100644 index 0000000000..546919be77 --- /dev/null +++ b/modulefiles/gsi_container.intel.lua @@ -0,0 +1,28 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.2" +local cmake_ver=os.getenv("cmake_ver") or "3.27.9" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsi_common") +load(pathJoin("prod_util", prod_util_ver)) + +pushenv("CFLAGS", "-march=ivybridge") +pushenv("FFLAGS", "-march=ivybridge") + +setenv("CC","mpiicc") +setenv("CXX","mpiicpc") +setenv("FC","mpiifort") +setenv("F90","mpiifort") +setenv("F77","mpiifort") +pushenv("USE_BUFR4", "YES") + +whatis("Description: GSI environment in a container with Intel Compilers") diff --git a/modulefiles/gsi_gaea.intel.lua b/modulefiles/gsi_gaeac5.intel.lua similarity index 74% rename from modulefiles/gsi_gaea.intel.lua rename to modulefiles/gsi_gaeac5.intel.lua index 799822caa8..de259f1741 100644 --- a/modulefiles/gsi_gaea.intel.lua +++ b/modulefiles/gsi_gaeac5.intel.lua @@ -4,8 +4,8 @@ help([[ prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" @@ -17,10 +17,6 @@ load(pathJoin("cmake", cmake_ver)) load("gsi_common") load(pathJoin("prod_util", prod_util_ver)) -local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" -prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) -pushenv("MKLROOT", MKLROOT) - pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f5/ufs-ard/world-shared/GSI_data/fix/gsi/20240208") setenv("CC","cc") @@ -29,4 +25,4 @@ setenv("CXX","CC") pushenv("CRAYPE_LINK_TYPE","dynamic") unload("cray-libsci") -whatis("Description: GSI environment on Gaea with Intel Compilers") +whatis("Description: GSI environment on GaeaC5 with Intel Compilers") diff --git a/modulefiles/gsi_gaeac6.intel.lua b/modulefiles/gsi_gaeac6.intel.lua new file mode 100644 index 0000000000..883bd02a72 --- /dev/null +++ b/modulefiles/gsi_gaeac6.intel.lua @@ -0,0 +1,29 @@ +help([[ +]]) + +--prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core") + +local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("gsi_common") +load(pathJoin("prod_util", prod_util_ver)) + +pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f6/bil-fire8/world-shared/GSI_data/fix/gsi/20240208") + +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") +pushenv("CRAYPE_LINK_TYPE","dynamic") + +unload("cray-libsci") +whatis("Description: GSI environment on GaeaC6 with Intel Compilers") diff --git a/regression/regression_param.sh b/regression/regression_param.sh index 34d5964d87..8c56bc2b93 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -24,11 +24,16 @@ case $machine in memnode=96 numcore=40 ;; - Gaea) - sub_cmd="sub_gaea" + gaeac5) + sub_cmd="sub_gaeac5" memnode=251 numcore=128 ;; + gaeac6) + sub_cmd="sub_gaeac6" + memnode=384 + numcore=192 + ;; wcoss2) sub_cmd="sub_wcoss2" memnode=512 @@ -73,7 +78,10 @@ case $regtest in elif [[ "$machine" = "Discover" ]]; then topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2" - elif [[ "$machine" = "Gaea" ]]; then + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" + elif [[ "$machine" = "gaeac6" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then @@ -103,9 +111,12 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1" - elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:15:00" ; popts[1]="40/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="40/5/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac6" ]]; then + topts[1]="0:15:00" ; popts[1]="40/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="40/5/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -133,7 +144,10 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1" - elif [[ "$machine" = "Gaea" ]]; then + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:15:00" ; popts[1]="32/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="64/4/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac6" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then @@ -162,7 +176,10 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1" - elif [[ "$machine" = "Gaea" ]]; then + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:15:00" ; popts[1]="32/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="64/4/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac6" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then @@ -192,9 +209,12 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" - elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:15:00" ; popts[1]="40/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="40/4/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac6" ]]; then + topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="64/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="64/2/" ; ropts[2]="/1" @@ -222,7 +242,10 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" - elif [[ "$machine" = "Gaea" ]]; then + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1" + elif [[ "$machine" = "gaeac6" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then @@ -252,7 +275,10 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" - elif [[ "$machine" = "Gaea" ]]; then + elif [[ "$machine" = "gaeac5" ]]; then + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + elif [[ "$machine" = "gaeac6" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then @@ -315,7 +341,14 @@ elif [[ "$machine" = "Jet" ]]; then export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" -elif [[ "$machine" = "Gaea" ]]; then +elif [[ "$machine" = "gaeac5" ]]; then + export OMP_STACKSIZE=1024M + export MPI_BUFS_PER_PROC=256 + export MPI_BUFS_PER_HOST=256 + export MPI_GROUP_MAX=256 + export FI_VERBS_PREFER_XRC=0 + export APRUN="srun --export=ALL -n \$ntasks" +elif [[ "$machine" = "gaeac6" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 diff --git a/regression/regression_var.sh b/regression/regression_var.sh index d4e7d99aa0..5e6c8e38d8 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -36,8 +36,10 @@ elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs5 ]]; then # Jet export machine="Jet" elif [[ -d /discover ]]; then # NCCS Discover export machine="Discover" -elif [[ -d /ncrc ]]; then # Gaea - export machine="Gaea" +elif [[ -d /gpfs/f5 ]]; then # GaeaC5 + export machine="gaeac5" +elif [[ -d /gpfs/f6 ]]; then # GaeaC6 + export machine="gaeac6" elif [[ -d /data/prod ]]; then # S4 export machine="S4" elif [[ -d /work ]]; then # Orion or Hercules @@ -57,7 +59,7 @@ fi echo "Running Regression Tests on '$machine'"; case $machine in - Gaea) + gaeac5) export queue="normal" export group="ufs-ard" export noscrub="/gpfs/f5/${group}/scratch/${USER}/$LOGNAME/gsi_tmp/noscrub" @@ -67,6 +69,16 @@ case $machine in export check_resource="no" export accnt="ufs-ard" ;; + gaeac6) + export queue="normal" + export group="bil-fire8" + export noscrub="/gpfs/f6/${group}/scratch/${USER}/${LOGNAME}/gsi_tmp/noscrub" + export ptmp="/gpfs/f6/${group}/scratch/${USER}/${LOGNAME}/gsi_tmp/ptmp" + export casesdir="/gpfs/f6/bil-fire8/world-shared/GSI_data/CASES/regtest" + + export check_resource="no" + export accnt="bil-fire8" + ;; wcoss2 | acorn) export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}" if [ -d $local_or_default ]; then diff --git a/src/gsi/CMakeLists.txt b/src/gsi/CMakeLists.txt index f894b0a8a8..05d9c9ec86 100644 --- a/src/gsi/CMakeLists.txt +++ b/src/gsi/CMakeLists.txt @@ -155,7 +155,11 @@ target_link_libraries(gsi_fortran_obj PUBLIC nemsio::nemsio) target_link_libraries(gsi_fortran_obj PUBLIC ncio::ncio) target_link_libraries(gsi_fortran_obj PUBLIC w3emc::w3emc_d) target_link_libraries(gsi_fortran_obj PUBLIC sp::sp_d) -target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d) +if(DEFINED ENV{USE_BUFR4}) + target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_4) +else() + target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d) +endif() target_link_libraries(gsi_fortran_obj PUBLIC crtm::crtm) if(GSI_MODE MATCHES "Regional") target_link_libraries(gsi_fortran_obj PUBLIC wrf_io::wrf_io) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index e06775dbbe..34ac68ef1e 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -21,8 +21,11 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 + + gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 + gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 @@ -61,7 +64,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then fi # Try searching based on paths since hostname may not match on compute nodes -if [[ -d /lfs/h3 ]]; then +if [[ -d /opt/spack-stack ]]; then + # We are in a container + MACHINE_ID=container +elif [[ -d /lfs/h3 ]]; then # We are on NOAA Cactus or Dogwood MACHINE_ID=wcoss2 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then @@ -81,9 +87,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 ]]; then + # We are on GAEAC5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 ]]; then + # We are on GAEAC6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 1f7a4462e2..e84e26420f 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -29,6 +29,13 @@ elif [[ $MACHINE_ID = orion* ]] ; then fi module purge +elif [[ $MACHINE_ID = container ]] ; then + # We are in a container + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /usr/lmod/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = s4* ]] ; then # We are on SSEC Wisconsin S4 if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/sub_gaea b/ush/sub_gaeac5 similarity index 97% rename from ush/sub_gaea rename to ush/sub_gaeac5 index 9c4e253c93..2ed9affe5e 100755 --- a/ush/sub_gaea +++ b/ush/sub_gaeac5 @@ -88,8 +88,8 @@ output=${output:-$jobname.out} myuser=$LOGNAME myhost=$(hostname) -if [ -d /gpfs/f5/epic/scratch/${USER}/$LOGNAME ]; then - DATA=/gpfs/f5/epic/scratch/${USER}/$LOGNAME/tmp +if [ -d /gpfs/f5/ufs-ard/scratch/${USER}/$LOGNAME ]; then + DATA=/gpfs/f5/ufs-ard/scratch/${USER}/$LOGNAME/tmp fi DATA=${DATA:-$ptmp/tmp} @@ -129,7 +129,7 @@ echo "" >>$cfile echo "module reset" >> $cfile echo "module use $modulefiles" >> $cfile -echo "module load gsi_gaea.intel" >> $cfile +echo "module load gsi_gaeac5.intel" >> $cfile echo "module list" >> $cfile echo "" >>$cfile diff --git a/ush/sub_gaeac6 b/ush/sub_gaeac6 new file mode 100755 index 0000000000..3617c36f49 --- /dev/null +++ b/ush/sub_gaeac6 @@ -0,0 +1,170 @@ +#!/bin/sh --login +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +export jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +if [ -d /gpfs/f6/bil-fire8/scratch/${USER}/$LOGNAME ]; then + DATA=/gpfs/f6/bil-fire8/scratch/${USER}/$LOGNAME/tmp +fi +DATA=${DATA:-$ptmp/tmp} + +mkdir -p $DATA + +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${task_node:-$procs} +size=$((nodes*task_node)) +envars=$envars +threads=${rcpu:-1} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/bash -l" >> $cfile +echo "" >> $cfile +echo "#SBATCH --output=$output" >> $cfile +echo "#SBATCH --job-name=$jobname" >> $cfile +echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --clusters=c6" >> $cfile +echo "#SBATCH --time=$timew" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile +echo "#SBATCH --account=$accnt" >> $cfile +echo "#SBATCH --mem=0" >> $cfile + +echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile +echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "ulimit -s unlimited" >> $cfile + +echo "" >>$cfile +echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile +echo "" >>$cfile + +echo "module reset" >> $cfile +echo "module use $modulefiles" >> $cfile +echo "module load gsi_gaeac6.intel" >> $cfile +echo "module list" >> $cfile +echo "" >>$cfile + +cat $exec >> $cfile + +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +sbatch=${sbatch:-sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc