Skip to content

Commit 2136b87

Browse files
DavidBurrows-NCODavidBurrows-NCO
authored
Add/fix build capability for Gaea-C5, Gaea-C6, and container (#800)
1 parent 7b84bbf commit 2136b87

10 files changed

+322
-34
lines changed

‎modulefiles/gsi_container.intel.lua

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
help([[
2+
]])
3+
4+
prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core")
5+
6+
local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
7+
local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.2"
8+
local cmake_ver=os.getenv("cmake_ver") or "3.27.9"
9+
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
10+
11+
load(pathJoin("stack-intel", stack_intel_ver))
12+
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
13+
load(pathJoin("cmake", cmake_ver))
14+
15+
load("gsi_common")
16+
load(pathJoin("prod_util", prod_util_ver))
17+
18+
pushenv("CFLAGS", "-march=ivybridge")
19+
pushenv("FFLAGS", "-march=ivybridge")
20+
21+
setenv("CC","mpiicc")
22+
setenv("CXX","mpiicpc")
23+
setenv("FC","mpiifort")
24+
setenv("F90","mpiifort")
25+
setenv("F77","mpiifort")
26+
pushenv("USE_BUFR4", "YES")
27+
28+
whatis("Description: GSI environment in a container with Intel Compilers")

‎modulefiles/gsi_gaea.intel.lua ‎modulefiles/gsi_gaeac5.intel.lua

+3-7
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ help([[
44
prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
55

66
local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
7-
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
8-
local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
7+
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
8+
local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
99
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
1010
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
1111

@@ -17,10 +17,6 @@ load(pathJoin("cmake", cmake_ver))
1717
load("gsi_common")
1818
load(pathJoin("prod_util", prod_util_ver))
1919

20-
local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
21-
prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
22-
pushenv("MKLROOT", MKLROOT)
23-
2420
pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f5/ufs-ard/world-shared/GSI_data/fix/gsi/20240208")
2521

2622
setenv("CC","cc")
@@ -29,4 +25,4 @@ setenv("CXX","CC")
2925
pushenv("CRAYPE_LINK_TYPE","dynamic")
3026

3127
unload("cray-libsci")
32-
whatis("Description: GSI environment on Gaea with Intel Compilers")
28+
whatis("Description: GSI environment on GaeaC5 with Intel Compilers")

‎modulefiles/gsi_gaeac6.intel.lua

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
help([[
2+
]])
3+
4+
--prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
5+
prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core")
6+
7+
local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
8+
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
9+
local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
10+
local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
11+
local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
12+
13+
load(pathJoin("stack-intel", stack_intel_ver))
14+
load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
15+
load(pathJoin("stack-python", stack_python_ver))
16+
load(pathJoin("cmake", cmake_ver))
17+
18+
load("gsi_common")
19+
load(pathJoin("prod_util", prod_util_ver))
20+
21+
pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f6/bil-fire8/world-shared/GSI_data/fix/gsi/20240208")
22+
23+
setenv("CC","cc")
24+
setenv("FC","ftn")
25+
setenv("CXX","CC")
26+
pushenv("CRAYPE_LINK_TYPE","dynamic")
27+
28+
unload("cray-libsci")
29+
whatis("Description: GSI environment on GaeaC6 with Intel Compilers")

‎regression/regression_param.sh

+47-14
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,16 @@ case $machine in
2424
memnode=96
2525
numcore=40
2626
;;
27-
Gaea)
28-
sub_cmd="sub_gaea"
27+
gaeac5)
28+
sub_cmd="sub_gaeac5"
2929
memnode=251
3030
numcore=128
3131
;;
32+
gaeac6)
33+
sub_cmd="sub_gaeac6"
34+
memnode=384
35+
numcore=192
36+
;;
3237
wcoss2)
3338
sub_cmd="sub_wcoss2"
3439
memnode=512
@@ -73,7 +78,10 @@ case $regtest in
7378
elif [[ "$machine" = "Discover" ]]; then
7479
topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1"
7580
topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2"
76-
elif [[ "$machine" = "Gaea" ]]; then
81+
elif [[ "$machine" = "gaeac5" ]]; then
82+
topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
83+
topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
84+
elif [[ "$machine" = "gaeac6" ]]; then
7785
topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
7886
topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
7987
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
@@ -103,9 +111,12 @@ case $regtest in
103111
elif [[ "$machine" = "Jet" ]]; then
104112
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
105113
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
106-
elif [[ "$machine" = "Gaea" ]]; then
107-
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
108-
topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
114+
elif [[ "$machine" = "gaeac5" ]]; then
115+
topts[1]="0:15:00" ; popts[1]="40/3/" ; ropts[1]="/1"
116+
topts[2]="0:15:00" ; popts[2]="40/5/" ; ropts[2]="/1"
117+
elif [[ "$machine" = "gaeac6" ]]; then
118+
topts[1]="0:15:00" ; popts[1]="40/3/" ; ropts[1]="/1"
119+
topts[2]="0:15:00" ; popts[2]="40/5/" ; ropts[2]="/1"
109120
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
110121
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
111122
topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -133,7 +144,10 @@ case $regtest in
133144
elif [[ "$machine" = "Jet" ]]; then
134145
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
135146
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
136-
elif [[ "$machine" = "Gaea" ]]; then
147+
elif [[ "$machine" = "gaeac5" ]]; then
148+
topts[1]="0:15:00" ; popts[1]="32/2/" ; ropts[1]="/1"
149+
topts[2]="0:15:00" ; popts[2]="64/4/" ; ropts[2]="/1"
150+
elif [[ "$machine" = "gaeac6" ]]; then
137151
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
138152
topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
139153
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
@@ -162,7 +176,10 @@ case $regtest in
162176
elif [[ "$machine" = "Jet" ]]; then
163177
topts[1]="0:15:00" ; popts[1]="5/4/" ; ropts[1]="/1"
164178
topts[2]="0:15:00" ; popts[2]="10/4/" ; ropts[2]="/1"
165-
elif [[ "$machine" = "Gaea" ]]; then
179+
elif [[ "$machine" = "gaeac5" ]]; then
180+
topts[1]="0:15:00" ; popts[1]="32/2/" ; ropts[1]="/1"
181+
topts[2]="0:15:00" ; popts[2]="64/4/" ; ropts[2]="/1"
182+
elif [[ "$machine" = "gaeac6" ]]; then
166183
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
167184
topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
168185
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
@@ -192,9 +209,12 @@ case $regtest in
192209
elif [[ "$machine" = "Jet" ]]; then
193210
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
194211
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
195-
elif [[ "$machine" = "Gaea" ]]; then
196-
topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
197-
topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
212+
elif [[ "$machine" = "gaeac5" ]]; then
213+
topts[1]="0:15:00" ; popts[1]="40/2/" ; ropts[1]="/1"
214+
topts[2]="0:15:00" ; popts[2]="40/4/" ; ropts[2]="/1"
215+
elif [[ "$machine" = "gaeac6" ]]; then
216+
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
217+
topts[2]="0:15:00" ; popts[2]="64/2/" ; ropts[2]="/1"
198218
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
199219
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
200220
topts[2]="0:15:00" ; popts[2]="64/2/" ; ropts[2]="/1"
@@ -222,7 +242,10 @@ case $regtest in
222242
elif [[ "$machine" = "Jet" ]]; then
223243
topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
224244
topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
225-
elif [[ "$machine" = "Gaea" ]]; then
245+
elif [[ "$machine" = "gaeac5" ]]; then
246+
topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
247+
topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1"
248+
elif [[ "$machine" = "gaeac6" ]]; then
226249
topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
227250
topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1"
228251
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
@@ -252,7 +275,10 @@ case $regtest in
252275
elif [[ "$machine" = "Jet" ]]; then
253276
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
254277
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
255-
elif [[ "$machine" = "Gaea" ]]; then
278+
elif [[ "$machine" = "gaeac5" ]]; then
279+
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
280+
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
281+
elif [[ "$machine" = "gaeac6" ]]; then
256282
topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
257283
topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
258284
elif [[ "$machine" = "wcoss2" || "$machine" = "acorn" ]]; then
@@ -315,7 +341,14 @@ elif [[ "$machine" = "Jet" ]]; then
315341
export MPI_BUFS_PER_HOST=256
316342
export MPI_GROUP_MAX=256
317343
export APRUN="srun -n \$ntasks --cpus-per-task=\$threads"
318-
elif [[ "$machine" = "Gaea" ]]; then
344+
elif [[ "$machine" = "gaeac5" ]]; then
345+
export OMP_STACKSIZE=1024M
346+
export MPI_BUFS_PER_PROC=256
347+
export MPI_BUFS_PER_HOST=256
348+
export MPI_GROUP_MAX=256
349+
export FI_VERBS_PREFER_XRC=0
350+
export APRUN="srun --export=ALL -n \$ntasks"
351+
elif [[ "$machine" = "gaeac6" ]]; then
319352
export OMP_STACKSIZE=1024M
320353
export MPI_BUFS_PER_PROC=256
321354
export MPI_BUFS_PER_HOST=256

‎regression/regression_var.sh

+15-3
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,10 @@ elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs5 ]]; then # Jet
3636
export machine="Jet"
3737
elif [[ -d /discover ]]; then # NCCS Discover
3838
export machine="Discover"
39-
elif [[ -d /ncrc ]]; then # Gaea
40-
export machine="Gaea"
39+
elif [[ -d /gpfs/f5 ]]; then # GaeaC5
40+
export machine="gaeac5"
41+
elif [[ -d /gpfs/f6 ]]; then # GaeaC6
42+
export machine="gaeac6"
4143
elif [[ -d /data/prod ]]; then # S4
4244
export machine="S4"
4345
elif [[ -d /work ]]; then # Orion or Hercules
@@ -57,7 +59,7 @@ fi
5759
echo "Running Regression Tests on '$machine'";
5860

5961
case $machine in
60-
Gaea)
62+
gaeac5)
6163
export queue="normal"
6264
export group="ufs-ard"
6365
export noscrub="/gpfs/f5/${group}/scratch/${USER}/$LOGNAME/gsi_tmp/noscrub"
@@ -67,6 +69,16 @@ case $machine in
6769
export check_resource="no"
6870
export accnt="ufs-ard"
6971
;;
72+
gaeac6)
73+
export queue="normal"
74+
export group="bil-fire8"
75+
export noscrub="/gpfs/f6/${group}/scratch/${USER}/${LOGNAME}/gsi_tmp/noscrub"
76+
export ptmp="/gpfs/f6/${group}/scratch/${USER}/${LOGNAME}/gsi_tmp/ptmp"
77+
export casesdir="/gpfs/f6/bil-fire8/world-shared/GSI_data/CASES/regtest"
78+
79+
export check_resource="no"
80+
export accnt="bil-fire8"
81+
;;
7082
wcoss2 | acorn)
7183
export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}"
7284
if [ -d $local_or_default ]; then

‎src/gsi/CMakeLists.txt

+5-1
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,11 @@ target_link_libraries(gsi_fortran_obj PUBLIC nemsio::nemsio)
155155
target_link_libraries(gsi_fortran_obj PUBLIC ncio::ncio)
156156
target_link_libraries(gsi_fortran_obj PUBLIC w3emc::w3emc_d)
157157
target_link_libraries(gsi_fortran_obj PUBLIC sp::sp_d)
158-
target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d)
158+
if(DEFINED ENV{USE_BUFR4})
159+
target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_4)
160+
else()
161+
target_link_libraries(gsi_fortran_obj PUBLIC bufr::bufr_d)
162+
endif()
159163
target_link_libraries(gsi_fortran_obj PUBLIC crtm::crtm)
160164
if(GSI_MODE MATCHES "Regional")
161165
target_link_libraries(gsi_fortran_obj PUBLIC wrf_io::wrf_io)

‎ush/detect_machine.sh

+15-6
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@ case $(hostname -f) in
2121
dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9
2222
dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10
2323

24-
gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58
25-
gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58
24+
gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58
25+
gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58
26+
27+
gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68
28+
gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68
2629

2730
hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
2831
hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
@@ -61,7 +64,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then
6164
fi
6265

6366
# Try searching based on paths since hostname may not match on compute nodes
64-
if [[ -d /lfs/h3 ]]; then
67+
if [[ -d /opt/spack-stack ]]; then
68+
# We are in a container
69+
MACHINE_ID=container
70+
elif [[ -d /lfs/h3 ]]; then
6571
# We are on NOAA Cactus or Dogwood
6672
MACHINE_ID=wcoss2
6773
elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then
@@ -81,9 +87,12 @@ elif [[ -d /work ]]; then
8187
else
8288
MACHINE_ID=orion
8389
fi
84-
elif [[ -d /gpfs && -d /ncrc ]]; then
85-
# We are on GAEA.
86-
MACHINE_ID=gaea
90+
elif [[ -d /gpfs/f5 ]]; then
91+
# We are on GAEAC5.
92+
MACHINE_ID=gaeac5
93+
elif [[ -d /gpfs/f6 ]]; then
94+
# We are on GAEAC6.
95+
MACHINE_ID=gaeac6
8796
elif [[ -d /data/prod ]]; then
8897
# We are on SSEC's S4
8998
MACHINE_ID=s4

‎ush/module-setup.sh

+7
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ elif [[ $MACHINE_ID = orion* ]] ; then
2929
fi
3030
module purge
3131

32+
elif [[ $MACHINE_ID = container ]] ; then
33+
# We are in a container
34+
if ( ! eval module help > /dev/null 2>&1 ) ; then
35+
source /usr/lmod/lmod/init/bash
36+
fi
37+
module purge
38+
3239
elif [[ $MACHINE_ID = s4* ]] ; then
3340
# We are on SSEC Wisconsin S4
3441
if ( ! eval module help > /dev/null 2>&1 ) ; then

‎ush/sub_gaea ‎ush/sub_gaeac5

+3-3
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,8 @@ output=${output:-$jobname.out}
8888
myuser=$LOGNAME
8989
myhost=$(hostname)
9090

91-
if [ -d /gpfs/f5/epic/scratch/${USER}/$LOGNAME ]; then
92-
DATA=/gpfs/f5/epic/scratch/${USER}/$LOGNAME/tmp
91+
if [ -d /gpfs/f5/ufs-ard/scratch/${USER}/$LOGNAME ]; then
92+
DATA=/gpfs/f5/ufs-ard/scratch/${USER}/$LOGNAME/tmp
9393
fi
9494
DATA=${DATA:-$ptmp/tmp}
9595

@@ -129,7 +129,7 @@ echo "" >>$cfile
129129

130130
echo "module reset" >> $cfile
131131
echo "module use $modulefiles" >> $cfile
132-
echo "module load gsi_gaea.intel" >> $cfile
132+
echo "module load gsi_gaeac5.intel" >> $cfile
133133
echo "module list" >> $cfile
134134
echo "" >>$cfile
135135

0 commit comments

Comments
 (0)