Skip to content

Commit 01f9c35

Browse files
DavidBurrows-NCODavidBurrows-NCO
authored
Clone, build, and run C48_ATM and C48_S2SW on Gaea C5 and C6 (#3106)
What: Correct build/run for C48_ATM and C48_S2SW on Gaea C5. Add build and run capability for C48_ATM, C48_S2SW, and C96_atm3DVar on Gaea C6. Why: After the C5 OS upgrade, submodules no longer built in the global-workflow. This PR correct that and adds build/run capability to C6. Resolves #3011
1 parent 4ab8cf8 commit 01f9c35

20 files changed

+274
-28
lines changed

‎env/GAEA.env renamed to ‎env/GAEAC5.env

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,19 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
2626
NTHREADS1=${threads_per_task:-1}
2727
[[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
2828
[[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
29-
# This may be useful when Gaea is fully ported, so ignore SC warning
29+
# This may be useful when GaeaC5 is fully ported, so ignore SC warning
3030
# shellcheck disable=SC2034
3131
APRUN_default="${launcher} -n ${ntasks}"
3232
else
33-
echo "ERROR config.resources must be sourced before sourcing GAEA.env"
33+
echo "ERROR config.resources must be sourced before sourcing GAEAC5.env"
3434
exit 2
3535
fi
3636

3737
if [[ "${step}" = "prep" ]]; then
3838

3939
export POE="NO"
4040
export BACK="NO"
41-
export sys_tp="GAEA"
41+
export sys_tp="GAEAC5"
4242
export launcher_PREP="srun"
4343

4444
elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
@@ -86,7 +86,7 @@ elif [[ "${step}" = "upp" ]]; then
8686

8787
elif [[ "${step}" = "atmos_products" ]]; then
8888

89-
export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea
89+
export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC5
9090

9191
elif [[ "${step}" = "oceanice_products" ]]; then
9292

@@ -98,4 +98,11 @@ elif [[ "${step}" = "fit2obs" ]]; then
9898
export NTHREADS_FIT2OBS=${NTHREADS1}
9999
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
100100

101+
elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
102+
103+
export CFP_MP="YES"
104+
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
105+
export wavempexec=${launcher}
106+
export wave_mpmd=${mpmd_opt}
107+
101108
fi

‎env/GAEAC6.env

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#! /usr/bin/env bash
2+
3+
if [[ $# -ne 1 ]]; then
4+
5+
echo "Must specify an input argument to set runtime environment variables!"
6+
exit 1
7+
8+
fi
9+
10+
step=$1
11+
12+
export launcher="srun -l --export=ALL"
13+
export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
14+
15+
export OMP_STACKSIZE=2048000
16+
export NTHSTACK=1024000000
17+
18+
ulimit -s unlimited
19+
ulimit -a
20+
21+
# Calculate common variables
22+
# Check first if the dependent variables are set
23+
if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
24+
max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
25+
NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
26+
NTHREADS1=${threads_per_task:-1}
27+
[[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
28+
[[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
29+
# This may be useful when GaeaC6 is fully ported, so ignore SC warning
30+
# shellcheck disable=SC2034
31+
APRUN_default="${launcher} -n ${ntasks}"
32+
else
33+
echo "ERROR config.resources must be sourced before sourcing GAEAC6.env"
34+
exit 2
35+
fi
36+
37+
if [[ "${step}" = "prep" ]]; then
38+
39+
export POE="NO"
40+
export BACK="NO"
41+
export sys_tp="GAEAC6"
42+
export launcher_PREP="srun"
43+
44+
elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
45+
46+
export MKL_NUM_THREADS=4
47+
export MKL_CBWR=AUTO
48+
49+
export CFP_MP=${CFP_MP:-"YES"}
50+
export USE_CFP=${USE_CFP:-"YES"}
51+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
52+
53+
export NTHREADS_GSI=${NTHREADSmax}
54+
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
55+
56+
export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
57+
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
58+
export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
59+
60+
export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
61+
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
62+
ntasks_cycle=${ntiles:-6}
63+
export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
64+
65+
export NTHREADS_GAUSFCANL=1
66+
ntasks_gausfcanl=${ntasks_gausfcanl:-1}
67+
export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
68+
69+
elif [[ "${step}" = "sfcanl" ]]; then
70+
71+
export NTHREADS_CYCLE=${threads_per_task:-14}
72+
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
73+
74+
elif [[ "${step}" = "fcst" ]]; then
75+
76+
(( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
77+
(( ufs_ntasks = nnodes*tasks_per_node ))
78+
# With ESMF threading, the model wants to use the full node
79+
export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
80+
unset nnodes ufs_ntasks
81+
82+
elif [[ "${step}" = "upp" ]]; then
83+
84+
export NTHREADS_UPP=${NTHREADS1}
85+
export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
86+
87+
elif [[ "${step}" = "atmos_products" ]]; then
88+
89+
export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC6
90+
91+
elif [[ "${step}" = "oceanice_products" ]]; then
92+
93+
export NTHREADS_OCNICEPOST=${NTHREADS1}
94+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
95+
96+
elif [[ "${step}" = "fit2obs" ]]; then
97+
98+
export NTHREADS_FIT2OBS=${NTHREADS1}
99+
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
100+
101+
elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
102+
103+
export CFP_MP="YES"
104+
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
105+
export wavempexec=${launcher}
106+
export wave_mpmd=${mpmd_opt}
107+
108+
fi

‎modulefiles/module_base.gaea.lua renamed to ‎modulefiles/module_base.gaeac5.lua

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
help([[
2-
Load environment to run GFS on Gaea
2+
Load environment to run GFS on Gaea C5
33
]])
44

55
local spack_mod_path=(os.getenv("spack_mod_path") or "None")

‎modulefiles/module_base.gaeac6.lua

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
help([[
2+
Load environment to run GFS on Gaea C6
3+
]])
4+
5+
local spack_mod_path=(os.getenv("spack_mod_path") or "None")
6+
prepend_path("MODULEPATH", spack_mod_path)
7+
8+
load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
9+
load(pathJoin("stack-cray-mpich", (os.getenv("stack_cray_mpich_ver") or "None")))
10+
load(pathJoin("python", (os.getenv("python_ver") or "None")))
11+
12+
load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
13+
load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
14+
load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
15+
load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
16+
load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
17+
load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
18+
load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None")))
19+
20+
load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
21+
load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
22+
load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
23+
load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
24+
load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
25+
load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
26+
load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
27+
load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
28+
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
29+
load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
30+
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
31+
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
32+
load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
33+
load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
34+
load(pathJoin("met", (os.getenv("met_ver") or "None")))
35+
load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
36+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
37+
38+
setenv("WGRIB2","wgrib2")
39+
setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
40+
41+
prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
42+
load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
43+
44+
prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
45+
load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
46+
47+
whatis("Description: GFS run setup environment")

‎modulefiles/module_gwsetup.gaea.lua renamed to ‎modulefiles/module_gwsetup.gaeac5.lua

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
help([[
2-
Load environment to run GFS workflow setup scripts on Gaea
2+
Load environment to run GFS workflow setup scripts on Gaea C5
33
]])
44

55
prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles")
66
load(pathJoin("rocoto"))
77

88
prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
99

10-
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
10+
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
1111
local python_ver=os.getenv("python_ver") or "3.10.13"
1212

1313
load(pathJoin("stack-intel", stack_intel_ver))

‎modulefiles/module_gwsetup.gaeac6.lua

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
help([[
2+
Load environment to run GFS workflow setup scripts on Gaea C6
3+
]])
4+
5+
prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles")
6+
load(pathJoin("rocoto"))
7+
8+
prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
9+
10+
local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
11+
local python_ver=os.getenv("python_ver") or "3.10.13"
12+
13+
load(pathJoin("stack-intel", stack_intel_ver))
14+
load(pathJoin("python", python_ver))
15+
load("py-jinja2")
16+
load("py-pyyaml")
17+
load("py-numpy")
18+
load("git-lfs")
19+
20+
whatis("Description: GFS run setup environment")

‎parm/config/gfs/config.resources

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,16 @@ case ${machine} in
4747
# shellcheck disable=SC2034
4848
mem_node_max="96GB"
4949
;;
50-
"GAEA")
50+
"GAEAC5")
5151
max_tasks_per_node=128
5252
# shellcheck disable=SC2034
5353
mem_node_max="251GB"
5454
;;
55+
"GAEAC6")
56+
max_tasks_per_node=192
57+
# shellcheck disable=SC2034
58+
mem_node_max="384GB"
59+
;;
5560
"ORION")
5661
max_tasks_per_node=40
5762
# shellcheck disable=SC2034

‎parm/config/gfs/config.resources.GAEA renamed to ‎parm/config/gfs/config.resources.GAEAC5

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#! /usr/bin/env bash
22

3-
# Gaea-specific job resources
3+
# GaeaC5-specific job resources
44

55
case ${step} in
66
"prep")
@@ -26,6 +26,8 @@ case ${step} in
2626

2727
esac
2828

29+
export FI_VERBS_PREFER_XRC=0
30+
2931
unset memory
3032
# shellcheck disable=SC2312
3133
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do

‎parm/config/gfs/config.resources.GAEAC6

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#! /usr/bin/env bash
2+
3+
# GaeaC6-specific job resources
4+
5+
unset memory
6+
# shellcheck disable=SC2312
7+
for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
8+
unset "${mem_var}"
9+
done

‎sorc/link_workflow.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ case "${machine}" in
7676
"hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
7777
"jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;;
7878
"s4") FIX_DIR="/data/prod/glopara/fix" ;;
79-
"gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
79+
"gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
80+
"gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/fix" ;;
8081
"noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;
8182
*)
8283
echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"

‎ush/detect_machine.sh

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@ case $(hostname -f) in
2121
dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9
2222
dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10
2323

24-
gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58
25-
gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58
24+
gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58
25+
gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58
26+
27+
gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68
28+
gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68
2629

2730
hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
2831
hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
@@ -81,9 +84,12 @@ elif [[ -d /work ]]; then
8184
else
8285
MACHINE_ID=orion
8386
fi
84-
elif [[ -d /gpfs && -d /ncrc ]]; then
85-
# We are on GAEA.
86-
MACHINE_ID=gaea
87+
elif [[ -d /gpfs/f5 ]]; then
88+
# We are on GAEAC5.
89+
MACHINE_ID=gaeac5
90+
elif [[ -d /gpfs/f6 ]]; then
91+
# We are on GAEAC6.
92+
MACHINE_ID=gaeac6
8793
elif [[ -d /data/prod ]]; then
8894
# We are on SSEC's S4
8995
MACHINE_ID=s4

‎ush/load_fv3gfs_modules.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver"
2020
module use "${HOMEgfs}/modulefiles"
2121

2222
case "${MACHINE_ID}" in
23-
"wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
23+
"wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "jet" | "s4" | "noaacloud")
2424
module load "module_base.${MACHINE_ID}"
2525
;;
2626
*)

‎ush/load_ufsda_modules.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ source "${HOMEgfs}/ush/module-setup.sh"
3434
module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
3535

3636
case "${MACHINE_ID}" in
37-
("hera" | "orion" | "hercules" | "wcoss2")
37+
("hera" | "orion" | "hercules" | "wcoss2" | "gaeac5" | "gaeac6")
3838
#TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2
3939
if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
4040
export LMOD_TMOD_FIND_FIRST=yes
@@ -48,7 +48,7 @@ case "${MACHINE_ID}" in
4848
NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
4949
export NETCDF
5050
;;
51-
("jet" | "gaea" | "s4" | "acorn")
51+
("jet" | "s4" | "acorn")
5252
echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM
5353
;;
5454
*)

‎versions/build.gaea.ver renamed to ‎versions/build.gaeac5.ver

File renamed without changes.

‎versions/run.gaea.ver renamed to ‎versions/build.gaeac6.ver

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
export stack_intel_ver=2023.1.0
22
export stack_cray_mpich_ver=8.1.25
33
export spack_env=gsi-addon-dev
4-
5-
export perl_ver=5.38.2
6-
74
source "${HOMEgfs:-}/versions/spack.ver"
85
export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

‎versions/run.gaeac5.ver

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
export stack_intel_ver=2023.2.0
2+
export stack_cray_mpich_ver=8.1.28
3+
export spack_env=gsi-addon-dev
4+
5+
export perl_ver=5.38.2
6+
7+
source "${HOMEgfs:-}/versions/spack.ver"
8+
export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

‎versions/run.gaeac6.ver

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
export stack_intel_ver=2023.2.0
2+
export stack_cray_mpich_ver=8.1.29
3+
export spack_env=gsi-addon
4+
5+
export perl_ver=5.38.2
6+
7+
source "${HOMEgfs:-}/versions/spack.ver"
8+
export spack_mod_path="/ncrc/proj/epic/spack-stack/c6/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

0 commit comments

Comments
 (0)