Skip to content

Commit b77a4d4

Browse files
author
Jeffrey Whitaker
authored
Enable DA cycling on gaea C5/C6 (#3255)
Follow on to PR #3106 that enables DA cycling to run on gaea c5/c6 Resolves issue #2985
1 parent 2dd94db commit b77a4d4

File tree

6 files changed

+415
-67
lines changed

6 files changed

+415
-67
lines changed

‎env/GAEAC5.env‎

Lines changed: 206 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,118 @@ else
3434
exit 2
3535
fi
3636

37-
if [[ "${step}" = "prep" ]]; then
37+
case ${step} in
38+
"prep" | "prepbufr")
3839

3940
export POE="NO"
40-
export BACK="NO"
41+
export BACK=${BACK:-"YES"}
4142
export sys_tp="GAEAC5"
4243
export launcher_PREP="srun"
44+
;;
45+
"prep_emissions")
4346

44-
elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
47+
export APRUN="${APRUN_default}"
48+
;;
49+
"waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
50+
51+
export CFP_MP="YES"
52+
[[ "${step}" = "waveprep" ]] && export MP_PULSE=0
53+
export wavempexec=${launcher}
54+
export wave_mpmd=${mpmd_opt}
55+
56+
;;
57+
"atmanlvar")
58+
59+
export NTHREADS_ATMANLVAR=${NTHREADSmax}
60+
export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}"
61+
;;
62+
"atmanlfv3inc")
63+
64+
export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
65+
export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
66+
;;
67+
"atmensanlobs")
68+
69+
export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
70+
export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
71+
;;
72+
"atmensanlsol")
73+
74+
export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
75+
export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
76+
;;
77+
"atmensanlletkf")
78+
79+
export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
80+
export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}"
81+
;;
82+
"atmensanlfv3inc")
83+
84+
export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
85+
export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
86+
;;
87+
"aeroanlvar")
88+
89+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
90+
91+
export NTHREADS_AEROANL=${NTHREADSmax}
92+
export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}"
93+
;;
94+
"aeroanlgenb")
95+
96+
export NTHREADS_AEROANLGENB=${NTHREADSmax}
97+
export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}"
98+
;;
99+
"prepobsaero")
100+
101+
export NTHREADS_PREPOBSAERO=${NTHREADS1}
102+
export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
103+
;;
104+
"snowanl")
105+
106+
export APRUN_CALCFIMS="${launcher} -n 1"
107+
108+
export NTHREADS_SNOWANL=${NTHREADSmax}
109+
export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}"
110+
111+
export APRUN_APPLY_INCR="${launcher} -n 6"
112+
;;
113+
"esnowrecen")
114+
115+
export NTHREADS_ESNOWRECEN=${NTHREADSmax}
116+
export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
117+
118+
export APRUN_APPLY_INCR="${launcher} -n 6"
119+
;;
120+
121+
"marinebmat")
122+
123+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
124+
export APRUN_MARINEBMAT="${APRUN_default}"
125+
;;
126+
"marineanlvar")
127+
128+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
129+
export APRUN_MARINEANLVAR="${APRUN_default}"
130+
;;
131+
"ocnanalecen")
132+
133+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
134+
135+
max_threads_per_task=$((max_tasks_per_node / tasks_per_node_ocnanalecen))
136+
137+
export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
138+
[[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
139+
export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
140+
;;
141+
"marineanlchkpt")
142+
143+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
144+
145+
export NTHREADS_OCNANAL=${NTHREADSmax}
146+
export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
147+
;;
148+
"anal" | "analcalc")
45149

46150
export MKL_NUM_THREADS=4
47151
export MKL_CBWR=AUTO
@@ -50,7 +154,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
50154
export USE_CFP=${USE_CFP:-"YES"}
51155
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
52156

53-
export NTHREADS_GSI=${NTHREADSmax}
157+
export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}}
54158
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
55159

56160
export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
@@ -65,44 +169,127 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
65169
export NTHREADS_GAUSFCANL=1
66170
ntasks_gausfcanl=${ntasks_gausfcanl:-1}
67171
export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
68-
69-
elif [[ "${step}" = "sfcanl" ]]; then
172+
;;
173+
"sfcanl")
70174

71175
export NTHREADS_CYCLE=${threads_per_task:-14}
176+
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
72177
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
178+
;;
179+
"eobs")
180+
181+
export MKL_NUM_THREADS=4
182+
export MKL_CBWR=AUTO
183+
184+
export CFP_MP=${CFP_MP:-"YES"}
185+
export USE_CFP=${USE_CFP:-"YES"}
186+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
187+
188+
189+
export NTHREADS_GSI=${NTHREADSmax}
190+
[[ ${NTHREADS_GSI} -gt ${max_threads_per_task} ]] && export NTHREADS_GSI=${max_threads_per_task}
191+
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
192+
;;
193+
"eupd")
194+
195+
export CFP_MP=${CFP_MP:-"YES"}
196+
export USE_CFP=${USE_CFP:-"YES"}
197+
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
198+
199+
200+
export NTHREADS_ENKF=${NTHREADSmax}
201+
export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}"
202+
;;
203+
"fcst" | "efcs")
73204

74-
elif [[ "${step}" = "fcst" ]]; then
205+
export OMP_STACKSIZE=512M
75206

76207
(( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
77208
(( ufs_ntasks = nnodes*tasks_per_node ))
78209
# With ESMF threading, the model wants to use the full node
79210
export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
80211
unset nnodes ufs_ntasks
212+
;;
81213

82-
elif [[ "${step}" = "upp" ]]; then
214+
"upp")
83215

84216
export NTHREADS_UPP=${NTHREADS1}
85217
export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
218+
;;
86219

87-
elif [[ "${step}" = "atmos_products" ]]; then
220+
"atmos_products")
88221

89-
export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC5
222+
export USE_CFP="YES" # Use MPMD for downstream product generation
223+
;;
90224

91-
elif [[ "${step}" = "oceanice_products" ]]; then
225+
"oceanice_products")
92226

93227
export NTHREADS_OCNICEPOST=${NTHREADS1}
94228
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
229+
;;
95230

96-
elif [[ "${step}" = "fit2obs" ]]; then
231+
"ecen")
97232

98-
export NTHREADS_FIT2OBS=${NTHREADS1}
99-
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
233+
export NTHREADS_ECEN=${NTHREADSmax}
234+
export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}"
235+
236+
export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
237+
[[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
238+
export APRUN_CHGRES="time"
239+
240+
export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
241+
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
242+
export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}"
243+
244+
;;
245+
"esfc")
100246

101-
elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
247+
export NTHREADS_ESFC=${NTHREADSmax}
248+
export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}"
249+
250+
export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
251+
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
252+
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
253+
254+
;;
255+
"epos")
256+
257+
export NTHREADS_EPOS=${NTHREADSmax}
258+
export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}"
259+
260+
;;
261+
"postsnd")
102262

103263
export CFP_MP="YES"
104-
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
105-
export wavempexec=${launcher}
106-
export wave_mpmd=${mpmd_opt}
107264

108-
fi
265+
export NTHREADS_POSTSND=${NTHREADS1}
266+
export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}"
267+
268+
export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
269+
[[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
270+
export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}"
271+
272+
;;
273+
"awips")
274+
275+
export NTHREADS_AWIPS=${NTHREADS1}
276+
export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}"
277+
278+
;;
279+
"gempak")
280+
281+
echo "WARNING: ${step} is not enabled on ${machine}!"
282+
283+
;;
284+
"fit2obs")
285+
286+
export NTHREADS_FIT2OBS=${NTHREADS1}
287+
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
288+
289+
;;
290+
*)
291+
# Some other job not yet defined here
292+
echo "WARNING: The job step ${step} does not specify GAEAC5-specific resources"
293+
;;
294+
esac
295+

0 commit comments

Comments
 (0)