Allow for C1152 ATM-Aero cycled DA to run on WCOSS2

env/WCOSS2.env

@@ -13,6 +13,10 @@
 export launcher="mpiexec -l"
 export mpmd_opt="--cpu-bind verbose,core cfp"
 
+# Add path to GDASApp libraries
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$HOMEgfs/sorc/gdas.cd/build/lib"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib"
+
 # Calculate common resource variables
 # Check first if the dependent variables are set
 if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
@@ -92,11 +96,11 @@
 
     export NTHREADS_SNOWANL=${NTHREADSmax}
     export APRUN_SNOWANL="${APRUN_default}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
 elif [[ "${step}" = "esnowanl" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
     export NTHREADS_ESNOWANL=${NTHREADSmax}
@@ -113,7 +117,7 @@
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
     export APRUN_MARINEANLVAR="${APRUN_default}"
 
 elif [[ "${step}" = "marineanlchkpt" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

parm/config/gfs/config.aeroanlgenb

@@ -12,17 +12,16 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero_bmat_jedi_config.yaml.j2"
 export JCB_BASE_YAML="${PARMgfs}/gdas/aero/jcb-base.yaml.j2"
 export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2"
 export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2"
-export aero_diffusion_iter=10
-export aero_diffusion_horiz_len=2500e3
-export aero_diffusion_fixed_val=1.0
+export aero_diffusion_iter=200
+export aero_diffusion_horiz_len=300e3
+export aero_diffusion_fixed_val=20.0
 export npx_clim_b=97
 export npy_clim_b=97
 export aero_diagb_weight=0.9
 export aero_staticb_rescaling_factor=2.0
-export aero_diagb_rescale=20.0
 export aero_diagb_n_halo=4
 export aero_diagb_n_neighbors=16
-export aero_diagb_smooth_horiz_iter=0
+export aero_diagb_smooth_horiz_iter=200
 export aero_diagb_smooth_vert_iter=0
 
 echo "END: config.aeroanlgenb"

parm/config/gfs/config.resources

@@ -420,7 +420,7 @@ case ${step} in
 
     export layout_x
     export layout_y
-    walltime="00:10:00"
+    walltime="00:30:00"
     ntasks=1
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
@@ -454,9 +454,9 @@ case ${step} in
     export layout_x
     export layout_y
 
-    walltime="00:30:00"
+    walltime="00:45:00"
     ntasks=$(( layout_x * layout_y * 6 ))
-    threads_per_task=1
+    threads_per_task=5
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     export is_exclusive=True
     ;;
@@ -498,11 +498,11 @@ case ${step} in
 
 
   "aeroanlfinal")
-    walltime="00:10:00"
+    walltime="00:30:00"
     ntasks=1
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
-    memory="3072M"
+    memory="13072M"
     ;;
 
   "marineanlinit")
@@ -903,7 +903,7 @@ case ${step} in
         ;;
       "C768" | "C1152")
         # Not valid resolutions for ensembles
-        declare -x "walltime_gdas"="00:50:00"
+        declare -x "walltime_gdas"="01:20:00"
         declare -x "walltime_gfs"="06:00:00"
         ;;
       *)

parm/config/gfs/config.resources.WCOSS2

@@ -5,6 +5,7 @@
 case ${step} in
   "prep")
     export is_exclusive=True
+    export tasks_per_node=5
     export memory="480GB"
     ;;
 

ush/module-setup.sh

@@ -52,7 +52,7 @@ elif [[ ${MACHINE_ID} = s4* ]] ; then
 elif [[ ${MACHINE_ID} = wcoss2 ]]; then
     # We are on WCOSS2
     # Ignore default modules of the same version lower in the search path (req'd by spack-stack)
-    #export LMOD_TMOD_FIND_FIRST=yes #TODO: Uncomment this when using spack-stack
+    export LMOD_TMOD_FIND_FIRST=yes
     module reset
 
 elif [[ ${MACHINE_ID} = cheyenne* ]] ; then