diff --git a/env/HERCULES.env b/env/HERCULES.env index cbb8dfc350..03ddf92336 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -12,7 +12,7 @@ fi step=$1 -export npe_node_max=40 +export npe_node_max=80 export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" @@ -26,11 +26,12 @@ export KMP_AFFINITY=scatter export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 #export LD_BIND_NOW=1 +export I_MPI_EXTRA_FILESYSTEM=1 +export I_MPI_EXTRA_FILESYSTEM_LIST=lustre ulimit -s unlimited ulimit -a -<<<<<<< HEAD case ${step} in "prep" | "prepbufr") diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua index d587b90c4f..49144efbef 100644 --- a/modulefiles/module_base.hercules.lua +++ b/modulefiles/module_base.hercules.lua @@ -8,7 +8,9 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("intel-oneapi-mkl", os.getenv("intel_mkl_ver"))) load(pathJoin("python", os.getenv("python_ver"))) +load(pathJoin("perl", os.getenv("perl_ver"))) -- TODO load NCL once the SAs remove the 'depends_on' statements within it -- NCL is a static installation and does not depend on any libraries diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index ca26a69ec0..9cf479e564 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -61,9 +61,15 @@ export DO_NPOESS="NO" # NPOESS products export DO_TRACKER="YES" # Hurricane track verification export DO_GENESIS="YES" # Cyclone genesis verification export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU) -export DO_VERFOZN="YES" # Ozone data assimilation monitoring -export DO_VERFRAD="YES" # Radiance data assimilation monitoring -export DO_VMINMON="YES" # GSI minimization monitoring +if [[ "${machine}" == "HERCULES" ]]; then + export DO_VERFOZN="NO" # Ozone data assimilation monitoring + export DO_VERFRAD="NO" # Radiance data assimilation monitoring + export DO_VMINMON="NO" # GSI minimization monitoring +else + export DO_VERFOZN="YES" # Ozone data assimilation monitoring + export DO_VERFRAD="YES" # Radiance data assimilation monitoring + export DO_VMINMON="YES" # GSI minimization monitoring +fi export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index b3319ecc1b..3e5ae4ecfb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -56,7 +56,7 @@ elif [[ "${machine}" = "AWSPW" ]]; then elif [[ ${machine} = "ORION" ]]; then export npe_node_max=40 elif [[ ${machine} = "HERCULES" ]]; then - export npe_node_max=40 + export npe_node_max=80 fi if [[ ${step} = "prep" ]]; then @@ -907,10 +907,16 @@ elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then export npe_node_eobs export npe_node_eomg=${npe_node_eobs} export is_exclusive=True - #The number of tasks and cores used must be the same for eobs - #For S4, this is accomplished by running 10 tasks/node + # The number of tasks and cores used must be the same for eobs + # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details + # For S4, this is accomplished by running 10 tasks/node if [[ ${machine} = "S4" ]]; then export npe_node_eobs=10 + elif [[ ${machine} = "HERCULES" ]]; then + #For Hercules, this is only an issue at C384; use 20 tasks/node + if [[ ${CASE} = "C384" ]]; then + export npe_node_eobs=20 + fi fi elif [[ ${step} = "ediag" ]]; then diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 000c8b1e99..46bf89f1af 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -72,9 +72,12 @@ case "${machine}" in "WCOSS2") npe_node_max=128 ;; - "HERA" | "ORION" | "HERCULES") + "HERA" | "ORION" ) npe_node_max=40 ;; + "HERCULES" ) + npe_node_max=80 + ;; "JET") case "${PARTITION_BATCH}" in "xjet") diff --git a/scripts/exglobal_archive.sh b/scripts/exglobal_archive.sh index 18217f4efc..2f7e3be972 100755 --- a/scripts/exglobal_archive.sh +++ b/scripts/exglobal_archive.sh @@ -33,7 +33,9 @@ source "${HOMEgfs}/ush/file_utils.sh" [[ ! -d ${ARCDIR} ]] && mkdir -p "${ARCDIR}" nb_copy "${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat" "${ARCDIR}/gsistat.${RUN}.${PDY}${cyc}" -nb_copy "${COM_CHEM_ANALYSIS}/${APREFIX}aerostat" "${ARCDIR}/aerostat.${RUN}.${PDY}${cyc}" +if [[ ${DO_AERO} = "YES" ]]; then + nb_copy "${COM_CHEM_ANALYSIS}/${APREFIX}aerostat" "${ARCDIR}/aerostat.${RUN}.${PDY}${cyc}" +fi nb_copy "${COM_ATMOS_GRIB_1p00}/${APREFIX}pgrb2.1p00.anl" "${ARCDIR}/pgbanl.${RUN}.${PDY}${cyc}.grib2" # Archive 1 degree forecast GRIB2 files for verification diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index f371890b9f..d20aaeb357 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit f371890b9fcb42312da5f6228d87b5a4829e7e3a +Subproject commit d20aaeb35767d6c24560c5d54d45a9c72e5c5ba2 diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 0a027c7537..c34fff1a84 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -355,8 +355,10 @@ if [[ ${type} == "gdas" ]]; then if [[ -s "${COM_ATMOS_ANALYSIS}/${head}oznstat" ]]; then echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}oznstat" fi - if [[ -s "${COM_CHEM_ANALYSIS}/${head}aerostat" ]]; then - echo "${COM_CHEM_ANALYSIS/${ROTDIR}\//}/${head}aerostat" + if [[ ${DO_AERO} = "YES" ]]; then + if [[ -s "${COM_CHEM_ANALYSIS}/${head}aerostat" ]]; then + echo "${COM_CHEM_ANALYSIS/${ROTDIR}\//}/${head}aerostat" + fi fi if [[ -s "${COM_ATMOS_ANALYSIS}/${head}radstat" ]]; then echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}radstat" diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver index 4bedeb1e96..144388c14f 100644 --- a/versions/run.hercules.ver +++ b/versions/run.hercules.ver @@ -1,5 +1,6 @@ export stack_intel_ver=2021.9.0 export stack_impi_ver=2021.9.0 +export intel_mkl_ver=2023.1.0 export ncl_ver=6.6.2