Add Hercules build capability to GDASApp

build.sh

@@ -70,7 +70,7 @@ while getopts "p:t:c:hvdfa" opt; do
 done
 
 case ${BUILD_TARGET} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Building GDASApp on $BUILD_TARGET"
     source $dir_root/ush/module-setup.sh
     module use $dir_root/modulefiles

modulefiles/GDAS/hercules.lua

@@ -0,0 +1,98 @@
+help([[
+Load environment for running the GDAS application with Intel compilers and MPI.
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core')
+prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')
+
+-- below two lines get us access to the spack-stack modules
+load("stack-intel/2021.9.0")
+load("stack-intel-oneapi-mpi/2021.9.0")
+--load("stack-python/3.10.8")
+-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed
+load("cmake/3.23.1")
+load("curl/8.1.2")
+load("zlib/1.2.13")
+load("git/2.40.0")
+--load("pkg-config/0.27.1")
+load("hdf5/1.14.0")
+load("parallel-netcdf/1.12.2")
+load("netcdf-c/4.9.2")
+load("nccmp/1.9.0.1")
+load("netcdf-fortran/4.6.0")
+load("nco/5.0.6")
+load("parallelio/2.5.10")
+load("wget/1.21.1")
+load("boost/1.78")
+load("bufr/12.0.1")
+load("git-lfs/3.1.2")
+load("ecbuild/3.7.2")
+load("openjpeg/2.4.0")
+load("eccodes/2.27.0")
+load("eigen/3.4.0")
+load("openblas/0.3.19")
+load("eckit/1.24.4")
+load("fftw/3.3.10")
+load("fckit/0.11.0")
+load("fiat/1.2.0")
+load("ectrans/1.2.0")
+load("atlas/0.35.0")
+load("sp/2.3.3")
+load("gsl-lite/0.37.0")
+load("libjpeg/2.1.0")
+load("krb5/1.20.1")
+load("libtirpc/1.2.6")
+load("hdf/4.2.15")
+load("jedi-cmake/1.4.0")
+load("libpng/1.6.37")
+load("libxt/1.1.5")
+load("libxmu/1.1.4")
+load("libxpm/3.5.12")
+load("libxaw/1.0.13")
+load("udunits/2.2.28")
+load("ncview/2.1.8")
+load("netcdf-cxx4/4.3.1")
+load("py-pybind11/2.8.1")
+--load("crtm/v2.4_jedi")
+load("contrib/0.1")
+load("noaatools/3.1")
+load("rocoto/1.3.5")
+
+load("hpc/1.2.0")
+unload("python/3.10.8")
+unload("python/3.9.2")
+load("miniconda3/4.6.14")
+load("gdasapp/1.0.0")
+
+load("boost/1.78.0")
+load("bufr/12.0.1")
+load("fckit/0.11.0")
+load("atlas/0.35.0")
+load("py-pybind11/2.8.1")
+
+-- below is a hack because of cmake finding the wrong python...
+setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/")
+
+setenv("CC","mpiicc")
+setenv("FC","mpiifort")
+setenv("CXX","mpiicpc")
+local mpiexec = '/opt/slurm/bin/srun'
+local mpinproc = '-n'
+setenv('MPIEXEC_EXEC', mpiexec)
+setenv('MPIEXEC_NPROC', mpinproc)
+
+setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml')
+setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0")
+setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data")
+prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
+
+execute{cmd="ulimit -s unlimited",modeA={"load"}}
+
+whatis("Name: ".. pkgName)
+whatis("Version: ".. pkgVersion)
+whatis("Category: GDASApp")
+whatis("Description: Load all libraries needed for GDASApp")

test/atm/global-workflow/jjob_ens_final.sh

@@ -33,22 +33,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 fi

test/atm/global-workflow/jjob_ens_init.sh

@@ -38,11 +38,11 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
@@ -69,15 +69,15 @@ dpath=gdas.$PDY/$cyc/obs
 mkdir -p $COM_OBS
 flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file
 done
 
 # Link radiance bias correction files
 dpath=gdas.$gPDY/$gcyc/analysis/atmos
 mkdir -p $COM_ATMOS_ANALYSIS_PREV
 flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file
 done
 
 # Link member atmospheric background on tiles and atmf006
@@ -104,10 +104,10 @@ for imem in $(seq 1 $NMEM_ENS); do
 done
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 fi

test/atm/global-workflow/jjob_ens_run.sh

@@ -36,22 +36,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
-elif [ $machine = 'ORION' ]; then
-    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --partition=orion --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 fi

test/atm/global-workflow/jjob_var_final.sh

@@ -33,22 +33,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
 fi

test/atm/global-workflow/jjob_var_init.sh

@@ -38,11 +38,11 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
@@ -71,23 +71,23 @@ dpath=gdas.$PDY/$cyc/obs
 mkdir -p $COM_OBS
 flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file
 done
 
 # Link radiance bias correction files
 dpath=gdas.$gPDY/$gcyc/analysis/atmos
 mkdir -p $COM_ATMOS_ANALYSIS_PREV
 flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file
 done
 
 # Link atmospheric background on gaussian grid
 dpath=gdas.$gPDY/$gcyc/model_data/atmos/history
 mkdir -p $COM_ATMOS_HISTORY_PREV
 flist="atmf006.nc"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/${gprefix}.${file}
 done
 
 # Link atmospheric bacgkround on tiles
@@ -96,7 +96,7 @@ COM_ATMOS_RESTART_PREV_DIRNAME=$(dirname $COM_ATMOS_RESTART_PREV)
 mkdir -p $COM_ATMOS_RESTART_PREV_DIRNAME
 flist="restart"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/$file
 done
 
 
@@ -125,10 +125,10 @@ done
 
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
 fi

test/atm/global-workflow/jjob_var_run.sh

@@ -36,22 +36,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
 fi

test/soca/gw/CMakeLists.txt

@@ -11,9 +11,14 @@ add_test(NAME test_gdasapp_soca_prep
 
 # Identify machine
 set(MACHINE "container")
-IF (IS_DIRECTORY /work2/noaa/da)
-  set(MACHINE "orion")
-  set(PARTITION "orion")
+IF (IS_DIRECTORY /work2)
+  IF (IS_DIRECTORY /apps/other)
+    set(MACHINE "hercules")
+    set(PARTITION "hercules")
+  ELSE()
+    set(MACHINE "orion")
+    set(PARTITION "orion")
+  ENDIF()
 ENDIF()
 IF (IS_DIRECTORY /scratch2/NCEPDEV/)
   set(MACHINE "hera")

ush/detect_machine.sh

@@ -25,6 +25,8 @@ case $(hostname -f) in
 
   Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4
 
+  Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4
+
   cheyenne[1-6].cheyenne.ucar.edu)     MACHINE_ID=cheyenne ;; ### cheyenne1-6
   cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6
   chadmin[1-6].ib0.cheyenne.ucar.edu)  MACHINE_ID=cheyenne ;; ### cheyenne1-6

ush/module-setup.sh

@@ -22,6 +22,13 @@ elif [[ $MACHINE_ID = orion* ]] ; then
     fi
     module purge
 
+elif [[ $MACHINE_ID = hercules* ]] ; then
+    # We are on Hercules
+    if ( ! eval module help > /dev/null 2>&1 ) ; then
+        source /apps/lmod/init/bash
+    fi
+    module purge
+
 elif [[ $MACHINE_ID = s4* ]] ; then
     # We are on SSEC Wisconsin S4
     if ( ! eval module help > /dev/null 2>&1 ) ; then

ush/soca/run_jjobs.py

@@ -6,7 +6,7 @@
 import argparse
 from datetime import datetime, timedelta
 
-machines = {"container", "hera", "orion"}
+machines = {"container", "hera", "orion", "hercules"}
 
 # Assume the default conda environement is gdassapp
 ENVS = {'JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY': 'eva'}