Enable HPSS via Globus

jobs/JGDAS_ENKF_GLOBUS_ARCH

@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "globus_earc" -c "base globus globus_earc"
+
+
+###############################################################
+# Run globus transfer script
+###############################################################
+
+"${SCRgfs}/exgdas_enkf_globus_earc.py"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+###############################################################
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/JGLOBAL_ARCHIVE_TARS

@@ -43,6 +43,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \
     COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \
     COMIN_CONF:COM_CONF_TMPL \
+    COMOUT_CONF:COM_CONF_TMPL \
     COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
 
 for grid in "0p25" "0p50" "1p00"; do

jobs/JGLOBAL_GLOBUS_ARCH

@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "globus_arch" -c "base globus"
+
+##############################################
+# Set variables used in the script
+##############################################
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_CONF:COM_CONF_TMPL
+
+###############################################################
+# Run globus script
+###############################################################
+
+${GLOBALGLOBUSARCHSH:-${SCRgfs}/exglobal_globus_arch.py}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/globus_arch.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+export job="globus_arch"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGLOBAL_GLOBUS_ARCH
+status=$?
+
+exit "${status}"

modulefiles/module_base.hercules.lua

@@ -36,17 +36,21 @@ load(pathJoin("met", (os.getenv("met_ver") or "None")))
 load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
 load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
+-- TODO remove this once we move to spack-stack 1.8.0+; as WGRIB2 will be assigned by the wgrib2 module
 setenv("WGRIB2","wgrib2")
 
--- Stop gap fix for wgrib with spack-stack 1.6.0
--- TODO Remove this when spack-stack issue #1097 is resolved
-setenv("WGRIB","wgrib")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
+-- Set the path for the Sven executables
+append_path("PATH", pathJoin((os.getenv("sven_root_path") or "None"), "bin"))
+
 prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
 
 prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
 
+-- Set the path for the globus package handler, Sven
+setenv("sven_dir", "/home/gfekete/sven")
+
 whatis("Description: GFS run environment")

modulefiles/module_base.orion.lua

@@ -34,12 +34,11 @@ load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "Non
 load(pathJoin("met", (os.getenv("met_ver") or "None")))
 load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
 load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+load(pathJoin("globus-cli", (os.getenv("globus_cli_ver") or "None")))
 
+-- TODO remove this once we move to spack-stack 1.8.0+; as WGRIB2 will be assigned by the wgrib2 module
 setenv("WGRIB2","wgrib2")
 
--- Stop gap fix for wgrib with spack-stack 1.6.0
--- TODO Remove this when spack-stack issue #1097 is resolved
-setenv("WGRIB","wgrib")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
 prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))

modulefiles/module_gwsetup.hercules.lua

@@ -16,5 +16,6 @@ load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
+try_load("globus-cli")
 
 whatis("Description: GFS run setup environment")

parm/config/gefs/config.arch_tars

@@ -0,0 +1 @@
+../gfs/config.arch_tars

parm/config/gefs/config.arch_vrfy

@@ -0,0 +1 @@
+../gfs/config.arch_vrfy

parm/config/gefs/config.base

@@ -334,24 +334,26 @@ export DO_METP="NO"         # Run METPLUS jobs - set METPLUS settings in config.
 export DO_FIT2OBS="NO"      # Run fit to observations package
 
 # Archiving options
-export VRFYARCH="@VRFYARCH@"        # save verification data locally
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
-export LOCALARCH="@LOCALARCH@"        # save data to local archive
-if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
-   echo "Both HPSS and local archiving selected.  Please choose one or the other."
-   exit 3
-elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then
+export LOCALARCH="@LOCALARCH@"      # save data to local archive
+export GLOBUSARCH="NO"              # send data to HPSS via globus xfers to Niagara (intended for MSU)
+count_arch_opts=0
+for arch_opt in "${HPSSARCH}" "${LOCALARCH}" "${GLOBUSARCH}"; do
+    if [[ "${arch_opt}" == "YES" ]]; then
+        (( count_arch_opts += 1 ));
+    fi
+done
+if [[ ${count_arch_opts} -gt 1 ]]; then
+   echo "FATAL ERROR: More than one archiving option selected.  Please choose no more than one."
+   exit 4
+elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]] || [[ ${GLOBUSARCH} = "YES" ]]; then
    export DO_ARCHTAR="YES"
 else
    export DO_ARCHTAR="NO"
 fi
-export ARCH_CYC=00           # Archive data at this cycle for warm start and/or forecast-only capabilities
-export ARCH_WARMICFREQ=4     # Archive frequency in days for warm start capability
-export ARCH_FCSTICFREQ=1     # Archive frequency in days for gdas and gfs forecast-only capability
-export ARCH_EXPDIR='YES'     # Archive the EXPDIR configs, XML, and database
-export ARCH_EXPDIR_FREQ=0    # How often to archive the EXPDIR in hours or 0 for first and last cycle only
-export ARCH_HASHES='YES'     # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR
-export ARCH_DIFFS='NO'       # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR
+
+# Globus UUID for this machine
+export CLIENT_GLOBUS_UUID='@CLIENT_GLOBUS_UUID@'
 
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}

parm/config/gefs/config.globus

@@ -0,0 +1 @@
+../gfs/config.globus

parm/config/gefs/config.resources

@@ -323,7 +323,7 @@ case ${step} in
     export is_exclusive=False
     ;;
 
-  "arch_tars")
+  "arch_tars" | "globus")
     export walltime="06:00:00"
     export ntasks=1
     export tasks_per_node=1

parm/config/gfs/config.arch_tars

@@ -8,8 +8,22 @@ echo "BEGIN: config.arch_tars"
 # Get task specific resources
 . "${EXPDIR}/config.resources" "arch_tars"
 
+export ARCH_CYC=00           # Archive data at this cycle for warm start and/or forecast-only capabilities
+export ARCH_WARMICFREQ=4     # Archive frequency in days for warm start capability
+export ARCH_FCSTICFREQ=1     # Archive frequency in days for gdas and gfs forecast-only capability
+export ARCH_EXPDIR='YES'     # Archive the EXPDIR configs, XML, and database
+export ARCH_EXPDIR_FREQ=0    # How often to archive the EXPDIR in hours or 0 for first and last cycle only
+export ARCH_HASHES='YES'     # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR
+export ARCH_DIFFS='NO'       # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR
+
 export ARCH_GAUSSIAN="YES"
-export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
-export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
+export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS:-}
+export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS:-}
+
+# If we are running globus archiving, create tarballs in a temporary location
+if [[ "${GLOBUSARCH}" == "YES" ]]; then
+    export ATARDIR="${DATAROOT}/archive_rotdir/${RUN}"
+    export LOCALARCH="YES"
+fi
 
 echo "END: config.arch_tars"

parm/config/gfs/config.arch_vrfy

@@ -8,8 +8,4 @@ echo "BEGIN: config.arch_vrfy"
 # Get task specific resources
 . "${EXPDIR}/config.resources" "arch_vrfy"
 
-export ARCH_GAUSSIAN="YES"
-export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
-export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
-
 echo "END: config.arch_vrfy"

parm/config/gfs/config.base

@@ -481,22 +481,25 @@ export DO_FETCH_LOCAL="NO"   # Copy from local disk onto COM
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
-export LOCALARCH="@LOCALARCH@"        # save data to local archive
-if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
-   echo "FATAL ERROR: Both HPSS and local archiving selected.  Please choose one or the other."
+export LOCALARCH="@LOCALARCH@"      # save data to local archive
+export GLOBUSARCH="NO"              # send data to HPSS via globus xfers to Niagara (intended for MSU)
+count_arch_opts=0
+for arch_opt in "${HPSSARCH}" "${LOCALARCH}" "${GLOBUSARCH}"; do
+    if [[ "${arch_opt}" == "YES" ]]; then
+        (( count_arch_opts += 1 ));
+    fi
+done
+if [[ ${count_arch_opts} -gt 1 ]]; then
+   echo "FATAL ERROR: More than one archiving option selected.  Please choose no more than one."
    exit 4
-elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then
+elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]] || [[ ${GLOBUSARCH} = "YES" ]]; then
    export DO_ARCHTAR="YES"
 else
    export DO_ARCHTAR="NO"
 fi
-export ARCH_CYC=00           # Archive data at this cycle for warm start and/or forecast-only capabilities
-export ARCH_WARMICFREQ=4     # Archive frequency in days for warm start capability
-export ARCH_FCSTICFREQ=1     # Archive frequency in days for gdas and gfs forecast-only capability
-export ARCH_EXPDIR='YES'     # Archive the EXPDIR configs, XML, and database
-export ARCH_EXPDIR_FREQ=0    # How often to archive the EXPDIR in hours or 0 for first and last cycle only
-export ARCH_HASHES='YES'     # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR
-export ARCH_DIFFS='NO'       # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR
+
+# Globus UUID for this machine
+export CLIENT_GLOBUS_UUID='@CLIENT_GLOBUS_UUID@'
 
 # The monitor jobs are not yet supported for JEDIATMVAR.
 if [[ ${DO_JEDIATMVAR} = "YES" ]]; then

parm/config/gfs/config.earc_tars

@@ -6,7 +6,7 @@
 echo "BEGIN: config.earc_tars"
 
 # Get task specific resources
-. "${EXPDIR}/config.resources" "earc_tars"
+. "${EXPDIR}/config.resources" "arch_tars"
 
 # Set the number of ensemble members to archive per earc_tars job
 case "${CASE_ENS}" in
@@ -32,4 +32,11 @@ esac
 export RMOLDSTD_ENKF=144
 export RMOLDEND_ENKF=24
 
+# If we are running globus archiving, create tarballs in a temporary location
+if [[ "${GLOBUSARCH}" == "YES" ]]; then
+    export ENSGRP="${ENSGRP:-0}"
+    export ATARDIR="${DATAROOT}/archive_rotdir/${pslot}/${RUN}/${ENSGRP}"
+    export LOCALARCH="YES"
+fi
+
 echo "END: config.earc_tars"

parm/config/gfs/config.globus

@@ -0,0 +1,34 @@
+#! /usr/bin/env bash
+
+########## config.globus ##########
+# Globus-specific variables for all globus-based jobs
+
+echo "BEGIN: config.globus"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" globus
+
+# Set the globus staging directory populated by the arch jobs
+export STAGE_DIR="${DATAROOT}/archive_rotdir/${PSLOT}"
+
+# Set variables used by the Sven and Doorman services
+# Niagara's globus UUID
+export SERVER_GLOBUS_UUID=1bfd8a79-52b2-4589-88b2-0648e0c0b35d
+
+# General delivery location on Niagara (staging area for data)
+export SERVER_HOME='/collab1/data/{{SERVER_USERNAME}}'
+
+# Sven's dropbox
+export SVEN_DROPBOX_ROOT="${DATA}/SVEN_DROPBOX"
+
+# Location of the doorman package on Niagara
+export DOORMAN_ROOT="/home/David.Huber/doorman"
+
+# Server name (should match ~/.ssh/config)
+export SERVER_NAME="niagara"
+
+# Server globus UUID
+niagara_UUID="1bfd8a79-52b2-4589-88b2-0648e0c0b35d"
+export SERVER_GLOBUS_UUID="${niagara_UUID}"
+
+echo "END: config.globus"

parm/config/gfs/config.resources

@@ -1059,7 +1059,7 @@ case ${step} in
     export is_exclusive=True
     ;;
 
-  "arch_tars" | "earc_tars" | "getic" | "fetch")
+  "arch_tars" | "getic" | "fetch" | "globus" | "globus_earc" )
     walltime="06:00:00"
     ntasks=1
     tasks_per_node=1

parm/config/gfs/yaml/defaults.yaml

@@ -24,6 +24,7 @@ base:
   FHMAX_ENKF_GFS: 12
   DOHYBVAR_OCN: "NO"
   DO_TEST_MODE: "NO"
+  GLOBUSARCH: "NO"
 
 atmanl:
   JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"