Skip to content

Commit 01eeb24

Browse files
TerrenceMcGuinness-NOAATerry McGuinnessaerorahul
authored
CI JJOB Tests using CMake (NOAA-EMC#3214)
Adding CI tests at the JJOB level using CMake/ctest wrappers: These individual JJOB tests have four distinct phases: - **Setup:** Creates a EXPDIR/COMROOT just for the individual JJOB - **Stage:** Moves the specific files into the COMROOT that are needed to run the JJOB specified in `${HOMEgfs}/ci/ctest/cases/{CASE}_{JJOB}.yaml` - **Execute:** Run the JJOB in batch (batch card extracted from XML via Rocoto) - **Validate:** Check the outputs also specified in the above yaml configure file (currently stubbed) Resolves NOAA-EMC#3204 --------- Co-authored-by: Terry McGuinness <[email protected]> Co-authored-by: Rahul Mahajan <[email protected]>
1 parent 57ce1b0 commit 01eeb24

13 files changed

+423
-0
lines changed

CMakeLists.txt

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# ------------------------------------------------------------------------- #
2+
# Global Workflow
3+
# ------------------------------------------------------------------------- #
4+
5+
# Check for minimum cmake requirement
6+
cmake_minimum_required( VERSION 3.20 FATAL_ERROR )
7+
8+
project(global_workflow VERSION 1.0.0)
9+
10+
include(GNUInstallDirs)
11+
enable_testing()
12+
13+
# Build type.
14+
if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
15+
message(STATUS "Setting build type to 'Release' as none was specified.")
16+
set(CMAKE_BUILD_TYPE
17+
"Release"
18+
CACHE STRING "Choose the type of build." FORCE)
19+
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
20+
"MinSizeRel" "RelWithDebInfo")
21+
endif()
22+
23+
# Build global-workflow source codes
24+
# add_subdirectory(sorc)
25+
26+
# Setup tests
27+
add_subdirectory(ctests)

ci/platforms/config.hera

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
44
export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
5+
6+
export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
57
export HPC_ACCOUNT=nems
68
export max_concurrent_cases=5
79
export max_concurrent_pr=4

ci/platforms/config.orion

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
44
export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
5+
export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
56
export HPC_ACCOUNT=nems
67
export max_concurrent_cases=5
78
export max_concurrent_pr=4

ctests/CMakeLists.txt

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# ------------------------------------------------------------------------- #
2+
# CTests for Global Workflow
3+
# ------------------------------------------------------------------------- #
4+
# These ctests correspond to JJOBs (individual Rocoto jobs) that can be
5+
# run independently, each requiring its own YAML definition of inputs
6+
# and configurations. By integrating with Rocoto, these jobs can be
7+
# validated, staged, and executed as self-contained tests using
8+
# their own data and test parameters.
9+
# ------------------------------------------------------------------------- #
10+
11+
# Function to set a variable from an environment variable or default value
12+
function(set_from_env_or_default VAR_NAME ENV_VAR DEFAULT_VALUE)
13+
if (DEFINED ENV{${ENV_VAR}} AND NOT DEFINED ${VAR_NAME})
14+
set(${VAR_NAME} $ENV{${ENV_VAR}} CACHE STRING "Set from environment variable ${ENV_VAR}")
15+
elseif(NOT DEFINED ${VAR_NAME} AND NOT ${DEFAULT_VALUE} STREQUAL "")
16+
set(${VAR_NAME} ${DEFAULT_VALUE} CACHE STRING "Default value for ${VAR_NAME}")
17+
endif()
18+
endfunction()
19+
20+
# Set HOMEgfs
21+
if (NOT DEFINED HOMEgfs)
22+
set(HOMEgfs ${PROJECT_SOURCE_DIR})
23+
endif()
24+
25+
# Set RUNTESTS
26+
set_from_env_or_default(RUNTESTS RUNTESTS "${CMAKE_CURRENT_BINARY_DIR}/RUNTESTS")
27+
28+
# Set HPC_ACCOUNT
29+
set_from_env_or_default(HPC_ACCOUNT HPC_ACCOUNT " ")
30+
if (NOT DEFINED HPC_ACCOUNT)
31+
message(WARNING "HPC_ACCOUNT must be set. CTests will not be created.")
32+
return()
33+
endif()
34+
35+
# Set ICSDIR_ROOT
36+
set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "")
37+
if (NOT DEFINED ICSDIR_ROOT)
38+
message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.")
39+
return()
40+
endif()
41+
42+
# Set STAGED_TESTS_DIR
43+
set_from_env_or_default(STAGED_TESTS_DIR STAGED_TESTS_DIR "")
44+
if (NOT DEFINED STAGED_TESTS_DIR)
45+
message(WARNING "STAGED_TESTS_DIR must be set. CTests will not be created.")
46+
return()
47+
endif()
48+
49+
message(STATUS "gw: global-workflow baselines will be used from: '${HOMEgfs}'")
50+
message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
51+
message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
52+
message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
53+
message(STATUS "gw: global-workflow tests will use staged data from: '${STAGED_TESTS_DIR}'")
54+
55+
# Prepare test scripts
56+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in
57+
${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY)
58+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in
59+
${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY)
60+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in
61+
${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY)
62+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/validate.sh.in
63+
${CMAKE_CURRENT_BINARY_DIR}/scripts/validate.sh @ONLY)
64+
65+
function(AddJJOBTest)
66+
67+
set(prefix ARG)
68+
set(novals NOTRAPFPE NOVALGRIND)
69+
set(singlevals CASE JOB TEST_DATE)
70+
set(multivals TEST_DEPENDS)
71+
72+
cmake_parse_arguments(${prefix}
73+
"${novals}" "${singlevals}" "${multivals}"
74+
${ARGN})
75+
76+
set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
77+
set(CASE_PATH ${HOMEgfs}/ci/cases/pr)
78+
set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
79+
80+
add_test(NAME test_${TEST_NAME}_setup
81+
COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE}
82+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
83+
set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}")
84+
85+
add_test(NAME test_${TEST_NAME}_stage
86+
COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE}
87+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
88+
set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}")
89+
90+
add_test(NAME test_${TEST_NAME}_execute
91+
COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE}
92+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
93+
set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}")
94+
95+
# TODO - This is a stub for the validation step
96+
add_test(NAME test_${TEST_NAME}_validate
97+
COMMAND ./validate.sh ${TEST_NAME} ${CASE_YAML}
98+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
99+
set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
100+
endfunction()
101+
102+
AddJJOBTest(
103+
CASE "C48_ATM"
104+
JOB "gfs_fcst_seg0"
105+
TEST_DATE "2021032312"
106+
)

ctests/README.md

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# CTest Framework for NOAA Global Workflow
2+
3+
This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations.
4+
5+
## Overview
6+
7+
The CTest framework consists of the following scripts:
8+
- **setup.sh.in**: Prepares the environment and creates the experiment.
9+
- **stage.sh.in**: Stages the input files needed to run a JJOB.
10+
- **execute.sh.in**: Executes the JJOB and monitors its status.
11+
- **validate.sh.in**: (TODO) Validates the results of the JJOB.
12+
13+
## Usage
14+
15+
### CMake Configuration
16+
17+
To configure the CTest framework using CMake, you need to provide several environment variables or default values. Here is an example of how to configure and build the project:
18+
19+
```bash
20+
# Set environment variables (may also be include at command line with -D)
21+
export HPC_ACCOUNT="your_hpc_account"
22+
export ICSDIR_ROOT="/path/to/icsdir_root"
23+
export STAGED_TESTS_DIR="/path/to/staged_tests_dir"
24+
25+
# Run CMake to configure the ctest framework
26+
cmake -S /path/to/HOMEgfs -B /path/to/build -DRUNTESTS=/path/to/runtests
27+
28+
```
29+
30+
### Running Tests with CTest
31+
32+
Once the project is configured, you can run the tests using CTest. Here are some examples:
33+
34+
#### Run All Tests
35+
36+
```bash
37+
cd /path/to/build
38+
ctest
39+
```
40+
41+
#### Run Tests for a Specific Case
42+
43+
You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case:
44+
45+
```bash
46+
cd /path/to/build
47+
ctest -L C48_ATM
48+
```
49+
50+
To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows:
51+
```cmake
52+
AddJJOBTest(
53+
CASE "C48_ATM"
54+
JOB "gfs_fcst_seg0"
55+
TEST_DATE "2021032312"
56+
)
57+
```
58+
Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml`
+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
input_files:
2+
mkdir:
3+
- "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input"
4+
copy:
5+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_ctrl.nc"]
6+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile1.nc"]
7+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile2.nc"]
8+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile3.nc"]
9+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile4.nc"]
10+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile5.nc"]
11+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile6.nc"]
12+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile1.nc"]
13+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile2.nc"]
14+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile3.nc"]
15+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile4.nc"]
16+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile5.nc"]
17+
- ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile6.nc"]

ctests/scripts/execute.sh.in

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
3+
set -xe
4+
5+
TEST_NAME=${1:?"Name of the test is required"}
6+
JOB=${2:?"Job name is required"}
7+
# TODO - adding idate by hand for now, need to get this from the test somehow
8+
idate=$3
9+
10+
#TODO - add rocotoboot_dryrun to repo some how
11+
rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
12+
CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
13+
cd "${CASEDIR}"
14+
rm -f ./*.db
15+
rm -f ./jobcard
16+
17+
yes | "${rocotoboot_dryrun}" -d "${TEST_NAME}.db" -w "${TEST_NAME}.xml" -v 10 -c "${idate}00" -t "${JOB}" 2> jobcard || true
18+
sed '/^{{\|^}}/d' < jobcard | sed '1d' > "${TEST_NAME}.sub" || true
19+
20+
#TODO - Generalize for batch system (hard coded to slurm)
21+
22+
output=$(sbatch "${TEST_NAME}.sub")
23+
job_id=$(echo "${output}" | awk '{print $4}')
24+
echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}"
25+
26+
# First loop: wait until job appears
27+
lack_of_job_count=0
28+
LACK_OF_JOB_LIMIT=5
29+
30+
while true; do
31+
job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1) || true
32+
if [[ -n "${job_status}" ]]; then
33+
echo "Job ${job_id} found in sacct."
34+
break
35+
fi
36+
echo "Job ${job_id} not in sacct yet, attempt ${lack_of_job_count}/${LACK_OF_JOB_LIMIT}."
37+
lack_of_job_count=$((lack_of_job_count + 1))
38+
if [[ "${lack_of_job_count}" -ge "${LACK_OF_JOB_LIMIT}" ]]; then
39+
echo "Job ${job_id} not found after ${lack_of_job_count} attempts. Exiting."
40+
exit 1
41+
fi
42+
sleep 30
43+
done
44+
45+
# Second loop: monitor job status until completion or failure
46+
timeout=0
47+
TIMEOUT=60
48+
while true; do
49+
# Trim trailing spaces from job_status
50+
job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | xargs) || true
51+
if [[ "${job_status}" == "COMPLETED" ]]; then
52+
echo "Job ${job_id} completed successfully."
53+
break
54+
elif [[ "${job_status}" =~ ^(FAILED|CANCELLED|TIMEOUT)$ ]]; then
55+
echo "Job ${job_id} failed with status: ${job_status}."
56+
exit 1
57+
else
58+
echo "Job ${job_id} is still running with status: ${job_status}."
59+
sleep 60
60+
timeout=$((timeout + 1))
61+
if [[ "${timeout}" -gt "${TIMEOUT}" ]]; then
62+
echo "Job ${job_id} has been running for more than ${TIMEOUT} minutes. Exiting."
63+
exit 1
64+
fi
65+
fi
66+
done

ctests/scripts/setup.sh.in

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
3+
set -ux
4+
5+
TEST_NAME=${1:?"Name of the test is required"}
6+
YAML_FILE=${2:?"Name of the CI yaml file for the test"}
7+
8+
# CMake to fill these variables
9+
HOMEgfs="@PROJECT_SOURCE_DIR@"
10+
RUNTESTS="@RUNTESTS@"
11+
ICSDIR_ROOT="@ICSDIR_ROOT@"
12+
HPC_ACCOUNT="@HPC_ACCOUNT@"
13+
14+
set +x
15+
source "${HOMEgfs}/workflow/gw_setup.sh"
16+
set -x
17+
18+
pslot="${TEST_NAME}" \
19+
RUNTESTS="${RUNTESTS}" \
20+
ICSDIR_ROOT="${ICSDIR_ROOT}" \
21+
HPC_ACCOUNT="${HPC_ACCOUNT}" \
22+
"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite
23+
rc=$?
24+
if [[ "${rc}" -ne 0 ]]; then
25+
set +x
26+
echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'"
27+
set -x
28+
exit "${rc}"
29+
fi
30+
31+
exit 0

ctests/scripts/stage.py

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env python3
2+
3+
"""
4+
stage.py
5+
6+
This script is part of the ctest framework for testing Rocoto JJOBS that stages the
7+
input files needed to run a JJOB independently from other jobs in the workflow.
8+
The YAML file specified at the command line contains the paths to the staged input files
9+
and their corresponding directories under the COMROOT of the experiment for the JJOB.
10+
11+
Usage:
12+
stage.py -y <yaml_file> [-d <test_date>]
13+
14+
Arguments:
15+
-y, --yaml Path to the YAML file describing the job test configuration (required)
16+
-d, --test_date Test date in YYYYMMDDHH format (optional)
17+
18+
Example:
19+
./stage.py -y /path/to/config.yaml -d 2021032312
20+
"""
21+
22+
import os
23+
import datetime
24+
25+
from argparse import ArgumentParser
26+
from pathlib import Path
27+
from wxflow import parse_j2yaml, FileHandler, Logger
28+
29+
# Initialize logger with environment variable for logging level
30+
logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
31+
32+
33+
def parse_args():
34+
"""
35+
Parse command line arguments.
36+
37+
Returns
38+
-------
39+
argparse.Namespace
40+
The parsed command line arguments, including:
41+
- yaml: Path to the YAML file describing the job test configuration.
42+
- test_date: Optional test date in YYYYMMDDHH format.
43+
"""
44+
description = """Arguments for creating and updating error log files
45+
"""
46+
parser = ArgumentParser(description=description)
47+
48+
# Add argument for YAML file path
49+
parser.add_argument('-y', '--yaml', help='full path to yaml file describing the job test configuration', type=Path, required=True)
50+
# Add optional argument for test date
51+
parser.add_argument('-d', '--test_date', help='test date in YYYYMMDDHH format', type=str, required=False)
52+
return parser.parse_args()
53+
54+
55+
if __name__ == '__main__':
56+
57+
# Parse command line arguments
58+
args = parse_args()
59+
60+
data = {}
61+
if args.test_date:
62+
# Parse test date from string to datetime object
63+
data['TEST_DATE'] = datetime.datetime.strptime(args.test_date, '%Y%m%d%H')
64+
# Parse YAML configuration file with optional data
65+
case_cfg = parse_j2yaml(path=args.yaml, data=data)
66+
# Synchronize input files as per the parsed configuration
67+
FileHandler(case_cfg.input_files).sync()

0 commit comments

Comments
 (0)