Skip to content

Commit f67d903

Browse files
authored
Merge pull request #673 from casparvl/map_tests_to_software_builds
Map tests to software builds
2 parents 50e7edd + 93b699f commit f67d903

File tree

5 files changed

+145
-2
lines changed

5 files changed

+145
-2
lines changed

create_tarball.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ if [ ! -d ${software_dir_overlay} ]; then
3535
exit 3
3636
fi
3737

38+
current_workdir=${PWD}
3839
cd ${overlay_upper_dir}/versions/
3940
echo ">> Collecting list of files/directories to include in tarball via ${PWD}..."
4041

@@ -88,6 +89,9 @@ echo "wrote file list to ${files_list}"
8889
echo "wrote module file list to ${module_files_list}"
8990
[ -r ${module_files_list} ] && cat ${module_files_list}
9091

92+
# Copy the module files list to current workindg dir for later use in the test step
93+
cp ${module_files_list} ${current_workdir}/module_files.list.txt
94+
9195
topdir=${cvmfs_repo}/versions/
9296

9397
echo ">> Creating tarball ${target_tgz} from ${topdir}..."

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ easyconfigs:
66
from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc
77
- gnuplot-5.4.6-GCCcore-12.2.0.eb
88
- h5py-3.8.0-foss-2022b.eb
9-
- MDAnalysis-2.4.2-foss-2022b.eb
9+
- MDAnalysis-2.4.2-foss-2022b.eb

test_suite.sh

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,22 @@ else
195195
fatal_error "Failed to run 'reframe --version'"
196196
fi
197197

198+
# Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node)
199+
module_list="module_files.list.txt"
200+
mapping_config="tests/eessi_test_mapping/software_to_tests.yml"
201+
# Run with --debug for easier debugging in case there are issues:
202+
python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug
203+
REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}")
204+
test_selection_exit_code=$?
205+
if [[ ${test_selection_exit_code} -eq 0 ]]; then
206+
echo_green "Succesfully extracted names of tests to run: ${REFRAME_NAME_ARGS}"
207+
else
208+
fatal_error "Failed to extract names of tests to run: ${REFRAME_NAME_ARGS}"
209+
exit ${test_selection_exit_code}
210+
fi
211+
export REFRAME_ARGS="--tag CI --tag 1_node --nocolor ${REFRAME_NAME_ARGS}"
212+
198213
# List the tests we want to run
199-
export REFRAME_ARGS='--tag CI --tag 1_node --nocolor'
200214
echo "Listing tests: reframe ${REFRAME_ARGS} --list"
201215
reframe ${REFRAME_ARGS} --list
202216
if [[ $? -eq 0 ]]; then
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import yaml
2+
import re
3+
import os
4+
import argparse
5+
6+
def load_mappings(file_path):
7+
"""Load the YAML mappings from a file."""
8+
if not os.path.exists(file_path):
9+
raise FileNotFoundError(f"Error: {file_path} does not exist.")
10+
with open(file_path, 'r') as file:
11+
config = yaml.safe_load(file)
12+
return config['mappings']
13+
14+
def read_software_names(file_path):
15+
"""Read software names from the module_files.list.txt file."""
16+
if not os.path.exists(file_path):
17+
raise FileNotFoundError(f"Error: {file_path} does not exist.")
18+
with open(file_path, 'r') as file:
19+
software_names = [line.strip() for line in file if line.strip()]
20+
return software_names
21+
22+
def get_tests_for_software(software_name, mappings):
23+
"""Get the list of tests for a given software name based on the first matching regex pattern."""
24+
25+
# Iterate over patterns in the order they appear in the YAML file
26+
for pattern, tests in mappings.items():
27+
if re.match(pattern, software_name):
28+
return tests
29+
30+
# If no matches are found, return the default tests if they exist
31+
if 'default_tests' in mappings:
32+
return mappings['default_tests']
33+
34+
return []
35+
36+
def main(yaml_file, module_file, debug):
37+
"""Main function to process software names and their tests."""
38+
mappings = load_mappings(yaml_file)
39+
if debug:
40+
print(f"Loaded mappings from '{yaml_file}'")
41+
42+
software_names = read_software_names(module_file)
43+
if debug:
44+
print(f"Read software names from '{module_file}'")
45+
46+
tests_to_run = []
47+
arg_string = ""
48+
# For each module name, get the relevant set of tests
49+
for software_name in software_names:
50+
additional_tests = get_tests_for_software(software_name, mappings)
51+
for test in additional_tests:
52+
if test not in tests_to_run:
53+
tests_to_run.append(test)
54+
55+
if additional_tests and debug:
56+
print(f"Software: {software_name} -> Tests: {additional_tests}")
57+
elif debug:
58+
print(f"Software: {software_name} -> No tests found")
59+
60+
# Always add the default set of tests, if default_tests is specified
61+
if 'default_tests' in mappings:
62+
additional_tests = mappings['default_tests']
63+
for test in additional_tests:
64+
if test not in tests_to_run:
65+
tests_to_run.append(test)
66+
67+
if additional_tests and debug:
68+
print(f"Adding default set of tests: {additional_tests}")
69+
70+
# Create argument string out of the list of tests to run
71+
if tests_to_run:
72+
arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run])
73+
74+
# Print final lists & argument string
75+
if debug:
76+
print(f"Full list of tests to run: {tests_to_run}")
77+
print(f"Argument string: {arg_string}")
78+
else:
79+
# This is the only thing this script should print, unless run with --debug
80+
print(f"{arg_string}")
81+
82+
if __name__ == "__main__":
83+
parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.")
84+
parser.add_argument('--mapping-file', type=str, help='Path to the YAML file containing the test mappings.')
85+
parser.add_argument('--module-list', type=str, help='Path to the file containing the list of software names.')
86+
parser.add_argument('--debug', action='store_true', help='Enable debug output.')
87+
88+
args = parser.parse_args()
89+
90+
main(args.mapping_file, args.module_list, args.debug)
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# This file creates a mapping between (regular expressions for) module names and test names from the EESSI test suite
2+
# If a module name matches one of the regular expressions, the listed set of tests will be run in the test step
3+
# For a given module name, the test list for the first matching regular expression is returned
4+
# E.g. for
5+
# mappings:
6+
# foo-v1:
7+
# - bar
8+
# foo-*
9+
# - bar2
10+
# only the bar test will be run for foo-v1 (even though it also matches the pattern (foo-*)
11+
# If a module name does not match anything, the default_tests will be run
12+
# Note that to list all available tests by name, one can do execute
13+
# reframe -R -c /path/to/eessi/test-suite/ --list | grep -Po "\bEESSI_\S+?(?=[\s'])" | uniq
14+
# Note that this regular expression is a bit sensitive to changes in the structure of ReFrame's output,
15+
# but is confirmed to work for ReFrame version 4.6.1
16+
mappings:
17+
PyTorch-Bundle/*:
18+
- EESSI_PyTorch_torchvision
19+
QuantumESPRESSO/*:
20+
- EESSI_QuantumESPRESSO
21+
CP2K/*:
22+
- EESSI_CP2K
23+
ESPResSo/*:
24+
- EESSI_ESPRESSO
25+
LAMMPS/*:
26+
- EESSI_LAMMPS
27+
OSU-Micro-Benchmarks/*:
28+
- EESSI_OSU_Micro_Benchmarks
29+
GROMACS/*:
30+
- EESSI_GROMACS
31+
default_tests:
32+
# Low level tests
33+
- EESSI_OSU_Micro_Benchmarks
34+
# A very quick-to-run high level application test
35+
- EESSI_LAMMPS

0 commit comments

Comments
 (0)