diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index 7e1203c16feff..21743702c60d1 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -156,7 +156,6 @@ jobs: -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON - -DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }} ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }} -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}} -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}} @@ -184,7 +183,8 @@ jobs: - name: Test adapters env: ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir build --output-on-failure -L "conformance" --timeout 600 -VV + LIT_OPTS: " --timeout 120" + run: cmake --build build -j $(nproc) -- check-unified-runtime-conformance - name: Get information about platform if: ${{ always() }} diff --git a/unified-runtime/CMakeLists.txt b/unified-runtime/CMakeLists.txt index 637111db49f5c..75ec8cf02643f 100644 --- a/unified-runtime/CMakeLists.txt +++ b/unified-runtime/CMakeLists.txt @@ -65,8 +65,7 @@ set(UR_SYCL_LIBRARY_DIR "" CACHE PATH set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING "List of sycl targets to build CTS device binaries for") set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for") -option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON) -option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" OFF) +set(UR_CONFORMANCE_SELECTOR "" CACHE STRING "If nonempty, the device selector for conformance tests") option(UR_USE_DEBUG_POSTFIX "Enable debug postfix 'd' for libraries" OFF) set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH "Path to external 'level_zero' adapter source dir") diff --git a/unified-runtime/README.md b/unified-runtime/README.md index 0f0d5a00236b6..7ffdb1d8208b1 100644 --- a/unified-runtime/README.md +++ b/unified-runtime/README.md @@ -128,8 +128,7 @@ List of options provided by CMake: | UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON | | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 | | UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` | -| UR_CONFORMANCE_ENABLE_MATCH_FILES | Enable CTS match files | ON/OFF | ON | -| UR_CONFORMANCE_TEST_LOADER | Additionally build and run "loader" tests for the CTS | ON/OFF | OFF | +| UR_CONFORMANCE_SELECTOR | `ONEAPI_DEVICE_SELECTOR` for conformance testing | string | All enabled adapters | | UR_BUILD_ADAPTER_L0 | Build the Level-Zero adapter | ON/OFF | OFF | | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter | ON/OFF | OFF | | UR_BUILD_ADAPTER_CUDA | Build the CUDA adapter | ON/OFF | OFF | diff --git a/unified-runtime/scripts/core/CONTRIB.rst b/unified-runtime/scripts/core/CONTRIB.rst index 1289e7d027156..33a01689d2e3a 100644 --- a/unified-runtime/scripts/core/CONTRIB.rst +++ b/unified-runtime/scripts/core/CONTRIB.rst @@ -272,14 +272,21 @@ Conformance tests *must* not make assumptions about the adapter under test. Tests fixtures or cases *must* query for support of optional features and skip testing if unsupported by the adapter. -All tests in the Unified Runtime project are configured to use CTest to run. -All conformance tests have the ``conformance`` label attached to them which -allows them to be run independently. To run all the conformance tests, execute -the following command from the build directory. +Conformance tests are ran as part of the main ``test`` target, but can also be +executed using the `check-unified-runtime-conformance` target as follows: .. code-block:: console - ctest -L "conformance" + cmake --build build --target check-unified-runtime-conformance + +A specific device/adapter can be specified by using ``ONEAPI_DEVICE_SELECTOR`` +in the same way as in the `DPC++ compiler +`_. + +A number of tests are skipped on certain adapters due to being known failures. +To force all tests to run, including known failures, +``UR_CTS_ALSO_RUN_KNOWN_FAILURES`` may be set to ``1`` as an environment +variable. Experimental Features ===================== diff --git a/unified-runtime/scripts/core/INTRO.rst b/unified-runtime/scripts/core/INTRO.rst index e95bdf8944404..302216a4cee32 100644 --- a/unified-runtime/scripts/core/INTRO.rst +++ b/unified-runtime/scripts/core/INTRO.rst @@ -417,26 +417,12 @@ CTS Environment Variables The following environment variables are used by the CTS runner and can be used to specify the platform that the test framework should run on. This can be used during development and testing to run CTS tests in case multiple platforms -are available. If both filters are specified, then they both must match a platform for it to be selected. If there are -no valid platforms, then the tests will fail. Command line arguments take priority over these variables. +are available. Conformance tests will be skipped if there are no valid platforms. -.. envvar:: UR_CTS_ADAPTER_PLATFORM +.. envvar:: ONEAPI_DEVICE_SELECTOR - A specifier list in the form of `[(backend):](platform name)[;[(backend)]:(platform name)]...`. If a backend - specific specifier is present in the list and the test is running for that backend, the device with the given name - is chosen. Otherwise, it must match the name of the specifier from the list with no backend. Backend names are case- - insensitive, however platform names must match exactly. - - For example, if the test device has multiple platforms and you want to run tests on the "ABC Corp" backend when - testing OpenCL and "XYZ Org" when testing level zero, you'd use `OPENCL:ABC Corp;LEVEL_ZERO:XYZ Org`. This form is - useful when running the `build` target with a build with multiple backends. - - For testing only one platform, the backend can be omitted. For example, just `ABC Corp` is sufficient if the tests - are only going to be testing OpenCL. - -.. envvar:: UR_CTS_BACKEND - - A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on. + If ``ONEAPI_DEVICE_SELECTOR`` is set in the environment, only devices + matched by the selector will be tested. .. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES diff --git a/unified-runtime/scripts/ctest_parser.py b/unified-runtime/scripts/ctest_parser.py deleted file mode 100755 index 97e2270b47a2d..0000000000000 --- a/unified-runtime/scripts/ctest_parser.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2022 Intel Corporation -# -# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -# See LICENSE.TXT -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from subprocess import Popen, DEVNULL, PIPE -import argparse -import os -import json - -TMP_RESULTS_FILE = "tmp-results-file.json" - - -def get_cts_test_suite_names(working_directory): - process = Popen( - ["ctest", "--show-only=json-v1"], - cwd=working_directory, - stdout=PIPE, - env=os.environ.copy(), - ) - out, _ = process.communicate() - testsuites = json.loads(out) - return [test["name"][: test["name"].rfind("-")] for test in testsuites["tests"]] - - -def percent(amount, total): - return round((amount / (total or 1)) * 100, 2) - - -def summarize_results(results): - total = results["Total"] - total_passed = len(results["Passed"]) - total_skipped = len(results["Skipped"]) - total_failed = len(results["Failed"]) - total_crashed = total - (total_passed + total_skipped + total_failed) - - pass_rate_incl_skipped = percent(total_passed + total_skipped, total) - pass_rate_excl_skipped = percent(total_passed, total) - - skipped_rate = percent(total_skipped, total) - failed_rate = percent(total_failed, total) - crash_rate = percent(total_crashed, total) - - ljust_param = len(str(total)) - - print( - f"""[CTest Parser] Results: - Total [{str(total).ljust(ljust_param)}] - Passed [{str(total_passed).ljust(ljust_param)}] ({pass_rate_incl_skipped}%) - ({pass_rate_excl_skipped}% with skipped tests excluded) - Skipped [{str(total_skipped).ljust(ljust_param)}] ({skipped_rate}%) - Failed [{str(total_failed).ljust(ljust_param)}] ({failed_rate}%) - Crashed [{str(total_crashed).ljust(ljust_param)}] ({crash_rate}%) -""" - ) - - -def parse_results(results): - parsed_results = { - "Passed": {}, - "Skipped": {}, - "Failed": {}, - "Crashed": {}, - "Total": 0, - "Success": True, - } - for _, result in results.items(): - if result["actual"] is None: - parsed_results["Success"] = False - parsed_results["Total"] += result["expected"]["tests"] - continue - - parsed_results["Total"] += result["actual"]["tests"] - for testsuite in result["actual"].get("testsuites"): - for test in testsuite.get("testsuite"): - test_name = f"{testsuite['name']}.{test['name']}" - test_time = test["time"] - if "failures" in test: - parsed_results["Failed"][test_name] = {"time": test_time} - elif test["result"] == "SKIPPED": - parsed_results["Skipped"][test_name] = {"time": test_time} - else: - parsed_results["Passed"][test_name] = {"time": test_time} - return parsed_results - - -def run(args): - results = {} - - tmp_results_file = f"{args.ctest_path}/{TMP_RESULTS_FILE}" - env = os.environ.copy() - env["GTEST_OUTPUT"] = f"json:{tmp_results_file}" - - test_suite_names = get_cts_test_suite_names(f"{args.ctest_path}/test/conformance/") - - ## try and list all the available tests - for suite in test_suite_names: - results[suite] = {} - test_executable = f"{args.ctest_path}/bin/test-{suite}" - process = Popen( - [test_executable, "--gtest_list_tests"], - env=env, - stdout=DEVNULL if args.quiet else None, - stderr=DEVNULL if args.quiet else None, - ) - process.wait() - try: - with open(tmp_results_file, "r") as test_list: - all_tests = json.load(test_list) - results[suite]["expected"] = all_tests - os.remove(tmp_results_file) - except FileNotFoundError: - print(f"Could not discover tests for {suite}") - - for suite in test_suite_names: - ctest_path = f"{args.ctest_path}/test/conformance/{suite}" - process = Popen( - ["ctest", ctest_path], - env=env, - cwd=ctest_path, - stdout=DEVNULL if args.quiet else None, - stderr=DEVNULL if args.quiet else None, - ) - process.wait() - - try: - with open(tmp_results_file, "r") as results_file: - json_data = json.load(results_file) - results[suite]["actual"] = json_data - os.remove(tmp_results_file) - except FileNotFoundError: - results[suite]["actual"] = None - print( - "\033[91m" - + f"Conformance test suite '{suite}' : likely crashed!" - + "\033[0m" - ) - - return results - - -def dir_path(string): - if os.path.isdir(string): - return os.path.abspath(string) - else: - raise NotADirectoryError(string) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "ctest_path", - type=dir_path, - nargs="?", - default=".", - help="Optional path to test directory containing " - "CTestTestfile. Defaults to current directory.", - ) - parser.add_argument( - "-q", "--quiet", action="store_true", help="Output only failed tests." - ) - args = parser.parse_args() - - raw_results = run(args) - parsed_results = parse_results(raw_results) - summarize_results(parsed_results) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - exit(130) diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 099a51d24f33c..ee66d9ac54fdd 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -31,8 +31,9 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py) add_custom_target(deps_check-unified-runtime) if(UR_STANDALONE_BUILD) add_custom_target(check-unified-runtime - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv DEPENDS deps_check-unified-runtime + USES_TERMINAL ) else() # Use the LLVM method to add the test suite - this also registers it under `check-all` @@ -60,7 +61,8 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv + USES_TERMINAL ) else() add_lit_testsuite(${TARGET} "Running Unified Runtime ${suite} tests" diff --git a/unified-runtime/test/conformance/CMakeLists.txt b/unified-runtime/test/conformance/CMakeLists.txt index 9e29e727e2e2a..3e489e9651289 100644 --- a/unified-runtime/test/conformance/CMakeLists.txt +++ b/unified-runtime/test/conformance/CMakeLists.txt @@ -3,94 +3,58 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +add_ur_lit_testsuite(conformance DEPENDS urinfo) + +set(UR_CONFORMANCE_TEST_DIR "${CMAKE_CURRENT_SOURCE_DIR}" CACHE INTERNAL "Internal cache variable for conformance test root") set(UR_CONFORMANCE_DEVICE_BINARIES_DIR "${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL "Internal cache variable for device binaries directory") -function(add_test_adapter name adapter backend) - set(TEST_TARGET_NAME test-${name}) - set(TEST_NAME ${name}-${adapter}) - - set(TEST_COMMAND - "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --gtest_filter=*${backend}*" - ) - - function(do_add_test tname env) - separate_arguments(TEST_COMMAND) - add_test(NAME ${tname} - COMMAND ${TEST_COMMAND} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - set_tests_properties(${tname} PROPERTIES - ENVIRONMENT "${env}" - LABELS "conformance;${adapter}") - endfunction() - - do_add_test(${TEST_NAME} UR_ADAPTERS_FORCE_LOAD="$") - if(UR_CONFORMANCE_TEST_LOADER) - do_add_test(${TEST_NAME}-loader "") - endif() -endfunction() - function(add_conformance_test name) - set(TEST_TARGET_NAME test-${name}) - add_ur_executable(${TEST_TARGET_NAME} + add_gtest_test(${name} ${ARGN} ${UR_CONFORMANCE_TEST_DIR}/source/environment.cpp - ${UR_CONFORMANCE_TEST_DIR}/source/main.cpp) - target_link_libraries(${TEST_TARGET_NAME} - PRIVATE - ${PROJECT_NAME}::loader - ${PROJECT_NAME}::headers - ${PROJECT_NAME}::testing - ${PROJECT_NAME}::common - GTest::gtest_main) - - if(UR_USE_CFI) - target_compile_definitions(${TEST_TARGET_NAME} PRIVATE UR_USE_CFI) - endif() - - if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL) - add_test_adapter(${name} adapter_cuda CUDA) - endif() - if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL) - add_test_adapter(${name} adapter_hip HIP) - endif() - if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL) - add_test_adapter(${name} adapter_level_zero Level_Zero) - endif() - if(UR_BUILD_ADAPTER_L0_V2) - add_test_adapter(${name} adapter_level_zero_v2 Level_Zero) - endif() - if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL) - add_test_adapter(${name} adapter_opencl OpenCL) - endif() - if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL) - add_test_adapter(${name} adapter_native_cpu NATIVE_CPU) - endif() + ${UR_CONFORMANCE_TEST_DIR}/source/main.cpp + ) endfunction() -function(add_conformance_test_with_kernels_environment name) +function(add_conformance_kernels_test name) add_conformance_test(${name} ${ARGN}) - target_compile_definitions("test-${name}" PRIVATE KERNELS_ENVIRONMENT PRIVATE + target_compile_definitions(${name}-test PRIVATE KERNELS_ENVIRONMENT PRIVATE KERNELS_DEFAULT_DIR="${UR_CONFORMANCE_DEVICE_BINARIES_DIR}") - target_include_directories("test-${name}" PRIVATE ${UR_CONFORMANCE_DEVICE_BINARIES_DIR}) - add_dependencies("test-${name}" generate_device_binaries) + target_include_directories(${name}-test PRIVATE ${UR_CONFORMANCE_DEVICE_BINARIES_DIR}) + add_dependencies(${name}-test generate_device_binaries kernel_names_header) endfunction() -function(add_conformance_test_with_devices_environment name) +function(add_conformance_devices_test name) add_conformance_test(${name} ${ARGN}) - target_compile_definitions("test-${name}" PRIVATE DEVICES_ENVIRONMENT) + target_compile_definitions(${name}-test PRIVATE DEVICES_ENVIRONMENT) endfunction() -function(add_conformance_test_with_platform_environment name) +function(add_conformance_platform_test name) add_conformance_test(${name} ${ARGN}) - target_compile_definitions("test-${name}" PRIVATE PLATFORM_ENVIRONMENT) + target_compile_definitions(${name}-test PRIVATE PLATFORM_ENVIRONMENT) endfunction() +foreach(adapter ${UR_ADAPTERS_LIST}) + if(NOT "${adapter}" STREQUAL "mock") + if(UR_STANDALONE_BUILD) + add_custom_target(check-unified-runtime-conformance-${adapter} + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -v -Dselector=${adapter}:* + DEPENDS deps_check-unified-runtime-conformance + ) + else() + add_lit_testsuite(check-unified-runtime-conformance-${adapter} "Running Unified Runtime ${adapter} conformance tests" + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS FileCheck deps_check-unified-runtime-conformance + PARAMS selector=${adapter}:* + EXCLUDE_FROM_CHECK_ALL + ) + endif() + endif() +endforeach() + add_subdirectory(testing) add_subdirectory(adapter) diff --git a/unified-runtime/test/conformance/context/CMakeLists.txt b/unified-runtime/test/conformance/context/CMakeLists.txt index d6a754583f562..07174206c8ad1 100644 --- a/unified-runtime/test/conformance/context/CMakeLists.txt +++ b/unified-runtime/test/conformance/context/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(context +add_conformance_devices_test(context urContextCreate.cpp urContextCreateWithNativeHandle.cpp urContextGetInfo.cpp diff --git a/unified-runtime/test/conformance/cts_exe.py b/unified-runtime/test/conformance/cts_exe.py deleted file mode 100755 index 3afaa3e9763d5..0000000000000 --- a/unified-runtime/test/conformance/cts_exe.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -""" -Copyright (C) 2024 Intel Corporation - -Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -See LICENSE.TXT -SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -""" -# Printing conformance test output from gtest and checking failed tests with match files. -# The match files contain tests that are expected to fail. - -import os -import sys -import argparse -import signal -import subprocess # nosec B404 - - -def _ci(): - return os.environ.get("CI") is not None - - -def _color(): - return sys.stdout.isatty() or os.environ.get("GTEST_COLOR").lower() == "yes" - - -def _print_header(header, *args): - if _ci(): - # GitHub CI interprets this as a "group header" and will provide buttons to fold/unfold it - print("##[group]{}".format(header.format(*args))) - elif _color(): - # Inverse color - print("\033[7m{}\033[27m".format(header.format(*args))) - else: - print("### {}".format(header.format(*args))) - - -def _print_end_header(): - if _ci(): - print("##[endgroup]") - - -def _print_error(header, *args): - if _color(): - # "!!!" on a red background - print("\033[41m!!!\033[0m {}".format(header.format(*args))) - else: - print("!!! {}".format(header.format(*args))) - - -def _print_format(msg, *args): - print(msg.format(*args)) - - -def _print_environ(env): - _print_header("Environment") - for k, v in env.items(): - _print_format("> {} = {}", k, v) - _print_end_header() - - -def _print_failure(code, stdout): - # Display some context in the CI log so the user doesn't have to click down the lines - if _ci(): - _print_format("stdout/stderr of failure:") - print(stdout) - - signalname = "???" - try: - signalname = signal.strsignal(abs(code)) - except ValueError: - pass - _print_format("Got error code {} ({})", code, signalname) - - -def _check_filter(cmd, filter): - """ - Checks that the filter matches at least one test for the given cmd - """ - sys.stdout.flush() - check = subprocess.Popen( # nosec B603 - cmd + ["--gtest_list_tests"], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - env=(os.environ | {"GTEST_FILTER": filter}), - ) - if not check.stdout.read(1): - return False - return True - - -def _run_cmd(cmd, comment, filter): - _print_header("Running suite for: {}", comment) - _print_format("### {}", " ".join(cmd)) - - # Check tests are found - if not _check_filter(cmd, filter): - _print_end_header() - _print_error("Could not find any tests with this filter") - return (2, "") - - sys.stdout.flush() - proc = subprocess.Popen( # nosec B603 - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=(os.environ | {"GTEST_FILTER": filter}), - ) - stdout = proc.communicate()[0].decode("utf-8") - returncode = proc.wait() - print(stdout) - _print_end_header() - return (returncode, stdout) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--test_command", help="Ctest test case") - parser.add_argument("--failslist", type=str, help="Failure list") - parser.add_argument("--", dest="ignored", action="store_true") - parser.add_argument("rest", nargs=argparse.REMAINDER) - args = parser.parse_args() - - if args.test_command is None or args.failslist is None: - print( - "Usage: cts_exe.py --test_command (test binary) --failslist (match file) -- (test arguments)" - ) - sys.exit(1) - - base_invocation = [args.test_command] + args.rest - - if os.environ.get("GTEST_OUTPUT") is not None: - # We are being ran purely to generate an output file (likely for ctest_parser.py); falling back to just using - # one test execution - sys.exit( - subprocess.call( # nosec B603 - base_invocation, stdout=sys.stdout, stderr=sys.stderr - ) - ) - - _print_environ(os.environ) - - # Parse fails list - _print_format("Loading fails from {}", args.failslist) - fail_patterns = [] - with open(args.failslist) as f: - for l in f: - optional = "{{OPT}}" in l - l = l.replace("{{OPT}}", "") - - if l.startswith("#"): - continue - if l.strip() == "": - continue - - fail_patterns.append( - { - "pattern": l.strip(), - "optional": optional, - } - ) - - _print_header("Known failing tests") - for fail in fail_patterns: - _print_format("> {}", fail) - _print_end_header() - if len(fail_patterns) == 0: - _print_error( - "Fail list is empty, if there are no more failures, please remove the file" - ) - sys.exit(2) - - final_result = 0 - - # First, run all the known good tests - gtest_filter = "-" + (":".join(map(lambda x: x["pattern"], fail_patterns))) - if _check_filter(base_invocation, gtest_filter): - (result, stdout) = _run_cmd(base_invocation, "known good tests", gtest_filter) - if result != 0: - _print_error("Tests we expected to pass have failed") - _print_failure(result, stdout) - final_result = result - else: - _print_format("Note: No tests in this suite are expected to pass") - - # Then run each known failing tests - for fail in fail_patterns: - (result, stdout) = _run_cmd( - base_invocation, "failing test {}".format(fail["pattern"]), fail["pattern"] - ) - - if result == 0 and not fail["optional"]: - _print_error( - "Test {} is passing when we expect it to fail!", fail["pattern"] - ) - _print_failure(result, stdout) - final_result = 1 - - sys.exit(final_result) diff --git a/unified-runtime/test/conformance/device/CMakeLists.txt b/unified-runtime/test/conformance/device/CMakeLists.txt index b376eab767b02..98f3868ea9ee3 100644 --- a/unified-runtime/test/conformance/device/CMakeLists.txt +++ b/unified-runtime/test/conformance/device/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(device +add_conformance_devices_test(device urDeviceCreateWithNativeHandle.cpp urDeviceGet.cpp urDeviceGetGlobalTimestamps.cpp diff --git a/unified-runtime/test/conformance/enqueue/CMakeLists.txt b/unified-runtime/test/conformance/enqueue/CMakeLists.txt index 5856d24257a36..74fc593cd5615 100644 --- a/unified-runtime/test/conformance/enqueue/CMakeLists.txt +++ b/unified-runtime/test/conformance/enqueue/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(enqueue +add_conformance_kernels_test(enqueue urEnqueueDeviceGlobalVariableRead.cpp urEnqueueDeviceGlobalVariableWrite.cpp urEnqueueEventsWait.cpp diff --git a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopy.cpp b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopy.cpp index 1c536ec9d912c..8adc4a7712574 100644 --- a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopy.cpp +++ b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopy.cpp @@ -110,6 +110,8 @@ using urEnqueueMemBufferCopyMultiDeviceTest = UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urEnqueueMemBufferCopyMultiDeviceTest); TEST_P(urEnqueueMemBufferCopyMultiDeviceTest, CopyReadDifferentQueues) { + UUR_KNOWN_FAILURE_ON(uur::CUDA{}); + // First queue does a fill. const uint32_t input = 42; ASSERT_SUCCESS(urEnqueueMemBufferFill( diff --git a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopyRect.cpp b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopyRect.cpp index bfce4fb71fd27..6161f9cdfe562 100644 --- a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopyRect.cpp +++ b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferCopyRect.cpp @@ -316,6 +316,8 @@ using urEnqueueMemBufferCopyRectMultiDeviceTest = UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urEnqueueMemBufferCopyRectMultiDeviceTest); TEST_P(urEnqueueMemBufferCopyRectMultiDeviceTest, CopyRectReadDifferentQueues) { + UUR_KNOWN_FAILURE_ON(uur::CUDA{}); + // First queue does a fill. const uint32_t input = 42; ASSERT_SUCCESS(urEnqueueMemBufferFill( diff --git a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferWriteRect.cpp b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferWriteRect.cpp index a7c5a7cd7ec7a..0769d11db01de 100644 --- a/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferWriteRect.cpp +++ b/unified-runtime/test/conformance/enqueue/urEnqueueMemBufferWriteRect.cpp @@ -87,7 +87,7 @@ TEST_P(urEnqueueMemBufferWriteRectTestWithParam, Success) { UUR_KNOWN_FAILURE_ON(uur::HIP{}); } - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); // Unpack the parameters. const auto host_size = getParam().src_size; diff --git a/unified-runtime/test/conformance/enqueue/urEnqueueMemImageCopy.cpp b/unified-runtime/test/conformance/enqueue/urEnqueueMemImageCopy.cpp index edfcc554604e5..808e2f3da2d43 100644 --- a/unified-runtime/test/conformance/enqueue/urEnqueueMemImageCopy.cpp +++ b/unified-runtime/test/conformance/enqueue/urEnqueueMemImageCopy.cpp @@ -269,6 +269,8 @@ using urEnqueueMemImageCopyMultiDeviceTest = UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urEnqueueMemImageCopyMultiDeviceTest); TEST_P(urEnqueueMemImageCopyMultiDeviceTest, CopyReadDifferentQueues) { + UUR_KNOWN_FAILURE_ON(uur::CUDA{}); + ur_mem_handle_t dstImage1D = nullptr; ASSERT_SUCCESS(urMemImageCreate(context, UR_MEM_FLAG_READ_WRITE, &format, &desc1D, nullptr, &dstImage1D)); diff --git a/unified-runtime/test/conformance/event/CMakeLists.txt b/unified-runtime/test/conformance/event/CMakeLists.txt index c692a73d2ff71..c739f822a814e 100644 --- a/unified-runtime/test/conformance/event/CMakeLists.txt +++ b/unified-runtime/test/conformance/event/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(event +add_conformance_devices_test(event urEventGetInfo.cpp urEventGetProfilingInfo.cpp urEventWait.cpp diff --git a/unified-runtime/test/conformance/event/urEventGetProfilingInfo.cpp b/unified-runtime/test/conformance/event/urEventGetProfilingInfo.cpp index b8b37df7d13c8..e4f3d1f894ee3 100644 --- a/unified-runtime/test/conformance/event/urEventGetProfilingInfo.cpp +++ b/unified-runtime/test/conformance/event/urEventGetProfilingInfo.cpp @@ -91,8 +91,8 @@ TEST_P(urEventGetProfilingInfoTest, SuccessCommandComplete) { } TEST_P(urEventGetProfilingInfoTest, Success) { - // AMD devices may report a "start" time before the "submit" time - UUR_KNOWN_FAILURE_ON(uur::HIP{}); + // AMD/CUDA devices may report a "start" time before the "submit" time + UUR_KNOWN_FAILURE_ON(uur::HIP{}, uur::CUDA{}); // If a and b are supported, asserts that a <= b auto test_timing = [=](ur_profiling_info_t a, ur_profiling_info_t b) { diff --git a/unified-runtime/test/conformance/exp_command_buffer/CMakeLists.txt b/unified-runtime/test/conformance/exp_command_buffer/CMakeLists.txt index 1c79cb79857ae..3fb47baeb4337 100644 --- a/unified-runtime/test/conformance/exp_command_buffer/CMakeLists.txt +++ b/unified-runtime/test/conformance/exp_command_buffer/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(exp_command_buffer +add_conformance_kernels_test(exp_command_buffer release.cpp retain.cpp commands.cpp diff --git a/unified-runtime/test/conformance/exp_command_buffer/native-command/CMakeLists.txt b/unified-runtime/test/conformance/exp_command_buffer/native-command/CMakeLists.txt index 7e941f48e9a72..aecbbb7f73f68 100644 --- a/unified-runtime/test/conformance/exp_command_buffer/native-command/CMakeLists.txt +++ b/unified-runtime/test/conformance/exp_command_buffer/native-command/CMakeLists.txt @@ -4,22 +4,22 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception if (UR_BUILD_ADAPTER_CUDA) - target_sources(test-exp_command_buffer PRIVATE "append_native_cuda.cpp") - target_link_libraries(test-exp_command_buffer PRIVATE cudadrv) + target_sources(exp_command_buffer-test PRIVATE "append_native_cuda.cpp") + target_link_libraries(exp_command_buffer-test PRIVATE cudadrv) endif() if (UR_BUILD_ADAPTER_HIP) - target_sources(test-exp_command_buffer PRIVATE "append_native_hip.cpp") - target_link_libraries(test-exp_command_buffer PRIVATE rocmdrv) + target_sources(exp_command_buffer-test PRIVATE "append_native_hip.cpp") + target_link_libraries(exp_command_buffer-test PRIVATE rocmdrv) endif() if (UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_L0_v2) - target_sources(test-exp_command_buffer PRIVATE "append_native_L0.cpp") - target_link_libraries(test-exp_command_buffer PRIVATE LevelZeroLoader LevelZeroLoader-Headers) + target_sources(exp_command_buffer-test PRIVATE "append_native_L0.cpp") + target_link_libraries(exp_command_buffer-test PRIVATE LevelZeroLoader LevelZeroLoader-Headers) endif() if (UR_BUILD_ADAPTER_OPENCL) - target_sources(test-exp_command_buffer PRIVATE "append_native_opencl.cpp") - target_compile_definitions(test-exp_command_buffer PRIVATE CL_TARGET_OPENCL_VERSION=300) - target_link_libraries(test-exp_command_buffer PRIVATE OpenCLICDLoader) + target_sources(exp_command_buffer-test PRIVATE "append_native_opencl.cpp") + target_compile_definitions(exp_command_buffer-test PRIVATE CL_TARGET_OPENCL_VERSION=300) + target_link_libraries(exp_command_buffer-test PRIVATE OpenCLICDLoader) endif() diff --git a/unified-runtime/test/conformance/exp_enqueue_native/CMakeLists.txt b/unified-runtime/test/conformance/exp_enqueue_native/CMakeLists.txt index 8638fa1349724..d950bf1956fc9 100644 --- a/unified-runtime/test/conformance/exp_enqueue_native/CMakeLists.txt +++ b/unified-runtime/test/conformance/exp_enqueue_native/CMakeLists.txt @@ -4,15 +4,15 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception if (UR_BUILD_ADAPTER_CUDA) - add_conformance_test_with_kernels_environment( + add_conformance_kernels_test( exp_enqueue_native enqueue_native_cuda.cpp ) - target_include_directories(test-exp_enqueue_native PRIVATE + target_include_directories(exp_enqueue_native-test PRIVATE ${PROJECT_SOURCE_DIR}/source ${PROJECT_SOURCE_DIR}/source/adapters/cuda ) - target_link_libraries(test-exp_enqueue_native PRIVATE cudadrv) + target_link_libraries(exp_enqueue_native-test PRIVATE cudadrv) endif() # TODO: Add more tests for different triples diff --git a/unified-runtime/test/conformance/exp_launch_properties/CMakeLists.txt b/unified-runtime/test/conformance/exp_launch_properties/CMakeLists.txt index db598831494d2..82b88259b97b0 100644 --- a/unified-runtime/test/conformance/exp_launch_properties/CMakeLists.txt +++ b/unified-runtime/test/conformance/exp_launch_properties/CMakeLists.txt @@ -3,7 +3,6 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(exp_launch_properties +add_conformance_kernels_test(exp_launch_properties launch_properties.cpp ) - diff --git a/unified-runtime/test/conformance/exp_usm_p2p/CMakeLists.txt b/unified-runtime/test/conformance/exp_usm_p2p/CMakeLists.txt index ab6edd1badf96..dd1af592c6599 100644 --- a/unified-runtime/test/conformance/exp_usm_p2p/CMakeLists.txt +++ b/unified-runtime/test/conformance/exp_usm_p2p/CMakeLists.txt @@ -3,6 +3,6 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(exp_usm_p2p +add_conformance_devices_test(exp_usm_p2p usm_p2p.cpp ) diff --git a/unified-runtime/test/conformance/integration/CMakeLists.txt b/unified-runtime/test/conformance/integration/CMakeLists.txt index 1689cf04f4376..e9e6f87bc9455 100644 --- a/unified-runtime/test/conformance/integration/CMakeLists.txt +++ b/unified-runtime/test/conformance/integration/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(integration +add_conformance_kernels_test(integration QueueEmptyStatus.cpp QueueUSM.cpp QueueBuffer.cpp diff --git a/unified-runtime/test/conformance/kernel/CMakeLists.txt b/unified-runtime/test/conformance/kernel/CMakeLists.txt index 73ab3f1101a05..a39aef8f875c7 100644 --- a/unified-runtime/test/conformance/kernel/CMakeLists.txt +++ b/unified-runtime/test/conformance/kernel/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(kernel +add_conformance_kernels_test(kernel urKernelCreate.cpp urKernelCreateWithNativeHandle.cpp urKernelGetGroupInfo.cpp diff --git a/unified-runtime/test/conformance/lit.cfg.py b/unified-runtime/test/conformance/lit.cfg.py index 24e6c2e5c1381..ed172be7cfd41 100644 --- a/unified-runtime/test/conformance/lit.cfg.py +++ b/unified-runtime/test/conformance/lit.cfg.py @@ -7,5 +7,50 @@ """ -# Disable lit testing (for now) -config.suffixes = [] +import lit.formats +import subprocess +from os import path + +config.name = "Unified Runtime Conformance" +config.test_format = lit.formats.GoogleTest("", "-test") +config.test_source_root = config.binary_dir +config.test_exec_root = config.binary_dir + +config.environment["UR_LOADER_USE_LEVEL_ZERO_V2"] = "1" if config.using_l0_v2 else "0" + +if "UR_CTS_ALSO_RUN_KNOWN_FAILURES" in os.environ: + config.environment["UR_CTS_ALSO_RUN_KNOWN_FAILURES"] = os.environ[ + "UR_CTS_ALSO_RUN_KNOWN_FAILURES" + ] + +if from_param := lit_config.params.get("selector"): + # Specified via passing `-Dselector="backend:*"` to lit + selector = from_param +elif from_env := os.environ.get("ONEAPI_DEVICE_SELECTOR"): + lit_config.warning( + "ONEAPI_DEVICE_SELECTOR set in environment, using that for UR conformance tests" + ) + selector = from_env +elif config.default_selector: + selector = config.default_selector +else: + selector = ";".join([f"{a}:*" for a in config.adapters_built if a != "mock"]) +config.environment["ONEAPI_DEVICE_SELECTOR"] = selector + +if not selector: + lit_config.warning( + "No device selector is set (are any adapters enabled?), conformance tests may not run as expected" + ) + +lit_config.note( + f"Running Unified Runtime conformance tests with ONEAPI_DEVICE_SELECTOR='{selector}':" +) +urinfo = subprocess.check_output( + path.join(config.runtime_dir, "urinfo"), + text=True, + env=config.environment, + stderr=subprocess.DEVNULL, +) +for l in urinfo.splitlines(): + if l: + lit_config.note(f" {l}") diff --git a/unified-runtime/test/conformance/lit.site.cfg.py.in b/unified-runtime/test/conformance/lit.site.cfg.py.in new file mode 100644 index 0000000000000..94dcb634c11cc --- /dev/null +++ b/unified-runtime/test/conformance/lit.site.cfg.py.in @@ -0,0 +1,10 @@ +config.runtime_dir = "@CMAKE_RUNTIME_OUTPUT_DIRECTORY@" +config.binary_dir = "@CMAKE_CURRENT_BINARY_DIR@" +config.using_l0_v2 = "@UR_BUILD_ADAPTER_L0_V2@" == "ON" +config.default_selector = "@UR_CONFORMANCE_SELECTOR@" + +config.adapters_built = [] +if r"@UR_ADAPTERS_LIST@": + config.adapters_built = r"@UR_ADAPTERS_LIST@".split(";") + +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py") diff --git a/unified-runtime/test/conformance/memory-migrate/CMakeLists.txt b/unified-runtime/test/conformance/memory-migrate/CMakeLists.txt index 785a99aa8417e..2b26265ae29a0 100644 --- a/unified-runtime/test/conformance/memory-migrate/CMakeLists.txt +++ b/unified-runtime/test/conformance/memory-migrate/CMakeLists.txt @@ -3,5 +3,5 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(memory-migrate +add_conformance_kernels_test(memory-migrate urMemBufferMigrateAcrossDevices.cpp) diff --git a/unified-runtime/test/conformance/memory/CMakeLists.txt b/unified-runtime/test/conformance/memory/CMakeLists.txt index 041f73a0792a5..4258dfa4afa44 100644 --- a/unified-runtime/test/conformance/memory/CMakeLists.txt +++ b/unified-runtime/test/conformance/memory/CMakeLists.txt @@ -3,15 +3,15 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(memory - urMemBufferCreate.cpp - urMemBufferCreateWithNativeHandle.cpp - urMemBufferPartition.cpp - urMemGetInfo.cpp - urMemGetNativeHandle.cpp +add_conformance_devices_test(memory + urMemBufferCreate.cpp + urMemBufferCreateWithNativeHandle.cpp + urMemBufferPartition.cpp + urMemGetInfo.cpp + urMemGetNativeHandle.cpp urMemImageCreate.cpp urMemImageCreateWithImageFormatParam.cpp - urMemImageCreateWithNativeHandle.cpp - urMemImageGetInfo.cpp - urMemRelease.cpp + urMemImageCreateWithNativeHandle.cpp + urMemImageGetInfo.cpp + urMemRelease.cpp urMemRetain.cpp) diff --git a/unified-runtime/test/conformance/platform/CMakeLists.txt b/unified-runtime/test/conformance/platform/CMakeLists.txt index e07689324be1c..0c0fb42182019 100644 --- a/unified-runtime/test/conformance/platform/CMakeLists.txt +++ b/unified-runtime/test/conformance/platform/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_platform_environment(platform +add_conformance_platform_test(platform urPlatformCreateWithNativeHandle.cpp urPlatformGet.cpp urPlatformGetApiVersion.cpp diff --git a/unified-runtime/test/conformance/program/CMakeLists.txt b/unified-runtime/test/conformance/program/CMakeLists.txt index 4db93881f44dd..d37423f8a40cf 100644 --- a/unified-runtime/test/conformance/program/CMakeLists.txt +++ b/unified-runtime/test/conformance/program/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(program +add_conformance_kernels_test(program urProgramBuild.cpp urProgramCompile.cpp urProgramCreateWithBinary.cpp diff --git a/unified-runtime/test/conformance/queue/CMakeLists.txt b/unified-runtime/test/conformance/queue/CMakeLists.txt index a8379b4d71f6f..3d2ab3a19c33e 100644 --- a/unified-runtime/test/conformance/queue/CMakeLists.txt +++ b/unified-runtime/test/conformance/queue/CMakeLists.txt @@ -3,12 +3,12 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(queue +add_conformance_devices_test(queue urQueueCreate.cpp - urQueueCreateWithNativeHandle.cpp + urQueueCreateWithNativeHandle.cpp urQueueFinish.cpp urQueueFlush.cpp urQueueGetInfo.cpp - urQueueGetNativeHandle.cpp + urQueueGetNativeHandle.cpp urQueueRetain.cpp urQueueRelease.cpp) diff --git a/unified-runtime/test/conformance/sampler/CMakeLists.txt b/unified-runtime/test/conformance/sampler/CMakeLists.txt index 92a53dd205850..c18dcbdd66873 100644 --- a/unified-runtime/test/conformance/sampler/CMakeLists.txt +++ b/unified-runtime/test/conformance/sampler/CMakeLists.txt @@ -3,11 +3,11 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(sampler +add_conformance_devices_test(sampler urSamplerCreate.cpp - urSamplerCreateWithNativeHandle.cpp - urSamplerGetInfo.cpp - urSamplerGetNativeHandle.cpp - urSamplerRelease.cpp + urSamplerCreateWithNativeHandle.cpp + urSamplerGetInfo.cpp + urSamplerGetNativeHandle.cpp + urSamplerRelease.cpp urSamplerRetain.cpp ) diff --git a/unified-runtime/test/conformance/source/environment.cpp b/unified-runtime/test/conformance/source/environment.cpp index d1fe951e5ce00..5f06857073665 100644 --- a/unified-runtime/test/conformance/source/environment.cpp +++ b/unified-runtime/test/conformance/source/environment.cpp @@ -128,11 +128,11 @@ DevicesEnvironment::DevicesEnvironment() : PlatformEnvironment() { for (auto &platform : platforms) { uint32_t platform_device_count = 0; - urDeviceGet(platform, UR_DEVICE_TYPE_ALL, 0, nullptr, - &platform_device_count); + urDeviceGetSelected(platform, UR_DEVICE_TYPE_ALL, 0, nullptr, + &platform_device_count); std::vector platform_devices(platform_device_count); - urDeviceGet(platform, UR_DEVICE_TYPE_ALL, platform_device_count, - platform_devices.data(), nullptr); + urDeviceGetSelected(platform, UR_DEVICE_TYPE_ALL, platform_device_count, + platform_devices.data(), nullptr); ur_adapter_handle_t adapter = nullptr; urPlatformGetInfo(platform, UR_PLATFORM_INFO_ADAPTER, sizeof(ur_adapter_handle_t), &adapter, nullptr); diff --git a/unified-runtime/test/conformance/usm/CMakeLists.txt b/unified-runtime/test/conformance/usm/CMakeLists.txt index f197447cbfe72..56da947a87310 100644 --- a/unified-runtime/test/conformance/usm/CMakeLists.txt +++ b/unified-runtime/test/conformance/usm/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_kernels_environment(usm +add_conformance_kernels_test(usm urUSMDeviceAlloc.cpp urUSMFree.cpp urUSMGetMemAllocInfo.cpp diff --git a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt index d05db83bd5a5d..14fa91c0c1360 100644 --- a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt +++ b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt @@ -3,7 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_conformance_test_with_devices_environment(virtual_memory +add_conformance_devices_test(virtual_memory urPhysicalMemCreate.cpp urPhysicalMemRelease.cpp urPhysicalMemRetain.cpp diff --git a/unified-runtime/test/lit.cfg.py b/unified-runtime/test/lit.cfg.py index 43e4a469c0223..53c1ede610552 100644 --- a/unified-runtime/test/lit.cfg.py +++ b/unified-runtime/test/lit.cfg.py @@ -68,6 +68,10 @@ def word_match(key, subst): ) config.available_features.add(f"adapter-{a}") +# If no adapters are built, don't include the conformance tests +if config.adapters_built == ["mock"]: + config.excludes.add("conformance") + config.substitutions.append(word_match("FileCheck", config.filecheck_path)) diff --git a/unified-runtime/third_party/requirements_testing.txt b/unified-runtime/third_party/requirements_testing.txt index 1e79c168c61fc..c41dbabde3fe3 100644 --- a/unified-runtime/third_party/requirements_testing.txt +++ b/unified-runtime/third_party/requirements_testing.txt @@ -1,3 +1,6 @@ # Note: Filecheck 1.0.1 requires Python 3.10 filecheck==1.0.1 lit==18.1.8 + +# For timeouts in lit +psutil==7.0.0