diff --git a/README.md b/README.md index 172c3e9c..2f57f3df 100644 --- a/README.md +++ b/README.md @@ -257,5 +257,6 @@ Find below the external actions listed: | [mirror-branch-action](external/mirror-branch-action/action.yml) | Mirror branch within the same repository | Apache-2.0 license | | [setup-ccache-action](external/setup-ccache-action/action.yml) | Setup CCache in a workflow | MIT license | | [setup-python](external/setup-python/action.yml) | Installs a version of Python or PyPy and (by default) adds it to the PATH | MIT license | +| [test-reporter](external/test-reporter/action.yml) | Display test results directly in GitHub | MIT license | | [upload-artifact](external/upload-artifact/action.yml) | Upload build artifact | MIT license | | [wait-on-check-action](external/wait-on-check-action/action.yml) | Wait on certain check | MIT license | diff --git a/external/test-reporter/action.yml b/external/test-reporter/action.yml new file mode 100644 index 00000000..4227fcf4 --- /dev/null +++ b/external/test-reporter/action.yml @@ -0,0 +1,157 @@ +name: 'test-reporter' +description: 'https://github.com/dorny/test-reporter' + +inputs: + artifact: + description: > + Name or regex of artifact containing test results. + Regular expression must be enclosed in '/'. + Values from captured groups will replace occurrences of $N in report name. + Example: + artifact: /test-results-(.*)/ + name: 'Test report $1' + -> Artifact 'test-result-ubuntu' would create report 'Test report ubuntu' + required: false + default: '' + + name: + description: > + Name of the Check Run which will be created + required: false + default: '' + + path: + description: > + Comma-separated list of paths to test results + Supports wildcards via [fast-glob](https://github.com/mrmlnc/fast-glob) + All matched result files must be of the same format + required: false + default: '' + + path-replace-backslashes: + description: > + The fast-glob library that is internally used interprets backslashes as escape characters. + If enabled, all backslashes in provided path will be replaced by forward slashes and act as directory separators. + It might be useful when path input variable is composed dynamically from existing directory paths on Windows. + required: false + default: 'false' + + reporter: + description: > + Format of test results. Supported options: + dart-json + dotnet-trx + flutter-json + java-junit (default) + jest-junit + mocha-json + required: false + default: 'java-junit' + + only-summary: + description: > + Allows you to generate only the summary. + If enabled, the report will contain a table listing each test results file and the number of passed, failed, and skipped tests. + Detailed listing of test suites and test cases will be skipped. + required: false + default: 'false' + + list-suites: + description: > + Limits which test suites are listed: + all + failed + required: false + default: 'all' + + list-tests: + description: > + Limits which test cases are listed: + all + failed + none + required: false + default: 'all' + + max-annotations: + description: > + Limits number of created annotations with error message and stack trace captured during test execution. + Must be less or equal to 50. + required: false + default: '10' + + fail-on-error: + description: > + Set action as failed if test report contains any failed test + required: false + default: 'true' + + fail-on-empty: + description: > + Set this action as failed if no test results were found + required: false + default: 'true' + + working-directory: + description: > + Relative path under $GITHUB_WORKSPACE where the repository was checked out. + required: false + default: '' + + token: + description: > + Personal access token used to interact with Github API + Default: github.token + required: false + default: ${{ github.token }} + +outputs: + conclusion: + description: "Success or failure" + value: ${{ steps.generate_test_report.outputs.conclusion }} + + passed: + description: "Count of passed tests" + value: ${{ steps.generate_test_report.outputs.passed }} + + failed: + description: "Count of failed tests" + value: ${{ steps.generate_test_report.outputs.failed }} + + skipped: + description: "Count of skipped tests" + value: ${{ steps.generate_test_report.outputs.skipped }} + + time: + description: "Test execution time [ms]" + value: ${{ steps.generate_test_report.outputs.time }} + + url: + description: "Check run URL" + value: ${{ steps.generate_test_report.outputs.url }} + + url_html: + description: "Check run URL HTML" + value: ${{ steps.generate_test_report.outputs.url_html }} + +runs: + using: composite + steps: + + - name: Generate Test Report + id: generate_test_report + uses: dorny/test-reporter@v1 + with: + artifact: ${{ inputs.artifact }} + name: ${{ inputs.name }} + path: ${{ inputs.path }} + path-replace-backslashes: ${{ inputs.path-replace-backslashes }} + reporter: ${{ inputs.reporter }} + only-summary: ${{ inputs.only-summary }} + list-suites: ${{ inputs.list-suites }} + list-tests: ${{ inputs.list-tests }} + max-annotations: ${{ inputs.max-annotations }} + fail-on-error: ${{ inputs.fail-on-error }} + fail-on-empty: ${{ inputs.fail-on-empty }} + working-directory: ${{ inputs.working-directory }} + token: ${{ inputs.token }} diff --git a/multiplatform/asan_build_test/action.yml b/multiplatform/asan_build_test/action.yml index f111b750..5bb2b5a0 100644 --- a/multiplatform/asan_build_test/action.yml +++ b/multiplatform/asan_build_test/action.yml @@ -32,11 +32,17 @@ inputs: required: false default: '' +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.build_and_test.outputs.ctest_results_path }} + runs: using: composite steps: - name: Build and test + id: build_and_test uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main with: packages_names: ${{ inputs.packages_names }} diff --git a/multiplatform/colcon_build_test/action.yml b/multiplatform/colcon_build_test/action.yml index f0fca6e0..76cc5db3 100644 --- a/multiplatform/colcon_build_test/action.yml +++ b/multiplatform/colcon_build_test/action.yml @@ -37,6 +37,11 @@ inputs: required: false default: '' +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.test.outputs.ctest_results_path }} + runs: using: composite steps: @@ -52,9 +57,11 @@ runs: cmake_build_type: ${{ inputs.cmake_build_type }} - name: Test + id: test uses: eProsima/eProsima-CI/multiplatform/colcon_test@main with: workspace: ${{ inputs.workspace }} - colcon_test_args: '--packages-select ${{ inputs.packages_names }} ${{ inputs.colcon_test_args }}' + packages_names: ${{ inputs.packages_names }} + colcon_test_args: ${{ inputs.colcon_test_args }} ctest_args: ' ${{ inputs.ctest_args }}' workspace_dependencies: ${{ inputs.workspace_dependencies }} diff --git a/multiplatform/colcon_build_test_flaky/action.yml b/multiplatform/colcon_build_test_flaky/action.yml index 7d0b8ff2..c4cc80ef 100644 --- a/multiplatform/colcon_build_test_flaky/action.yml +++ b/multiplatform/colcon_build_test_flaky/action.yml @@ -32,6 +32,11 @@ inputs: required: false default: '' +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.test.outputs.ctest_results_path }} + runs: using: composite steps: @@ -46,8 +51,11 @@ runs: workspace_dependencies: ${{ inputs.workspace_dependencies }} - name: Test + id: test uses: eProsima/eProsima-CI/multiplatform/colcon_test@main with: workspace: ${{ inputs.workspace }} - colcon_test_args: '--packages-select ${{ inputs.packages_names }} ${{ inputs.colcon_test_args }}' - ctest_args: ' --label-regex "xfail" ${{ inputs.ctest_args }}' + packages_names: ${{ inputs.packages_names }} + colcon_test_args: ${{ inputs.colcon_test_args }} + ctest_args: ' --label-regex "xfail" ${{ inputs.ctest_args }} ' + workspace_dependencies: ${{ inputs.workspace_dependencies }} diff --git a/multiplatform/colcon_test/action.yml b/multiplatform/colcon_test/action.yml index f0309cf5..5f8e8d97 100644 --- a/multiplatform/colcon_test/action.yml +++ b/multiplatform/colcon_test/action.yml @@ -23,6 +23,10 @@ inputs: required: false default: '--timeout 60' + packages_names: + description: 'Name of the colcon packages to build and test' + required: true + workspace: description: 'Workspace where built has been done' required: false @@ -33,11 +37,17 @@ inputs: required: false default: '' +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.test_ubuntu.outputs.ctest_results_path || steps.test_windows.outputs.ctest_results_path }} + runs: using: composite steps: - name: Run in ubuntu + id: test_ubuntu uses: eProsima/eProsima-CI/ubuntu/colcon_test@main if: runner.os == 'Linux' with: @@ -45,10 +55,12 @@ runs: colcon_test_args_default: ${{ inputs.colcon_test_args_default }} ctest_args: ${{ inputs.ctest_args }} ctest_args_default: ${{ inputs.ctest_args_default }} + packages_names: ${{ inputs.packages_names }} workspace: ${{ inputs.workspace }} workspace_dependencies: ${{ inputs.workspace_dependencies }} - name: Run in windows + id: test_windows uses: eProsima/eProsima-CI/windows/colcon_test@main if: runner.os == 'Windows' with: @@ -56,5 +68,6 @@ runs: colcon_test_args_default: ${{ inputs.colcon_test_args_default }} ctest_args: ${{ inputs.ctest_args }} ctest_args_default: ${{ inputs.ctest_args_default }} + packages_names: ${{ inputs.packages_names }} workspace: ${{ inputs.workspace }} workspace_dependencies: ${{ inputs.workspace_dependencies }} diff --git a/multiplatform/tsan_build_test/action.yml b/multiplatform/tsan_build_test/action.yml index 89e198d7..2e4bcf2c 100644 --- a/multiplatform/tsan_build_test/action.yml +++ b/multiplatform/tsan_build_test/action.yml @@ -32,11 +32,17 @@ inputs: required: false default: '' +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.build_and_test.outputs.ctest_results_path }} + runs: using: composite steps: - name: Build and test + id: build_and_test uses: eProsima/eProsima-CI/multiplatform/colcon_build_test@main with: packages_names: ${{ inputs.packages_names }} diff --git a/ubuntu/colcon_test/action.yml b/ubuntu/colcon_test/action.yml index 0860fdc3..e259c29f 100644 --- a/ubuntu/colcon_test/action.yml +++ b/ubuntu/colcon_test/action.yml @@ -23,6 +23,10 @@ inputs: required: false default: '--timeout 60' + packages_names: + description: 'Name of the colcon packages to build and test' + required: true + workspace: description: 'Workspace where built has been done' required: false @@ -33,13 +37,25 @@ inputs: required: false default: '' + test_report_artifact: + description: > + Name of the artifact to be uploaded containing the test report in JUnit format. + Default: test_report__ + required: false + default: '' + +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.test.outputs.ctest_results_path }} + runs: using: composite steps: - - name: Build workspace with colcon + - name: Run tests with colcon + id: test run: | - echo "::group::Test colcon ${{ inputs.workspace }}" if [[ ! -z "${{ inputs.workspace_dependencies }}" ]]; then @@ -47,14 +63,42 @@ runs: fi cd ${{ inputs.workspace }} - - colcon test \ - ${{ inputs.colcon_test_args_default }} \ - ${{ inputs.colcon_test_args }} \ - --ctest-args \ - ${{ inputs.ctest_args_default }} \ - ${{ inputs.ctest_args }} + mkdir test_results + echo "ctest_results_path=${{ inputs.workspace }}/test_results/" >> $GITHUB_OUTPUT echo "::endgroup::" + set +e + exit_code=0 + + for package in ${{ inputs.packages_names }}; do + echo "::group::Testing ${package} ..." + + colcon test \ + ${{ inputs.colcon_test_args_default }} \ + ${{ inputs.colcon_test_args }} \ + --packages-select ${package} \ + --ctest-args \ + ${{ inputs.ctest_args_default }} \ + ${{ inputs.ctest_args }} \ + --output-junit ${{ inputs.workspace }}/test_results/${package}_test_results.xml + + last_exit_code=$? + + if [ $last_exit_code -gt 0 ]; then + exit_code=$last_exit_code + fi + + echo "::endgroup::" + done + + exit $exit_code + shell: bash + + - name: Upload test report in JUnit format + uses: eProsima/eProsima-CI/external/upload-artifact@main + if: success() || failure() + with: + name: ${{ inputs.test_report_artifact || format('test_report_{0}_{1}', github.job, join(matrix.*, '_')) }} + path: ${{ inputs.workspace }}/test_results diff --git a/windows/colcon_test/action.yml b/windows/colcon_test/action.yml index 5f9ad12b..fd537a23 100644 --- a/windows/colcon_test/action.yml +++ b/windows/colcon_test/action.yml @@ -23,6 +23,10 @@ inputs: required: false default: '--timeout 60' + packages_names: + description: 'Name of the colcon packages to build and test' + required: true + workspace: description: 'Workspace where built has been done' required: false @@ -33,25 +37,75 @@ inputs: required: false default: '' + test_report_artifact: + description: > + Name of the artifact to be uploaded containing the test report in JUnit format + Default: test_report__ + required: false + default: '' + +outputs: + ctest_results_path: + description: "Path to test results" + value: ${{ steps.test.outputs.ctest_results_path }} + runs: using: composite steps: - - name: Build workspace with colcon + - name: Run tests with colcon + id: test shell: pwsh run: | - "::group::Run tests using colcon" + $ErrorActionPreference = 'Continue' + + Write-Host "::group::Run tests using colcon" if ([string]::IsNullOrEmpty("${{ inputs.workspace_dependencies }}")) { ${{ inputs.workspace_dependencies }}/setup.ps1 } cd ${{ inputs.workspace }} + $resultsPath = "${{ inputs.workspace }}\test_results\" + New-Item -ItemType Directory -Path $resultsPath + Add-Content -Path $env:GITHUB_OUTPUT -Value "ctest_results_path=$resultsPath" + + Write-Host "::endgroup::" + + $exitCode = 0 + $packages_names = "${{ inputs.packages_names }}" -split '\s+' + + foreach ($package in $packages_names) { + Write-Host "::group::Testing $package ..." + + $testCommand = @" + colcon test + ${{ inputs.colcon_test_args_default }} + ${{ inputs.colcon_test_args }} + --packages-select $package + --ctest-args + ${{ inputs.ctest_args_default }} + ${{ inputs.ctest_args }} + --output-junit "$resultsPath\${package}_test_results.xml" + "@ + + $testCommand = $testCommand -replace "[`r`n\t\s]+", " " -replace '^\s+', '' + + Invoke-Expression $testCommand + + if ($lastexitcode -ne 0) { + $exitCode = $lastexitcode + } + + Write-Host "::endgroup::" + } - colcon test ` - ${{ inputs.colcon_test_args_default }} ` - ${{ inputs.colcon_test_args }} ` - --ctest-args ${{ inputs.ctest_args_default }} ${{ inputs.ctest_args }} + Exit $exitCode - "::endgroup::" + - name: Upload test report in JUnit format + uses: eProsima/eProsima-CI/external/upload-artifact@main + if: success() || failure() + with: + name: ${{ inputs.test_report_artifact || format('test_report_{0}_{1}', github.job, join(matrix.*, '_')) }} + path: "${{ inputs.workspace }}\\test_results"