From 3e8d2291105f1cad074168958a99df56ed4b1bd8 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko Date: Thu, 30 Jan 2025 20:50:25 +0100 Subject: [PATCH] Assert test run status in CI Vis instrumentation tests --- .../src/test/groovy/CucumberTest.groovy | 44 ++- .../src/test/groovy/JUnit413Test.groovy | 40 +- .../src/test/groovy/MUnitTest.groovy | 28 +- .../src/test/groovy/JUnit4Test.groovy | 132 +++---- .../src/test/groovy/CucumberTest.groovy | 76 ++-- .../src/test/groovy/JUnit58Test.groovy | 65 +++- .../src/test/groovy/SpockTest.groovy | 82 ++-- ...FailedThenSucceedParameterizedSpock.groovy | 2 +- .../events.ftl | 9 +- .../src/test/groovy/JUnit5Test.groovy | 137 ++++--- .../KarateExecutionInstrumentation.java | 16 +- .../instrumentation/karate/KarateUtils.java | 7 + .../karate/src/test/groovy/KarateTest.groovy | 78 ++-- .../resources/test-retry-failed/events.ftl | 10 +- .../test-retry-parameterized/events.ftl | 10 +- .../src/test/groovy/ScalatestTest.groovy | 39 +- .../instrumentation/testng/build.gradle | 1 + .../instrumentation/testng/TestNGTest.groovy | 128 +++---- .../test-failed-then-succeed-6/coverages.ftl | 1 - .../test-failed-then-succeed-6/events.ftl | 254 ------------ .../test-retry-failed-6/coverages.ftl | 1 - .../resources/test-retry-failed-6/events.ftl | 361 ------------------ 22 files changed, 533 insertions(+), 988 deletions(-) delete mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/coverages.ftl delete mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/events.ftl delete mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/coverages.ftl delete mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/events.ftl diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy index 4728a90ca9c..ab090c83c1a 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy @@ -15,21 +15,21 @@ class CucumberTest extends CiVisibilityInstrumentationTest { def runner = new JUnitCore() def "test #testcaseName"() { - runFeatures(features) + runFeatures(features, success) assertSpansData(testcaseName) where: - testcaseName | features - "test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] - "test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] - "test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] - "test-multiple-features-${version()}" | [ + testcaseName | success | features + "test-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic.feature"] + "test-scenario-outline-${version()}" | true | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] + "test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] + "test-multiple-features-${version()}" | false | [ "org/example/cucumber/calculator/basic_arithmetic.feature", "org/example/cucumber/calculator/basic_arithmetic_failed.feature" ] - "test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"] - "test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"] + "test-name-with-brackets" | true | ["org/example/cucumber/calculator/name_with_brackets.feature"] + "test-empty-name-${version()}" | true | ["org/example/cucumber/calculator/empty_scenario_name.feature"] } def "test ITR #testcaseName"() { @@ -56,17 +56,17 @@ class CucumberTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runFeatures(features) + runFeatures(features, success) assertSpansData(testcaseName) where: - testcaseName | features | retriedTests - "test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] - "test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + testcaseName | success | features | retriedTests + "test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] + "test-retry-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) ] - "test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [ + "test-retry-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null) ] } @@ -93,7 +93,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { return CucumberTracingListener.FRAMEWORK_VERSION < "7" ? CucumberTracingListener.FRAMEWORK_VERSION : "latest" } - private void runFeatures(List classpathFeatures) { + private void runFeatures(List classpathFeatures, boolean expectSuccess = true) { System.setProperty(Constants.GLUE_PROPERTY_NAME, "org.example.cucumber.calculator") System.setProperty(Constants.FILTER_TAGS_PROPERTY_NAME, "not @Disabled") System.setProperty(Constants.FEATURES_PROPERTY_NAME, classpathFeatures.stream() @@ -101,8 +101,20 @@ class CucumberTest extends CiVisibilityInstrumentationTest { collect(Collectors.joining(","))) TestEventsHandlerHolder.start() - runner.run(TestSucceedCucumber) - TestEventsHandlerHolder.stop() + try { + def result = runner.run(TestSucceedCucumber) + if (expectSuccess) { + if (result.getFailureCount() > 0) { + throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures()) + } + } else { + if (result.getFailureCount() == 0) { + throw new AssertionError("Expected a failed execution, got no failures") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override diff --git a/dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy b/dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy index 9837a252c4f..9e2209f1b6c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy @@ -19,32 +19,40 @@ class JUnit413Test extends CiVisibilityInstrumentationTest { def runner = new JUnitCore() def "test #testcaseName"() { - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-succeed-before-after" | [TestSucceedBeforeAfter] - "test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass] - "test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam] - "test-failed-before-class" | [TestFailedBeforeClass] - "test-failed-after-class" | [TestFailedAfterClass] - "test-failed-before" | [TestFailedBefore] - "test-failed-after" | [TestFailedAfter] - "test-failed-before-param" | [TestFailedBeforeParam] - "test-failed-after-param" | [TestFailedAfterParam] + testcaseName | success | tests + "test-succeed-before-after" | true | [TestSucceedBeforeAfter] + "test-succeed-before-class-after-class" | true | [TestSucceedBeforeClassAfterClass] + "test-succeed-before-param-after-param" | true | [TestSucceedBeforeParamAfterParam] + "test-failed-before-class" | false | [TestFailedBeforeClass] + "test-failed-after-class" | false | [TestFailedAfterClass] + "test-failed-before" | false | [TestFailedBefore] + "test-failed-after" | false | [TestFailedAfter] + "test-failed-before-param" | false | [TestFailedBeforeParam] + "test-failed-after-param" | false | [TestFailedAfterParam] } - private void runTests(Collection> tests) { + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { Class[] array = tests.toArray(new Class[0]) - runner.run(array) - } catch (Throwable ignored) { - // Ignored + def result = runner.run(array) + if (expectSuccess) { + if (result.getFailureCount() > 0) { + throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures()) + } + } else { + if (result.getFailureCount() == 0) { + throw new AssertionError("Expected a failed execution, got no failures") + } + } + } finally { + TestEventsHandlerHolder.stop() } - TestEventsHandlerHolder.stop() } @Override diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index 8064aec3e90..7f26ff3b58c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -36,15 +36,15 @@ class MUnitTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailedMUnit] | [] - "test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] - "test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)] + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailedMUnit] | [] + "test-retry-failed" | false | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)] } def "test early flakiness detection #testcaseName"() { @@ -79,15 +79,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } - private void runTests(Collection> tests) { + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { Class[] array = tests.toArray(new Class[0]) - runner.run(array) - } catch (Throwable ignored) { - // Ignored + def result = runner.run(array) + if (expectSuccess) { + if (result.getFailureCount() > 0) { + throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures()) + } + } else { + if (result.getFailureCount() == 0) { + throw new AssertionError("Expected a failed execution, got no failures") + } + } + } finally { + TestEventsHandlerHolder.stop() } - TestEventsHandlerHolder.stop() } String version() { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index ab1f9844f0e..3b3c228476b 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -5,33 +5,7 @@ import datadog.trace.civisibility.diff.FileDiff import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder import junit.runner.Version -import org.example.TestAssumption -import org.example.TestAssumptionAndSucceed -import org.example.TestError -import org.example.TestFailed -import org.example.TestFailedAndSucceed -import org.example.TestFailedParameterized -import org.example.TestFailedSuiteSetUpAssumption -import org.example.TestFailedSuiteSetup -import org.example.TestFailedSuiteTearDown -import org.example.TestFailedThenSucceed -import org.example.TestInheritance -import org.example.TestParameterized -import org.example.TestParameterizedJUnitParams -import org.example.TestSkipped -import org.example.TestSkippedClass -import org.example.TestSucceed -import org.example.TestSucceedAndSkipped -import org.example.TestSucceedExpectedException -import org.example.TestSucceedKotlin -import org.example.TestSucceedLegacy -import org.example.TestSucceedParameterizedKotlin -import org.example.TestSucceedSlow -import org.example.TestSucceedSuite -import org.example.TestSucceedUnskippable -import org.example.TestSucceedUnskippableSuite -import org.example.TestSucceedVerySlow -import org.example.TestSucceedWithCategories +import org.example.* import org.junit.runner.JUnitCore @DisableTestTrace(reason = "avoid self-tracing") @@ -40,34 +14,34 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { def runner = new JUnitCore() def "test #testcaseName"() { - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-succeed" | [TestSucceed] - "test-inheritance" | [TestInheritance] - "test-failed" | [TestFailed] - "test-error" | [TestError] - "test-skipped" | [TestSkipped] - "test-class-skipped" | [TestSkippedClass] - "test-success-and-skipped" | [TestSucceedAndSkipped] - "test-success-and-failure" | [TestFailedAndSucceed] - "test-suite-teardown-failure" | [TestFailedSuiteTearDown] - "test-suite-setup-failure" | [TestFailedSuiteSetup] - "test-assumption-failure" | [TestAssumption] - "test-categories-are-included-in-spans" | [TestSucceedWithCategories] - "test-assumption-failure-during-suite-setup" | [TestFailedSuiteSetUpAssumption] - "test-assumption-failure-in-a-multi-test-case-suite" | [TestAssumptionAndSucceed] - "test-multiple-successful-suites" | [TestSucceed, TestSucceedAndSkipped] - "test-successful-suite-and-failing-suite" | [TestSucceed, TestFailedAndSucceed] - "test-parameterized" | [TestParameterized] - "test-suite-runner" | [TestSucceedSuite] - "test-legacy" | [TestSucceedLegacy] - "test-parameterized-junit-params" | [TestParameterizedJUnitParams] - "test-succeed-kotlin" | [TestSucceedKotlin] - "test-succeed-parameterized-kotlin" | [TestSucceedParameterizedKotlin] + testcaseName | success | tests + "test-succeed" | true | [TestSucceed] + "test-inheritance" | true | [TestInheritance] + "test-failed" | false | [TestFailed] + "test-error" | false | [TestError] + "test-skipped" | true | [TestSkipped] + "test-class-skipped" | true | [TestSkippedClass] + "test-success-and-skipped" | true | [TestSucceedAndSkipped] + "test-success-and-failure" | false | [TestFailedAndSucceed] + "test-suite-teardown-failure" | false | [TestFailedSuiteTearDown] + "test-suite-setup-failure" | false | [TestFailedSuiteSetup] + "test-assumption-failure" | true | [TestAssumption] + "test-categories-are-included-in-spans" | true | [TestSucceedWithCategories] + "test-assumption-failure-during-suite-setup" | true | [TestFailedSuiteSetUpAssumption] + "test-assumption-failure-in-a-multi-test-case-suite" | true | [TestAssumptionAndSucceed] + "test-multiple-successful-suites" | true | [TestSucceed, TestSucceedAndSkipped] + "test-successful-suite-and-failing-suite" | false | [TestSucceed, TestFailedAndSucceed] + "test-parameterized" | true | [TestParameterized] + "test-suite-runner" | true | [TestSucceedSuite] + "test-legacy" | true | [TestSucceedLegacy] + "test-parameterized-junit-params" | true | [TestParameterizedJUnitParams] + "test-succeed-kotlin" | true | [TestSucceedKotlin] + "test-succeed-parameterized-kotlin" | true | [TestSucceedParameterizedKotlin] } def "test ITR #testcaseName"() { @@ -95,44 +69,44 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailed] | [] - "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-failed-then-succeed" | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] - "test-assumption-is-not-retried" | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] - "test-skipped-is-not-retried" | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] - "test-retry-parameterized" | [TestFailedParameterized] | [ + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailed] | [] + "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] + "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] + "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] + "test-retry-parameterized" | false | [TestFailedParameterized] | [ new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", /* backend cannot provide parameters for flaky parameterized tests yet */ null) ] - "test-expected-exception-is-not-retried" | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] + "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] } def "test early flakiness detection #testcaseName"() { givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] - "test-efd-new-test" | [TestSucceed] | [] - "test-efd-new-parameterized-test" | [TestParameterized] | [] - "test-efd-known-tests-and-new-test" | [TestFailedAndSucceed] | [ + testcaseName | success | tests | knownTestsList + "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] + "test-efd-new-test" | true | [TestSucceed] | [] + "test-efd-new-parameterized-test" | true | [TestParameterized] | [] + "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) ] - "test-efd-new-slow-test" | [TestSucceedSlow] | [] // is executed only twice - "test-efd-new-very-slow-test" | [TestSucceedVerySlow] | [] // is executed only once - "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | [] + "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice + "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once + "test-efd-faulty-session-threshold" | false | [TestFailedAndSucceed] | [] } def "test impacted tests detection #testcaseName"() { @@ -152,15 +126,23 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } - private void runTests(Collection> tests) { + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { Class[] array = tests.toArray(new Class[0]) - runner.run(array) - } catch (Throwable ignored) { - // Ignored + def result = runner.run(array) + if (expectSuccess) { + if (result.getFailureCount() > 0) { + throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures()) + } + } else { + if (result.getFailureCount() == 0) { + throw new AssertionError("Expected a failed execution, got no failures") + } + } + } finally { + TestEventsHandlerHolder.stop() } - TestEventsHandlerHolder.stop() } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index 71ab1b5f3c5..756322286da 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -5,12 +5,17 @@ import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder import io.cucumber.core.api.TypeRegistry import io.cucumber.core.options.Constants import org.junit.platform.engine.DiscoverySelector +import org.junit.platform.engine.TestExecutionResult import org.junit.platform.launcher.EngineFilter import org.junit.platform.launcher.Launcher import org.junit.platform.launcher.LauncherDiscoveryRequest +import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClasspathResource @DisableTestTrace(reason = "avoid self-tracing") @@ -22,16 +27,16 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | features | parallel - "test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | false - "test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | false - "test-skipped" | ["org/example/cucumber/calculator/basic_arithmetic_skipped.feature"] | false - "test-skipped-feature" | ["org/example/cucumber/calculator/basic_arithmetic_skipped_feature.feature"] | false - "test-skipped-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_skipped.feature"] | false + testcaseName | features | parallel + "test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | false + "test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | false + "test-skipped" | ["org/example/cucumber/calculator/basic_arithmetic_skipped.feature"] | false + "test-skipped-feature" | ["org/example/cucumber/calculator/basic_arithmetic_skipped_feature.feature"] | false + "test-skipped-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_skipped.feature"] | false "test-parallel" | [ "org/example/cucumber/calculator/basic_arithmetic.feature", "org/example/cucumber/calculator/basic_arithmetic_skipped.feature" - ] | true + ] | true } def "test ITR #testcaseName"() { @@ -58,20 +63,20 @@ class CucumberTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runFeatures(features, false) + runFeatures(features, false, success) assertSpansData(testcaseName) where: - testcaseName | features | retriedTests - "test-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] - "test-retry-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + testcaseName | success | features | retriedTests + "test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] + "test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) ] - "test-failed-then-succeed" | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"] | [ + "test-failed-then-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature:Basic Arithmetic", "Addition", null) ] - "test-failed-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | [ + "test-failed-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature:Basic Arithmetic With Examples", "Many additions.Single digits.${parameterizedTestNameSuffix()}", null) ] } @@ -104,7 +109,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { return version != null ? "latest" : "5.4.0" // older releases do not have package version populated } - protected void runFeatures(List classpathFeatures, boolean parallel) { + protected void runFeatures(List classpathFeatures, boolean parallel, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[classpathFeatures.size()] @@ -112,18 +117,33 @@ class CucumberTest extends CiVisibilityInstrumentationTest { selectors[i] = selectClasspathResource(classpathFeatures[i]) } - LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() - .filters(EngineFilter.includeEngines("cucumber")) - .configurationParameter(Constants.GLUE_PROPERTY_NAME, "org.example.cucumber.calculator") - .configurationParameter(Constants.FILTER_TAGS_PROPERTY_NAME, "not @Disabled") - .configurationParameter(io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, "$parallel") - .selectors(selectors) - .build() + LauncherDiscoveryRequest launcherReq = LauncherDiscoveryRequestBuilder.request() + .filters(EngineFilter.includeEngines("cucumber")) + .configurationParameter(Constants.GLUE_PROPERTY_NAME, "org.example.cucumber.calculator") + .configurationParameter(Constants.FILTER_TAGS_PROPERTY_NAME, "not @Disabled") + .configurationParameter(io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, "$parallel") + .selectors(selectors) + .build() Launcher launcher = LauncherFactory.create() - launcher.execute(request) - - TestEventsHandlerHolder.stop() + def listener = new TestResultListener() + launcher.registerTestExecutionListeners(listener) + try { + launcher.execute(launcherReq) + + def failedTests = listener.testsByStatus[TestExecutionResult.Status.FAILED] + if (expectSuccess) { + if (failedTests != null && !failedTests.isEmpty()) { + throw new AssertionError("Expected successful execution, the following tests were reported as failed: " + failedTests) + } + } else { + if (failedTests == null || failedTests.isEmpty()) { + throw new AssertionError("Expected a failed execution, got no failed tests") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override @@ -136,4 +156,12 @@ class CucumberTest extends CiVisibilityInstrumentationTest { def version = TypeRegistry.package.getImplementationVersion() return version != null ? version : "5.4.0" // older releases do not have package version populated } + + private static final class TestResultListener implements TestExecutionListener { + private final Map> testsByStatus = new ConcurrentHashMap<>() + + void executionFinished(org.junit.platform.launcher.TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + testsByStatus.computeIfAbsent(testExecutionResult.status, k -> new CopyOnWriteArrayList<>()).add(testIdentifier) + } + } } diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy index c082c9446b8..7079be44751 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy @@ -8,10 +8,15 @@ import org.junit.jupiter.api.MethodOrderer import org.junit.jupiter.engine.Constants import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.core.LauncherConfig import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") @@ -24,18 +29,18 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { } def "test #testcaseName"() { - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-before-each-after-each" | [TestSucceedBeforeEachAfterEach] - "test-before-all-after-all" | [TestSucceedBeforeAllAfterAll] - "test-failed-before-all" | [TestFailedBeforeAll] - "test-failed-after-all" | [TestFailedAfterAll] - "test-failed-before-each" | [TestFailedBeforeEach] - "test-failed-after-each" | [TestFailedAfterEach] + testcaseName | success | tests + "test-before-each-after-each" | true | [TestSucceedBeforeEachAfterEach] + "test-before-all-after-all" | true | [TestSucceedBeforeAllAfterAll] + "test-failed-before-all" | false | [TestFailedBeforeAll] + "test-failed-after-all" | false | [TestFailedAfterAll] + "test-failed-before-each" | false | [TestFailedBeforeEach] + "test-failed-after-each" | false | [TestFailedAfterEach] } def "test known tests ordering #testcaseName"() { @@ -83,7 +88,7 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { ] } - private static void runTests(List> tests) { + private static void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[tests.size()] @@ -92,24 +97,36 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { } def launcherReq = LauncherDiscoveryRequestBuilder.request() - .configurationParameter(Constants.DEFAULT_TEST_CLASS_ORDER_PROPERTY_NAME, ClassOrderer.ClassName.name) - .configurationParameter(Constants.DEFAULT_TEST_METHOD_ORDER_PROPERTY_NAME, MethodOrderer.MethodName.name) - .selectors(selectors) - .build() + .configurationParameter(Constants.DEFAULT_TEST_CLASS_ORDER_PROPERTY_NAME, ClassOrderer.ClassName.name) + .configurationParameter(Constants.DEFAULT_TEST_METHOD_ORDER_PROPERTY_NAME, MethodOrderer.MethodName.name) + .selectors(selectors) + .build() def launcherConfig = LauncherConfig - .builder() - .enableTestEngineAutoRegistration(false) - .addTestEngines(new JupiterTestEngine()) - .build() + .builder() + .enableTestEngineAutoRegistration(false) + .addTestEngines(new JupiterTestEngine()) + .build() def launcher = LauncherFactory.create(launcherConfig) + def listener = new TestResultListener() + launcher.registerTestExecutionListeners(listener) try { launcher.execute(launcherReq) - } catch (Throwable ignored) { - } - TestEventsHandlerHolder.stop() + def failedTests = listener.testsByStatus[TestExecutionResult.Status.FAILED] + if (expectSuccess) { + if (failedTests != null && !failedTests.isEmpty()) { + throw new AssertionError("Expected successful execution, the following tests were reported as failed: " + failedTests) + } + } else { + if (failedTests == null || failedTests.isEmpty()) { + throw new AssertionError("Expected a failed execution, got no failed tests") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override @@ -121,4 +138,12 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { String instrumentedLibraryVersion() { return JupiterTestEngine.getPackage().getImplementationVersion() } + + private static final class TestResultListener implements TestExecutionListener { + private final Map> testsByStatus = new ConcurrentHashMap<>() + + void executionFinished(org.junit.platform.launcher.TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + testsByStatus.computeIfAbsent(testExecutionResult.status, k -> new CopyOnWriteArrayList<>()).add(testIdentifier) + } + } } diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index cc6a1ed49da..bc2899c7a06 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -19,12 +19,17 @@ import org.example.TestSucceedSpockUnskippable import org.example.TestSucceedSpockUnskippableSuite import org.example.TestSucceedSpockVerySlow import org.junit.platform.engine.DiscoverySelector +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.core.LauncherConfig import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory import org.spockframework.runtime.SpockEngine import org.spockframework.util.SpockReleaseInfo +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") @@ -75,40 +80,40 @@ class SpockTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailedSpock] | [] - "test-retry-failed" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] - "test-failed-then-succeed" | [TestFailedThenSucceedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedSpock", "test failed then succeed", null)] - "test-retry-parameterized" | [TestFailedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedParameterizedSpock", "test add 4 and 4", null)] - "test-parameterized-failed-then-succeed" | [TestFailedThenSucceedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedParameterizedSpock", "test add 1 and 2", null)] + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailedSpock] | [] + "test-retry-failed" | false | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedSpock", "test failed then succeed", null)] + "test-retry-parameterized" | false | [TestFailedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedParameterizedSpock", "test add 4 and 4", null)] + "test-parameterized-failed-then-succeed" | true | [TestFailedThenSucceedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedThenSucceedParameterizedSpock", "test add 1 and 2", null)] } def "test early flakiness detection #testcaseName"() { givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceedSpock] | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null)] - "test-efd-known-parameterized-test" | [TestParameterizedSpock] | [ + testcaseName | success | tests | knownTestsList + "test-efd-known-test" | true | [TestSucceedSpock] | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null)] + "test-efd-known-parameterized-test" | true | [TestParameterizedSpock] | [ new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null), new TestIdentifier("org.example.TestParameterizedSpock", "test add 4 and 4", null) ] - "test-efd-new-test" | [TestSucceedSpock] | [] - "test-efd-new-parameterized-test" | [TestParameterizedSpock] | [] - "test-efd-known-tests-and-new-test" | [TestParameterizedSpock] | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null)] - "test-efd-new-slow-test" | [TestSucceedSpockSlow] | [] // is executed only twice - "test-efd-new-very-slow-test" | [TestSucceedSpockVerySlow] | [] // is executed only once - "test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | [] + "test-efd-new-test" | true | [TestSucceedSpock] | [] + "test-efd-new-parameterized-test" | true | [TestParameterizedSpock] | [] + "test-efd-known-tests-and-new-test" | true | [TestParameterizedSpock] | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null)] + "test-efd-new-slow-test" | true | [TestSucceedSpockSlow] | [] // is executed only twice + "test-efd-new-very-slow-test" | true | [TestSucceedSpockVerySlow] | [] // is executed only once + "test-efd-faulty-session-threshold" | false | [TestSucceedAndFailedSpock] | [] } def "test impacted tests detection #testcaseName"() { @@ -128,7 +133,7 @@ class SpockTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceedSpock] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } - private static void runTests(List> classes) { + private static void runTests(List> classes, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[classes.size()] @@ -137,19 +142,34 @@ class SpockTest extends CiVisibilityInstrumentationTest { } def launcherReq = LauncherDiscoveryRequestBuilder.request() - .selectors(selectors) - .build() + .selectors(selectors) + .build() def launcherConfig = LauncherConfig - .builder() - .enableTestEngineAutoRegistration(false) - .addTestEngines(new SpockEngine()) - .build() + .builder() + .enableTestEngineAutoRegistration(false) + .addTestEngines(new SpockEngine()) + .build() def launcher = LauncherFactory.create(launcherConfig) - launcher.execute(launcherReq) - - TestEventsHandlerHolder.stop() + def listener = new TestResultListener() + launcher.registerTestExecutionListeners(listener) + try { + launcher.execute(launcherReq) + + def failedTests = listener.testsByStatus[TestExecutionResult.Status.FAILED] + if (expectSuccess) { + if (failedTests != null && !failedTests.isEmpty()) { + throw new AssertionError("Expected successful execution, the following tests were reported as failed: " + failedTests) + } + } else { + if (failedTests == null || failedTests.isEmpty()) { + throw new AssertionError("Expected a failed execution, got no failed tests") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override @@ -161,4 +181,12 @@ class SpockTest extends CiVisibilityInstrumentationTest { String instrumentedLibraryVersion() { return SpockReleaseInfo.version } + + private static final class TestResultListener implements TestExecutionListener { + private final Map> testsByStatus = new ConcurrentHashMap<>() + + void executionFinished(org.junit.platform.launcher.TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + testsByStatus.computeIfAbsent(testExecutionResult.status, k -> new CopyOnWriteArrayList<>()).add(testIdentifier) + } + } } diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedParameterizedSpock.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedParameterizedSpock.groovy index 535643ad1a7..481708c2de0 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedParameterizedSpock.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedParameterizedSpock.groovy @@ -8,7 +8,7 @@ class TestFailedThenSucceedParameterizedSpock extends Specification { def "test add #a and #b"() { expect: - c == 3 && ++testExecutionCount > 2 + ++testExecutionCount > 2 where: a | b | c diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-parameterized-failed-then-succeed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-parameterized-failed-then-succeed/events.ftl index ae1d49379e1..2918bcb57f2 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-parameterized-failed-then-succeed/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-parameterized-failed-then-succeed/events.ftl @@ -194,16 +194,13 @@ }, { "content" : { "duration" : ${content_duration_5}, - "error" : 1, + "error" : 0, "meta" : { "_dd.profiling.ctx" : "test", "_dd.tracer_host" : ${content_meta__dd_tracer_host}, "component" : "junit", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message_3}, - "error.stack" : ${content_meta_error_stack_3}, - "error.type" : "org.spockframework.runtime.ConditionNotSatisfiedError", "language" : "jvm", "library_version" : ${content_meta_library_version}, "runtime-id" : ${content_meta_runtime_id}, @@ -217,7 +214,7 @@ "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", "test.source.file" : "dummy_source_path", "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", - "test.status" : "fail", + "test.status" : "pass", "test.suite" : "org.example.TestFailedThenSucceedParameterizedSpock", "test.type" : "test", "test_session.name" : "session-name" @@ -309,4 +306,4 @@ }, "type" : "test_module_end", "version" : 1 -} ] \ No newline at end of file +} ] diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index 8e40e4eece4..b51a0e11658 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -36,45 +36,50 @@ import org.example.TestSuiteSetUpAssumption import org.example.TestTemplate import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.core.LauncherConfig import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class JUnit5Test extends CiVisibilityInstrumentationTest { def "test #testcaseName"() { - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-succeed" | [TestSucceed] - "test-inheritance" | [TestInheritance] - "test-parameterized" | [TestParameterized] - "test-repeated" | [TestRepeated] - "test-template" | [TestTemplate] - "test-factory" | [TestFactory] - "test-failed" | [TestFailed] - "test-error" | [TestError] - "test-skipped" | [TestSkipped] - "test-skipped-class" | [TestSkippedClass] - "test-assumption-failed" | [TestAssumption] - "test-assumption-failed-legacy" | [TestAssumptionLegacy] - "test-succeed-and-skipped" | [TestSucceedAndSkipped] - "test-succeed-and-failed" | [TestFailedAndSucceed] - "test-suite-teardown-failed" | [TestFailedSuiteTearDown] - "test-suite-setup-failed" | [TestFailedSuiteSetup] - "test-categories" | [TestSucceedWithCategories] - "test-suite-setup-assumption-failed" | [TestSuiteSetUpAssumption] - "test-suite-setup-assumption-failed-multi-test-case" | [TestAssumptionAndSucceed] - "test-succeed-multiple-suites" | [TestSucceed, TestSucceedAndSkipped] - "test-succeed-and-failed-multiple-suites" | [TestSucceed, TestFailedAndSucceed] - "test-succeed-nested-suites" | [TestSucceedNested] - "test-skipped-nested-suites" | [TestSkippedNested] + testcaseName | success | tests + "test-succeed" | true | [TestSucceed] + "test-inheritance" | true | [TestInheritance] + "test-parameterized" | true | [TestParameterized] + "test-repeated" | true | [TestRepeated] + "test-template" | true | [TestTemplate] + "test-factory" | true | [TestFactory] + "test-failed" | false | [TestFailed] + "test-error" | false | [TestError] + "test-skipped" | true | [TestSkipped] + "test-skipped-class" | true | [TestSkippedClass] + "test-assumption-failed" | true | [TestAssumption] + "test-assumption-failed-legacy" | true | [TestAssumptionLegacy] + "test-succeed-and-skipped" | true | [TestSucceedAndSkipped] + "test-succeed-and-failed" | false | [TestFailedAndSucceed] + "test-suite-teardown-failed" | false | [TestFailedSuiteTearDown] + "test-suite-setup-failed" | false | [TestFailedSuiteSetup] + "test-categories" | true | [TestSucceedWithCategories] + "test-suite-setup-assumption-failed" | true | [TestSuiteSetUpAssumption] + "test-suite-setup-assumption-failed-multi-test-case" | true | [TestAssumptionAndSucceed] + "test-succeed-multiple-suites" | true | [TestSucceed, TestSucceedAndSkipped] + "test-succeed-and-failed-multiple-suites" | false | [TestSucceed, TestFailedAndSucceed] + "test-succeed-nested-suites" | true | [TestSucceedNested] + "test-skipped-nested-suites" | true | [TestSkippedNested] } def "test ITR #testcaseName"() { @@ -102,44 +107,44 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailed] | [] - "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-failed-then-succeed" | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] - "test-retry-template" | [TestFailedTemplate] | [new TestIdentifier("org.example.TestFailedTemplate", "test_template", null)] - "test-retry-factory" | [TestFailedFactory] | [new TestIdentifier("org.example.TestFailedFactory", "test_factory", null)] - "test-assumption-is-not-retried" | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] - "test-skipped-is-not-retried" | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] - "test-retry-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] - "test-expected-exception-is-not-retried" | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailed] | [] + "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed_then_succeed", null)] + "test-retry-template" | false | [TestFailedTemplate] | [new TestIdentifier("org.example.TestFailedTemplate", "test_template", null)] + "test-retry-factory" | false | [TestFailedFactory] | [new TestIdentifier("org.example.TestFailedFactory", "test_factory", null)] + "test-assumption-is-not-retried" | true | [TestAssumption] | [new TestIdentifier("org.example.TestAssumption", "test_fail_assumption", null)] + "test-skipped-is-not-retried" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] + "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] + "test-expected-exception-is-not-retried" | true | [TestSucceedExpectedException] | [new TestIdentifier("org.example.TestSucceedExpectedException", "test_succeed", null)] } def "test early flakiness detection #testcaseName"() { givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "test_parameterized", null)] - "test-efd-new-test" | [TestSucceed] | [] - "test-efd-new-parameterized-test" | [TestParameterized] | [] - "test-efd-known-tests-and-new-test" | [TestFailedAndSucceed] | [ + testcaseName | success | tests | knownTestsList + "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "test_parameterized", null)] + "test-efd-new-test" | true | [TestSucceed] | [] + "test-efd-new-parameterized-test" | true | [TestParameterized] | [] + "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) ] - "test-efd-new-slow-test" | [TestSucceedSlow] | [] // is executed only twice - "test-efd-new-very-slow-test" | [TestSucceedVerySlow] | [] // is executed only once - "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | [] + "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice + "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once + "test-efd-faulty-session-threshold" | false | [TestFailedAndSucceed] | [] } def "test impacted tests detection #testcaseName"() { @@ -159,7 +164,7 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } - private static void runTests(List> tests) { + protected void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[tests.size()] @@ -168,22 +173,34 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } def launcherReq = LauncherDiscoveryRequestBuilder.request() - .selectors(selectors) - .build() + .selectors(selectors) + .build() def launcherConfig = LauncherConfig - .builder() - .enableTestEngineAutoRegistration(false) - .addTestEngines(new JupiterTestEngine()) - .build() + .builder() + .enableTestEngineAutoRegistration(false) + .addTestEngines(new JupiterTestEngine()) + .build() def launcher = LauncherFactory.create(launcherConfig) + def listener = new TestResultListener() + launcher.registerTestExecutionListeners(listener) try { launcher.execute(launcherReq) - } catch (Throwable ignored) { - } - TestEventsHandlerHolder.stop() + def failedTests = listener.testsByStatus[TestExecutionResult.Status.FAILED] + if (expectSuccess) { + if (failedTests != null && !failedTests.isEmpty()) { + throw new AssertionError("Expected successful execution, the following tests were reported as failed: " + failedTests) + } + } else { + if (failedTests == null || failedTests.isEmpty()) { + throw new AssertionError("Expected a failed execution, got no failed tests") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override @@ -195,4 +212,12 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { String instrumentedLibraryVersion() { return JupiterTestEngine.getPackage().getImplementationVersion() } + + private static final class TestResultListener implements TestExecutionListener { + private final Map> testsByStatus = new ConcurrentHashMap<>() + + void executionFinished(org.junit.platform.launcher.TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + testsByStatus.computeIfAbsent(testExecutionResult.status, k -> new CopyOnWriteArrayList<>()).add(testIdentifier) + } + } } diff --git a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index 4cd782b2c96..b7386e7a673 100644 --- a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -9,12 +9,14 @@ import com.intuit.karate.RuntimeHook; import com.intuit.karate.core.Result; import com.intuit.karate.core.Scenario; +import com.intuit.karate.core.ScenarioResult; import com.intuit.karate.core.ScenarioRuntime; import com.intuit.karate.core.StepResult; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.Config; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; +import datadog.trace.bootstrap.CallDepthThreadLocalMap; import datadog.trace.bootstrap.InstrumentationContext; import java.util.Collections; import java.util.Map; @@ -83,6 +85,11 @@ public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) { @Advice.OnMethodExit public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { + if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) { + // nested call + return; + } + Scenario scenario = scenarioRuntime.scenario; ExecutionContext context = InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario); @@ -90,16 +97,23 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { return; } + ScenarioResult finalResult = scenarioRuntime.result; + TestExecutionPolicy executionPolicy = context.getExecutionPolicy(); long duration = System.currentTimeMillis() - context.getStartTimestamp(); - if (executionPolicy.retry(!context.getAndResetFailed(), duration)) { + while (executionPolicy.retry(!context.getAndResetFailed(), duration)) { ScenarioRuntime retry = new ScenarioRuntime(scenarioRuntime.featureRuntime, scenarioRuntime.scenario); retry.magicVariables.put( KarateUtils.RETRY_MAGIC_VARIABLE, executionPolicy.currentExecutionRetryReason()); retry.run(); retry.featureRuntime.result.addResult(retry.result); + finalResult = retry.result; } + + KarateUtils.setResult(scenarioRuntime, finalResult); + + CallDepthThreadLocalMap.reset(ScenarioRuntime.class); } // Karate 1.0.0 and above diff --git a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java index 74cc1bc4ab3..b93213b0cb1 100644 --- a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java +++ b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java @@ -6,6 +6,7 @@ import com.intuit.karate.core.FeatureRuntime; import com.intuit.karate.core.Result; import com.intuit.karate.core.Scenario; +import com.intuit.karate.core.ScenarioResult; import com.intuit.karate.core.ScenarioRuntime; import com.intuit.karate.core.Tag; import datadog.trace.api.civisibility.config.TestIdentifier; @@ -41,6 +42,8 @@ private KarateUtils() {} // static method to create aborted result has a different signature starting with Karate 1.4.1 private static final MethodHandle ABORTED_RESULT_STARTTIME_DURATION_NANOS = METHOD_HANDLES.method(Result.class, "aborted", long.class, long.class); + private static final MethodHandle SCENARIO_RUNTIME_RESULT_SETTER = + METHOD_HANDLES.privateFieldSetter(ScenarioRuntime.class, "result"); public static Feature getFeature(FeatureRuntime featureRuntime) { if (FEATURE_RUNTIME_FEATURE_CALL_GETTER != null) { @@ -124,4 +127,8 @@ public static boolean isBeforeHookExecuted(FeatureRuntime featureRuntime) { public static void resetBeforeHook(FeatureRuntime featureRuntime) { METHOD_HANDLES.invoke(FEATURE_RUNTIME_BEFORE_HOOK_DONE_SETTER, featureRuntime, false); } + + public static void setResult(ScenarioRuntime runtime, ScenarioResult result) { + METHOD_HANDLES.invoke(SCENARIO_RUNTIME_RESULT_SETTER, runtime, result); + } } diff --git a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy index 5ed144f1c96..b818fbbf985 100644 --- a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy @@ -19,10 +19,15 @@ import org.example.TestWithSetupKarate import org.junit.jupiter.api.Assumptions import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.core.LauncherConfig import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") @@ -31,19 +36,19 @@ class KarateTest extends CiVisibilityInstrumentationTest { def "test #testcaseName"() { Assumptions.assumeTrue(assumption) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | assumption - "test-succeed" | [TestSucceedKarate] | true - "test-succeed-parallel" | [TestSucceedParallelKarate] | true - "test-with-setup" | [TestWithSetupKarate] | isSetupTagSupported(FileUtils.KARATE_VERSION) - "test-parameterized" | [TestParameterizedKarate] | true - "test-failed" | [TestFailedKarate] | true - "test-skipped-feature" | [TestSkippedFeatureKarate] | true - "test-built-in-retry" | [TestFailedBuiltInRetryKarate] | true + testcaseName | success | tests | assumption + "test-succeed" | true | [TestSucceedKarate] | true + "test-succeed-parallel" | true | [TestSucceedParallelKarate] | true + "test-with-setup" | true | [TestWithSetupKarate] | isSetupTagSupported(FileUtils.KARATE_VERSION) + "test-parameterized" | true | [TestParameterizedKarate] | true + "test-failed" | false | [TestFailedKarate] | true + "test-skipped-feature" | true | [TestSkippedFeatureKarate] | true + "test-built-in-retry" | true | [TestFailedBuiltInRetryKarate] | true } def "test ITR #testcaseName"() { @@ -68,16 +73,16 @@ class KarateTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailedKarate] | [] - "test-retry-failed" | [TestFailedKarate] | [new TestIdentifier("[org/example/test_failed] test failed", "second scenario", null)] - "test-failed-then-succeed" | [TestFailedThenSucceedKarate] | [new TestIdentifier("[org/example/test_failed_then_succeed] test failed", "flaky scenario", null)] - "test-retry-parameterized" | [TestFailedParameterizedKarate] | [ + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailedKarate] | [] + "test-retry-failed" | false | [TestFailedKarate] | [new TestIdentifier("[org/example/test_failed] test failed", "second scenario", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestIdentifier("[org/example/test_failed_then_succeed] test failed", "flaky scenario", null)] + "test-retry-parameterized" | false | [TestFailedParameterizedKarate] | [ new TestIdentifier("[org/example/test_failed_parameterized] test parameterized", "first scenario as an outline", null) ] } @@ -102,7 +107,7 @@ class KarateTest extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] } - private void runTests(List> tests) { + private void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() DiscoverySelector[] selectors = new DiscoverySelector[tests.size()] @@ -111,19 +116,34 @@ class KarateTest extends CiVisibilityInstrumentationTest { } def launcherReq = LauncherDiscoveryRequestBuilder.request() - .selectors(selectors) - .build() + .selectors(selectors) + .build() def launcherConfig = LauncherConfig - .builder() - .enableTestEngineAutoRegistration(false) - .addTestEngines(new JupiterTestEngine()) - .build() + .builder() + .enableTestEngineAutoRegistration(false) + .addTestEngines(new JupiterTestEngine()) + .build() def launcher = LauncherFactory.create(launcherConfig) - launcher.execute(launcherReq) - - TestEventsHandlerHolder.stop() + def listener = new TestResultListener() + launcher.registerTestExecutionListeners(listener) + try { + launcher.execute(launcherReq) + + def failedTests = listener.testsByStatus[TestExecutionResult.Status.FAILED] + if (expectSuccess) { + if (failedTests != null && !failedTests.isEmpty()) { + throw new AssertionError("Expected successful execution, the following tests were reported as failed: " + failedTests) + } + } else { + if (failedTests == null || failedTests.isEmpty()) { + throw new AssertionError("Expected a failed execution, got no failed tests") + } + } + } finally { + TestEventsHandlerHolder.stop() + } } @Override @@ -144,4 +164,12 @@ class KarateTest extends CiVisibilityInstrumentationTest { boolean isSetupTagSupported(String frameworkVersion) { frameworkVersion >= "1.3.0" } + + private static final class TestResultListener implements TestExecutionListener { + private final Map> testsByStatus = new ConcurrentHashMap<>() + + void executionFinished(org.junit.platform.launcher.TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + testsByStatus.computeIfAbsent(testExecutionResult.status, k -> new CopyOnWriteArrayList<>()).add(testIdentifier) + } + } } diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-failed/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-failed/events.ftl index 220f0f7e0ff..121cd6320ea 100644 --- a/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-failed/events.ftl +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-failed/events.ftl @@ -229,7 +229,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_2}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -274,7 +274,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_3}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -321,7 +321,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_4}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -368,7 +368,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_5}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -415,7 +415,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_6}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-parameterized/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-parameterized/events.ftl index a935537b286..c895f305e1b 100644 --- a/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-retry-parameterized/events.ftl @@ -475,7 +475,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_2}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -521,7 +521,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_3}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -569,7 +569,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_4}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -617,7 +617,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_5}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", @@ -665,7 +665,7 @@ "component" : "karate", "dummy_ci_tag" : "dummy_ci_tag_value", "env" : "none", - "error.message" : ${content_meta_error_message}, + "error.message" : ${content_meta_error_message_2}, "error.stack" : ${content_meta_error_stack_6}, "error.type" : "com.intuit.karate.KarateException", "language" : "jvm", diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index 976201b755a..6b21cb3c247 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -21,20 +21,20 @@ import org.scalatest.tools.Runner class ScalatestTest extends CiVisibilityInstrumentationTest { def "test #testcaseName"() { - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-succeed" | [TestSucceed] - "test-succeed-flat-spec" | [TestSucceedFlatSpec] - "test-succeed-parameterized" | [TestSucceedParameterized] - "test-failed" | [TestFailed] - "test-ignored" | [TestIgnored] - "test-canceled" | [TestIgnoredCanceled] - "test-pending" | [TestIgnoredPending] - "test-failed-suite" | [TestFailedSuite] + testcaseName | success | tests + "test-succeed" | true | [TestSucceed] + "test-succeed-flat-spec" | true | [TestSucceedFlatSpec] + "test-succeed-parameterized" | true | [TestSucceedParameterized] + "test-failed" | false | [TestFailed] + "test-ignored" | true | [TestIgnored] + "test-canceled" | true | [TestIgnoredCanceled] + "test-pending" | true | [TestIgnoredPending] + "test-failed-suite" | false | [TestFailedSuite] } def "test ITR #testcaseName"() { @@ -54,18 +54,18 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests) + runTests(tests, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed" | [TestFailed] | [] - "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] - "test-retry-parameterized" | [TestFailedParameterized] | [ + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailed] | [] + "test-retry-failed" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + "test-retry-parameterized" | false | [TestFailedParameterized] | [ new TestIdentifier("org.example.TestFailedParameterized", "addition should correctly add two numbers", null) ] - "test-failed-then-succeed" | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "Example.add adds two numbers", null)] + "test-failed-then-succeed" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "Example.add adds two numbers", null)] } def "test early flakiness detection #testcaseName"() { @@ -111,13 +111,16 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { return ScalatestUtils.scalatestVersion } - void runTests(List> tests) { + void runTests(List> tests, boolean expectSuccess = true) { def runnerArguments = ["-o"] // standard out reporting for (Class test : tests) { runnerArguments += ["-s", test.name] } - Runner.run((String[]) runnerArguments.toArray(new String[0])) + def result = Runner.run((String[]) runnerArguments.toArray(new String[0])) + if (result != expectSuccess) { + throw new AssertionError("Expected $expectSuccess execution status, got $result") + } } } diff --git a/dd-java-agent/instrumentation/testng/build.gradle b/dd-java-agent/instrumentation/testng/build.gradle index be0a5249f6d..dc239f8b9b6 100644 --- a/dd-java-agent/instrumentation/testng/build.gradle +++ b/dd-java-agent/instrumentation/testng/build.gradle @@ -10,6 +10,7 @@ dependencies { testFixturesApi testFixtures(project(':dd-java-agent:agent-ci-visibility')) testFixturesImplementation group: 'org.testng', name: 'testng', version: '6.4' + testFixturesApi group: 'org.apache.maven', name: 'maven-artifact', version: '3.9.9' } // gradle can't downgrade the testng dependencies with `strictly` and IntelliJ IDEA reports diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index b5bd4007da7..f0eb856e518 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -4,29 +4,8 @@ import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.FileDiff import datadog.trace.civisibility.diff.LineDiff -import org.example.TestError -import org.example.TestFailed -import org.example.TestFailedAndSucceed -import org.example.TestFailedParameterized -import org.example.TestFailedSuiteSetup -import org.example.TestFailedSuiteTearDown -import org.example.TestFailedThenSucceed -import org.example.TestFailedWithSuccessPercentage -import org.example.TestInheritance -import org.example.TestParameterized -import org.example.TestParameterizedModifiesParams -import org.example.TestSkipped -import org.example.TestSkippedClass -import org.example.TestSkippedNested -import org.example.TestSucceed -import org.example.TestSucceedAndSkipped -import org.example.TestSucceedDataProvider -import org.example.TestSucceedGroups -import org.example.TestSucceedMultiple -import org.example.TestSucceedNested -import org.example.TestSucceedSlow -import org.example.TestSucceedUnskippable -import org.example.TestSucceedVerySlow +import org.apache.maven.artifact.versioning.ComparableVersion +import org.example.* import org.junit.jupiter.api.Assumptions import org.testng.TestNG import org.testng.xml.SuiteXmlParser @@ -36,32 +15,35 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { static testOutputDir = "build/tmp/test-output" + static ComparableVersion currentTestNGVersion = new ComparableVersion(TracingListener.FRAMEWORK_VERSION) + static ComparableVersion testNGv75 = new ComparableVersion("7.5") + def "test #testcaseName"() { - runTests(tests, null) + runTests(tests, null, success) assertSpansData(testcaseName) where: - testcaseName | tests - "test-succeed" | [TestSucceed] - "test-inheritance" | [TestInheritance] - "test-failed-${version()}" | [TestFailed] - "test-failed-with-success-percentage-${version()}" | [TestFailedWithSuccessPercentage] - "test-error" | [TestError] - "test-skipped" | [TestSkipped] - "test-parameterized" | [TestParameterized] - "test-parameterized-modifies-params" | [TestParameterizedModifiesParams] - "test-success-with-groups" | [TestSucceedGroups] - "test-class-skipped" | [TestSkippedClass] - "test-success-and-skipped" | [TestSucceedAndSkipped] - "test-success-and-failure-${version()}" | [TestFailedAndSucceed] - "test-suite-teardown-failure" | [TestFailedSuiteTearDown] - "test-suite-setup-failure" | [TestFailedSuiteSetup] - "test-multiple-successful-suites" | [TestSucceed, TestSucceedAndSkipped] - "test-successful-suite-and-failed-suite-${version()}" | [TestSucceed, TestFailedAndSucceed] - "test-nested-successful-suites" | [TestSucceedNested, TestSucceedNested.NestedSuite] - "test-nested-skipped-suites-${version()}" | [TestSkippedNested] - "test-factory-data-provider" | [TestSucceedDataProvider] + testcaseName | success | tests + "test-succeed" | true | [TestSucceed] + "test-inheritance" | true | [TestInheritance] + "test-failed-${version()}" | false | [TestFailed] + "test-failed-with-success-percentage-${version()}" | true | [TestFailedWithSuccessPercentage] + "test-error" | false | [TestError] + "test-skipped" | true | [TestSkipped] + "test-parameterized" | true | [TestParameterized] + "test-parameterized-modifies-params" | true | [TestParameterizedModifiesParams] + "test-success-with-groups" | true | [TestSucceedGroups] + "test-class-skipped" | true | [TestSkippedClass] + "test-success-and-skipped" | true | [TestSucceedAndSkipped] + "test-success-and-failure-${version()}" | false | [TestFailedAndSucceed] + "test-suite-teardown-failure" | false | [TestFailedSuiteTearDown] + "test-suite-setup-failure" | false | [TestFailedSuiteSetup] + "test-multiple-successful-suites" | true | [TestSucceed, TestSucceedAndSkipped] + "test-successful-suite-and-failed-suite-${version()}" | false | [TestSucceed, TestFailedAndSucceed] + "test-nested-successful-suites" | true | [TestSucceedNested, TestSucceedNested.NestedSuite] + "test-nested-skipped-suites-${version()}" | true | [TestSkippedNested] + "test-factory-data-provider" | true | [TestSucceedDataProvider] } def "test parallel execution #testcaseName"() { @@ -112,20 +94,22 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + Assumptions.assumeTrue(isExceptionSuppressionSupported()) + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) - runTests(tests, null) + runTests(tests, null, success) assertSpansData(testcaseName) where: - testcaseName | tests | retriedTests - "test-failed-${version()}" | [TestFailed] | [] - "test-skipped" | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] - "test-retry-failed-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] - "test-retry-error" | [TestError] | [new TestIdentifier("org.example.TestError", "test_error", null)] - "test-retry-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "parameterized_test_succeed", null)] - "test-failed-then-succeed-${version()}" | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed", null)] + testcaseName | success | tests | retriedTests + "test-failed-${version()}" | false | [TestFailed] | [] + "test-skipped" | true | [TestSkipped] | [new TestIdentifier("org.example.TestSkipped", "test_skipped", null)] + "test-retry-failed-${version()}" | false | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-retry-error" | false | [TestError] | [new TestIdentifier("org.example.TestError", "test_error", null)] + "test-retry-parameterized" | false | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "parameterized_test_succeed", null)] + "test-failed-then-succeed-${version()}" | true | [TestFailedThenSucceed] | [new TestIdentifier("org.example.TestFailedThenSucceed", "test_failed", null)] } def "test early flakiness detection #testcaseName"() { @@ -134,23 +118,23 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) - runTests(tests) + runTests(tests, null, success) assertSpansData(testcaseName) where: - testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] - "test-efd-known-parameterized-test" | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] - "test-efd-new-test" | [TestSucceed] | [] - "test-efd-new-parameterized-test" | [TestParameterized] | [] - "test-efd-known-tests-and-new-test" | [TestFailedAndSucceed] | [ + testcaseName | success | tests | knownTestsList + "test-efd-known-test" | true | [TestSucceed] | [new TestIdentifier("org.example.TestSucceed", "test_succeed", null)] + "test-efd-known-parameterized-test" | true | [TestParameterized] | [new TestIdentifier("org.example.TestParameterized", "parameterized_test_succeed", null)] + "test-efd-new-test" | true | [TestSucceed] | [] + "test-efd-new-parameterized-test" | true | [TestParameterized] | [] + "test-efd-known-tests-and-new-test" | false | [TestFailedAndSucceed] | [ new TestIdentifier("org.example.TestFailedAndSucceed", "test_failed", null), new TestIdentifier("org.example.TestFailedAndSucceed", "test_succeed", null) ] - "test-efd-new-slow-test" | [TestSucceedSlow] | [] // is executed only twice - "test-efd-new-very-slow-test" | [TestSucceedVerySlow] | [] // is executed only once - "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | [] + "test-efd-new-slow-test" | true | [TestSucceedSlow] | [] // is executed only twice + "test-efd-new-very-slow-test" | true | [TestSucceedVerySlow] | [] // is executed only once + "test-efd-faulty-session-threshold" | false | [TestFailedAndSucceed] | [] } def "test impacted tests detection #testcaseName"() { @@ -171,10 +155,14 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { } private static boolean isEFDSupported() { - TracingListener.FRAMEWORK_VERSION >= "7.5" + currentTestNGVersion >= testNGv75 } - protected void runTests(List testClasses, String parallelMode = null) { + private static boolean isExceptionSuppressionSupported() { + currentTestNGVersion >= testNGv75 + } + + protected void runTests(List testClasses, String parallelMode = null, boolean expectSuccess = true) { TestEventsHandlerHolder.start() def testNG = new TestNG() @@ -187,8 +175,16 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { try { testNG.run() - } catch (Throwable ignored) { - // Ignored + if (expectSuccess && testNG.hasFailure()) { + throw new AssertionError("Expected successful execution, but reported status is failed") + } + if (!expectSuccess && !testNG.hasFailure()) { + throw new AssertionError("Expected failed execution, but reported status is successful") + } + } catch (Throwable t) { + if (expectSuccess) { + throw new AssertionError("Expected successful execution, got error", t) + } } TestEventsHandlerHolder.stop() diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/coverages.ftl deleted file mode 100644 index 8878e547a79..00000000000 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/coverages.ftl +++ /dev/null @@ -1 +0,0 @@ -[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/events.ftl deleted file mode 100644 index de4b40ef19e..00000000000 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-failed-then-succeed-6/events.ftl +++ /dev/null @@ -1,254 +0,0 @@ -[ { - "content" : { - "duration" : ${content_duration}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "library_version" : ${content_meta_library_version}, - "span.kind" : "test_suite_end", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.source.file" : "dummy_source_path", - "test.status" : "fail", - "test.suite" : "org.example.TestFailedThenSucceed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, - "test.source.end" : 19, - "test.source.start" : 11 - }, - "name" : "testng.test_suite", - "resource" : "org.example.TestFailedThenSucceed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} - }, - "type" : "test_suite_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_2}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailedThenSucceed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailedThenSucceed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id}, - "start" : ${content_start_2}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_3}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack_2}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailedThenSucceed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailedThenSucceed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_2}, - "start" : ${content_start_3}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_2} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_4}, - "error" : 0, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "pass", - "test.suite" : "org.example.TestFailedThenSucceed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailedThenSucceed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_3}, - "start" : ${content_start_4}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_3} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_5}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_2}, - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test_session_end", - "test.command" : "testng-6", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.status" : "fail", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id} - }, - "name" : "testng.test_session", - "resource" : "testng-6", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_5}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_session_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_6}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_3}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "library_version" : ${content_meta_library_version}, - "span.kind" : "test_module_end", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.status" : "fail", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} - }, - "name" : "testng.test_module", - "resource" : "testng-6", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_6}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_module_end", - "version" : 1 -} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/coverages.ftl deleted file mode 100644 index 8878e547a79..00000000000 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/coverages.ftl +++ /dev/null @@ -1 +0,0 @@ -[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/events.ftl deleted file mode 100644 index c6d698a14cc..00000000000 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-retry-failed-6/events.ftl +++ /dev/null @@ -1,361 +0,0 @@ -[ { - "content" : { - "duration" : ${content_duration}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "library_version" : ${content_meta_library_version}, - "span.kind" : "test_suite_end", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.source.file" : "dummy_source_path", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, - "test.source.end" : 19, - "test.source.start" : 11 - }, - "name" : "testng.test_suite", - "resource" : "org.example.TestFailed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} - }, - "type" : "test_suite_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_2}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id}, - "start" : ${content_start_2}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_3}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack_2}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_2}, - "start" : ${content_start_3}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_2} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_4}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack_3}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_3}, - "start" : ${content_start_4}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_3} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_5}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack_4}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_4}, - "start" : ${content_start_5}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_4} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_6}, - "error" : 1, - "meta" : { - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "error.message" : ${content_meta_error_message}, - "error.stack" : ${content_meta_error_stack_5}, - "error.type" : "java.lang.AssertionError", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test", - "test.codeowners" : "[\"owner1\",\"owner2\"]", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_retry" : "true", - "test.module" : "testng-6", - "test.name" : "test_failed", - "test.retry_reason" : "atr", - "test.source.file" : "dummy_source_path", - "test.source.method" : "test_failed()V", - "test.status" : "fail", - "test.suite" : "org.example.TestFailed", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.source.end" : 18, - "test.source.start" : 12 - }, - "name" : "testng.test", - "parent_id" : ${content_parent_id}, - "resource" : "org.example.TestFailed.test_failed", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "span_id" : ${content_span_id_5}, - "start" : ${content_start_6}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id_5} - }, - "type" : "test", - "version" : 2 -}, { - "content" : { - "duration" : ${content_duration_7}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_2}, - "_dd.profiling.ctx" : "test", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "runtime-id" : ${content_meta_runtime_id}, - "span.kind" : "test_session_end", - "test.command" : "testng-6", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.status" : "fail", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id} - }, - "name" : "testng.test_session", - "resource" : "testng-6", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_7}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_session_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_8}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_3}, - "component" : "testng", - "dummy_ci_tag" : "dummy_ci_tag_value", - "env" : "none", - "library_version" : ${content_meta_library_version}, - "span.kind" : "test_module_end", - "test.framework" : "testng", - "test.framework_version" : ${content_meta_test_framework_version}, - "test.module" : "testng-6", - "test.status" : "fail", - "test.type" : "test", - "test_session.name" : "session-name" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_8} - }, - "name" : "testng.test_module", - "resource" : "testng-6", - "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", - "start" : ${content_start_8}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_module_end", - "version" : 1 -} ] \ No newline at end of file