Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove expected traces count param from CI Vis snapshot tests #8311

Merged
merged 1 commit into from
Jan 30, 2025
Merged

Remove expected traces count param from CI Vis snapshot tests #8311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions ...src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import datadog.trace.civisibility.test.ExecutionStrategy
import datadog.trace.civisibility.utils.ConcurrentHashMapContextStore
import datadog.trace.civisibility.writer.ddintake.CiTestCovMapperV2
import datadog.trace.civisibility.writer.ddintake.CiTestCycleMapperV1
import datadog.trace.common.writer.ListWriter
import datadog.trace.common.writer.RemoteMapper
import datadog.trace.core.DDSpan
import org.msgpack.jackson.dataformat.MessagePackFactory
Expand All @@ -52,6 +53,8 @@ import java.nio.ByteBuffer
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.TimeUnit
import java.util.function.Predicate

abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {

Expand Down Expand Up @@ -216,6 +219,35 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
"RANDOM"
}

final ListWriter.Filter spanFilter = new ListWriter.Filter() {
private final Object lock = new Object()
private Collection<DDSpan> spans = new ArrayList<>()

@Override
boolean accept(List<DDSpan> trace) {
synchronized (lock) {
spans.addAll(trace)
lock.notifyAll()
}
return true
}

boolean waitForSpan(Predicate<DDSpan> predicate, long timeoutMillis) {
long deadline = System.currentTimeMillis() + timeoutMillis
synchronized (lock) {
while (!spans.stream().anyMatch(predicate)) {
def timeLeft = deadline - System.currentTimeMillis()
if (timeLeft > 0) {
lock.wait(timeLeft)
} else {
return false
}
}
return true
}
}
}

@Override
void setup() {
skippableTests.clear()
Expand All @@ -226,6 +258,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
flakyRetryEnabled = false
earlyFlakinessDetectionEnabled = false
impactedTestsDetectionEnabled = false

TEST_WRITER.setFilter(spanFilter)
}

def givenSkippableTests(List<TestIdentifier> tests) {
Expand Down Expand Up @@ -280,8 +314,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
Files.deleteIfExists(agentKeyFile)
}

def assertSpansData(String testcaseName, int expectedTracesCount, Map<String, String> replacements = [:]) {
TEST_WRITER.waitForTraces(expectedTracesCount)
def assertSpansData(String testcaseName, Map<String, String> replacements = [:]) {
Predicate<DDSpan> sessionSpan = span -> span.spanType == "test_session_end"
spanFilter.waitForSpan(sessionSpan, TimeUnit.SECONDS.toMillis(20))

def traces = TEST_WRITER.toList()

def events = getEventsAsJson(traces)
Expand Down
26 changes: 15 additions & 11 deletions ...isibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.Option
import com.jayway.jsonpath.ReadContext
import com.jayway.jsonpath.WriteContext
import freemarker.core.Environment
import freemarker.core.InvalidReferenceException
import freemarker.template.Template
import freemarker.template.TemplateException
import freemarker.template.TemplateExceptionHandler
import org.skyscreamer.jsonassert.JSONAssert
import org.skyscreamer.jsonassert.JSONCompareMode
Expand Down Expand Up @@ -114,11 +116,22 @@ abstract class CiVisibilityTestUtils {
}
}

static final TemplateExceptionHandler SUPPRESS_EXCEPTION_HANDLER = new TemplateExceptionHandler() {
@Override
void handleTemplateException(TemplateException e, Environment environment, Writer writer) throws TemplateException {
if (e instanceof InvalidReferenceException) {
writer.write('"<VALUE_MISSING>"')
} else {
throw e
}
}
}

static final freemarker.template.Configuration FREEMARKER = new freemarker.template.Configuration(freemarker.template.Configuration.VERSION_2_3_30) { {
setClassLoaderForTemplateLoading(CiVisibilityTestUtils.classLoader, "")
setDefaultEncoding("UTF-8")
setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER)
setLogTemplateExceptions(true)
setTemplateExceptionHandler(SUPPRESS_EXCEPTION_HANDLER)
setLogTemplateExceptions(false)
setWrapUncheckedExceptions(true)
setFallbackOnNullLoopVariable(false)
setNumberFormat("0.######")
Expand All @@ -131,15 +144,6 @@ abstract class CiVisibilityTestUtils {
StringWriter coveragesOut = new StringWriter()
coveragesTemplate.process(replacements, coveragesOut)
return coveragesOut.toString()
} catch (InvalidReferenceException e) {
def missingExpression = e.getBlamedExpressionString()
if (missingExpression != null) {
replacements.put(missingExpression, "<VALUE_MISSING>") // to simplify debugging failures
return getFreemarkerTemplate(templatePath, replacements, replacementsSource)

} else {
throw new RuntimeException("Could not get Freemarker template " + templatePath + "; replacements map: " + replacements + "; replacements source: " + replacementsSource, e)
}

} catch (Exception e) {
throw new RuntimeException("Could not get Freemarker template " + templatePath + "; replacements map: " + replacements + "; replacements source: " + replacementsSource, e)
Expand Down
48 changes: 24 additions & 24 deletions ...ava-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,37 +17,37 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
def "test #testcaseName"() {
runFeatures(features)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | features | expectedTracesCount
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | 5
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 2
testcaseName | features
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
"test-multiple-features-${version()}" | [
"org/example/cucumber/calculator/basic_arithmetic.feature",
"org/example/cucumber/calculator/basic_arithmetic_failed.feature"
] | 3
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"] | 2
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"] | 2
]
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"]
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
}

def "test ITR #testcaseName"() {
givenSkippableTests(skippedTests)

runFeatures(features)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | features | expectedTracesCount | skippedTests
"test-itr-skipping" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2 | [
testcaseName | features | skippedTests
"test-itr-skipping" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null)
]
"test-itr-unskippable" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"] | 2 | [
"test-itr-unskippable" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_unskippable.feature:Basic Arithmetic", "Addition", null)
]
"test-itr-unskippable-suite" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"] | 2 | [
"test-itr-unskippable-suite" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature:Basic Arithmetic", "Addition", null)
]
}
Expand All @@ -58,15 +58,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest {

runFeatures(features)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | features | expectedTracesCount | retriedTests
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 2 | []
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 6 | [
testcaseName | features | retriedTests
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | 5 | [
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null)
]
}
Expand All @@ -77,16 +77,16 @@ class CucumberTest extends CiVisibilityInstrumentationTest {

runFeatures(features)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | features | expectedTracesCount | knownTestsList
"test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2 | [
testcaseName | features | knownTestsList
"test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null)
]
"test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 4 | []
"test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | 9 | []
"test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | 3 | []
"test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | []
"test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | []
"test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | []
}

private String version() {
Expand Down
22 changes: 11 additions & 11 deletions dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,19 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
def "test #testcaseName"() {
runTests(tests)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | tests | expectedTracesCount
"test-succeed-before-after" | [TestSucceedBeforeAfter] | 3
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass] | 3
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam] | 2
"test-failed-before-class" | [TestFailedBeforeClass] | 1
"test-failed-after-class" | [TestFailedAfterClass] | 3
"test-failed-before" | [TestFailedBefore] | 3
"test-failed-after" | [TestFailedAfter] | 3
"test-failed-before-param" | [TestFailedBeforeParam] | 2
"test-failed-after-param" | [TestFailedAfterParam] | 2
testcaseName | tests
"test-succeed-before-after" | [TestSucceedBeforeAfter]
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass]
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam]
"test-failed-before-class" | [TestFailedBeforeClass]
"test-failed-after-class" | [TestFailedAfterClass]
"test-failed-before" | [TestFailedBefore]
"test-failed-after" | [TestFailedAfter]
"test-failed-before-param" | [TestFailedBeforeParam]
"test-failed-after-param" | [TestFailedAfterParam]
}

private void runTests(Collection<Class<?>> tests) {
Expand Down
46 changes: 23 additions & 23 deletions dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
def "test #testcaseName"() {
runTests(tests)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | tests | expectedTracesCount
"test-succeed" | [TestSucceedMUnit] | 2
"test-skipped" | [TestSkippedMUnit] | 2
"test-skipped-suite" | [TestSkippedSuiteMUnit] | 3
"test-failed-assumption-${version()}" | [TestFailedAssumptionMUnit] | 2
testcaseName | tests
"test-succeed" | [TestSucceedMUnit]
"test-skipped" | [TestSkippedMUnit]
"test-skipped-suite" | [TestSkippedSuiteMUnit]
"test-failed-assumption-${version()}" | [TestFailedAssumptionMUnit]
}

def "test flaky retries #testcaseName"() {
Expand All @@ -38,13 +38,13 @@ class MUnitTest extends CiVisibilityInstrumentationTest {

runTests(tests)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | tests | expectedTracesCount | retriedTests
"test-failed" | [TestFailedMUnit] | 2 | []
"test-retry-failed" | [TestFailedMUnit] | 6 | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | 4 | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
testcaseName | tests | retriedTests
"test-failed" | [TestFailedMUnit] | []
"test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
}

def "test early flakiness detection #testcaseName"() {
Expand All @@ -53,13 +53,13 @@ class MUnitTest extends CiVisibilityInstrumentationTest {

runTests(tests)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | tests | expectedTracesCount | knownTestsList
"test-efd-known-test" | [TestSucceedMUnit] | 2 | [new TestIdentifier("org.example.TestSucceedMUnit", "Calculator.add", null)]
"test-efd-new-test" | [TestSucceedMUnit] | 4 | []
"test-efd-new-slow-test" | [TestSucceedMUnitSlow] | 3 | [] // is executed only twice
testcaseName | tests | knownTestsList
"test-efd-known-test" | [TestSucceedMUnit] | [new TestIdentifier("org.example.TestSucceedMUnit", "Calculator.add", null)]
"test-efd-new-test" | [TestSucceedMUnit] | []
"test-efd-new-slow-test" | [TestSucceedMUnitSlow] | [] // is executed only twice
}

def "test impacted tests detection #testcaseName"() {
Expand All @@ -68,15 +68,15 @@ class MUnitTest extends CiVisibilityInstrumentationTest {

runTests(tests)

assertSpansData(testcaseName, expectedTracesCount)
assertSpansData(testcaseName)

where:
testcaseName | tests | expectedTracesCount | prDiff
"test-succeed" | [TestSucceedMUnit] | 2 | LineDiff.EMPTY
"test-succeed" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet())
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
"test-succeed" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
testcaseName | tests | prDiff
"test-succeed" | [TestSucceedMUnit] | LineDiff.EMPTY
"test-succeed" | [TestSucceedMUnit] | new FileDiff(new HashSet())
"test-succeed-impacted" | [TestSucceedMUnit] | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
"test-succeed" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
"test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
}

private void runTests(Collection<Class<?>> tests) {
Expand Down
Loading
Loading