Skip to content

Commit 5d18c24

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Do strict JSON comparison when running CI Vis instrumentation tests locally (#8327)
1 parent 38b8211 commit 5d18c24

File tree

37 files changed

+795
-121
lines changed

37 files changed

+795
-121
lines changed

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

+7-7
Original file line numberDiff line numberDiff line change
@@ -341,13 +341,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
341341
] + replacements
342342

343343
// uncomment to generate expected data templates
344-
// def clazz = this.getClass()
345-
// def resourceName = clazz.name.replace('.', '/') + ".class"
346-
// def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
347-
// def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
348-
// def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
349-
// CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
350-
// return [:]
344+
// def clazz = this.getClass()
345+
// def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
346+
// def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
347+
// def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
348+
// def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
349+
// CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
350+
// return [:]
351351

352352
return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
353353
}

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy

+13-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ import com.jayway.jsonpath.JsonPath
77
import com.jayway.jsonpath.Option
88
import com.jayway.jsonpath.ReadContext
99
import com.jayway.jsonpath.WriteContext
10+
import datadog.trace.api.Config
11+
import datadog.trace.civisibility.ci.CIProviderInfoFactory
12+
import datadog.trace.civisibility.ci.GitLabInfo
13+
import datadog.trace.civisibility.ci.GithubActionsInfo
14+
import datadog.trace.civisibility.ci.env.CiEnvironment
15+
import datadog.trace.civisibility.ci.env.CiEnvironmentImpl
1016
import freemarker.core.Environment
1117
import freemarker.core.InvalidReferenceException
1218
import freemarker.template.Template
@@ -87,18 +93,23 @@ abstract class CiVisibilityTestUtils {
8793
replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"")
8894
}
8995

96+
def environment = System.getenv()
97+
def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null
98+
def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE
99+
90100
def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events)
91101
def actualEvents = JSON_MAPPER.writeValueAsString(events)
102+
92103
try {
93-
JSONAssert.assertEquals(expectedEvents, actualEvents, JSONCompareMode.LENIENT)
104+
JSONAssert.assertEquals(expectedEvents, actualEvents, comparisonMode)
94105
} catch (AssertionError e) {
95106
throw new org.opentest4j.AssertionFailedError("Events mismatch", expectedEvents, actualEvents, e)
96107
}
97108

98109
def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages)
99110
def actualCoverages = JSON_MAPPER.writeValueAsString(coverages)
100111
try {
101-
JSONAssert.assertEquals(expectedCoverages, actualCoverages, JSONCompareMode.LENIENT)
112+
JSONAssert.assertEquals(expectedCoverages, actualCoverages, comparisonMode)
102113
} catch (AssertionError e) {
103114
throw new org.opentest4j.AssertionFailedError("Coverages mismatch", expectedCoverages, actualCoverages, e)
104115
}

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy

+3-3
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
6969

7070
where:
7171
testcaseName | success | features | retriedTests
72-
"test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
72+
"test-no-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
7373
"test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
7474
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
7575
]
@@ -148,13 +148,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
148148
assertSpansData(testcaseName)
149149

150150
where:
151-
testcaseName | features | quarantined | known
151+
testcaseName | features | quarantined | known
152152
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
153153
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
154154
] | [
155155
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
156156
]
157-
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
157+
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
158158
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
159159
] | []
160160
}

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/java/org/example/cucumber/calculator/CalculatorSteps.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public void a_calculator_I_just_turned_on() {
2323

2424
@Given("a flaky calculator I just turned on")
2525
public void a_flaky_calculator_I_just_turned_on() {
26-
calc = ++flakyCounter >= 3 ? new Calculator() : null;
26+
calc = new Calculator(++flakyCounter < 3);
2727
}
2828

2929
@When("I add {int} and {int}")
@@ -94,6 +94,15 @@ public void setOperation(String operation) {
9494
static class Calculator {
9595
private static final List<String> OPS = asList("-", "+", "*", "/");
9696
private final Deque<Number> stack = new LinkedList<>();
97+
private final boolean broken;
98+
99+
public Calculator() {
100+
this(false);
101+
}
102+
103+
public Calculator(boolean broken) {
104+
this.broken = broken;
105+
}
97106

98107
public void push(Object arg) {
99108
if (OPS.contains(arg)) {
@@ -116,6 +125,9 @@ public void push(Object arg) {
116125
}
117126

118127
public Number value() {
128+
if (broken) {
129+
throw new RuntimeException("This test is flaky");
130+
}
119131
return stack.getLast();
120132
}
121133
}

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-then-succeed/events.ftl

+4-2
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@
4040
"component" : "junit",
4141
"dummy_ci_tag" : "dummy_ci_tag_value",
4242
"env" : "none",
43+
"error.message" : ${content_meta_error_message},
4344
"error.stack" : ${content_meta_error_stack},
44-
"error.type" : "java.lang.NullPointerException",
45+
"error.type" : "java.lang.RuntimeException",
4546
"language" : "jvm",
4647
"library_version" : ${content_meta_library_version},
4748
"runtime-id" : ${content_meta_runtime_id},
@@ -85,8 +86,9 @@
8586
"component" : "junit",
8687
"dummy_ci_tag" : "dummy_ci_tag_value",
8788
"env" : "none",
89+
"error.message" : ${content_meta_error_message},
8890
"error.stack" : ${content_meta_error_stack_2},
89-
"error.type" : "java.lang.NullPointerException",
91+
"error.type" : "java.lang.RuntimeException",
9092
"language" : "jvm",
9193
"library_version" : ${content_meta_library_version},
9294
"runtime-id" : ${content_meta_runtime_id},

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/coverages.ftl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[ {
2+
"files" : [ {
3+
"filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature"
4+
} ],
5+
"span_id" : ${content_span_id},
6+
"test_session_id" : ${content_test_session_id},
7+
"test_suite_id" : ${content_test_suite_id}
8+
} ]

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/events.ftl

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"component" : "junit",
8+
"dummy_ci_tag" : "dummy_ci_tag_value",
9+
"env" : "none",
10+
"library_version" : ${content_meta_library_version},
11+
"span.kind" : "test_suite_end",
12+
"test.framework" : "cucumber",
13+
"test.framework_version" : ${content_meta_test_framework_version},
14+
"test.module" : "cucumber-junit-5",
15+
"test.status" : "fail",
16+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
17+
"test.type" : "test",
18+
"test_session.name" : "session-name"
19+
},
20+
"metrics" : {
21+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
22+
},
23+
"name" : "junit.test_suite",
24+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
25+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
26+
"start" : ${content_start},
27+
"test_module_id" : ${content_test_module_id},
28+
"test_session_id" : ${content_test_session_id},
29+
"test_suite_id" : ${content_test_suite_id}
30+
},
31+
"type" : "test_suite_end",
32+
"version" : 1
33+
}, {
34+
"content" : {
35+
"duration" : ${content_duration_2},
36+
"error" : 1,
37+
"meta" : {
38+
"_dd.profiling.ctx" : "test",
39+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
40+
"component" : "junit",
41+
"dummy_ci_tag" : "dummy_ci_tag_value",
42+
"env" : "none",
43+
"error.message" : ${content_meta_error_message},
44+
"error.stack" : ${content_meta_error_stack},
45+
"error.type" : "java.lang.AssertionError",
46+
"language" : "jvm",
47+
"library_version" : ${content_meta_library_version},
48+
"runtime-id" : ${content_meta_runtime_id},
49+
"span.kind" : "test",
50+
"test.framework" : "cucumber",
51+
"test.framework_version" : ${content_meta_test_framework_version},
52+
"test.module" : "cucumber-junit-5",
53+
"test.name" : "Addition",
54+
"test.status" : "fail",
55+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
56+
"test.traits" : "{\"category\":[\"foo\"]}",
57+
"test.type" : "test",
58+
"test_session.name" : "session-name"
59+
},
60+
"metrics" : {
61+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
62+
"_dd.profiling.enabled" : 0,
63+
"_dd.trace_span_attribute_schema" : 0,
64+
"process_id" : ${content_metrics_process_id}
65+
},
66+
"name" : "junit.test",
67+
"parent_id" : ${content_parent_id},
68+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
69+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
70+
"span_id" : ${content_span_id},
71+
"start" : ${content_start_2},
72+
"test_module_id" : ${content_test_module_id},
73+
"test_session_id" : ${content_test_session_id},
74+
"test_suite_id" : ${content_test_suite_id},
75+
"trace_id" : ${content_trace_id}
76+
},
77+
"type" : "test",
78+
"version" : 2
79+
}, {
80+
"content" : {
81+
"duration" : ${content_duration_3},
82+
"error" : 0,
83+
"meta" : {
84+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
85+
"_dd.profiling.ctx" : "test",
86+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
87+
"component" : "junit",
88+
"dummy_ci_tag" : "dummy_ci_tag_value",
89+
"env" : "none",
90+
"language" : "jvm",
91+
"library_version" : ${content_meta_library_version},
92+
"runtime-id" : ${content_meta_runtime_id},
93+
"span.kind" : "test_session_end",
94+
"test.command" : "cucumber-junit-5",
95+
"test.framework" : "cucumber",
96+
"test.framework_version" : ${content_meta_test_framework_version},
97+
"test.status" : "fail",
98+
"test.type" : "test",
99+
"test_session.name" : "session-name"
100+
},
101+
"metrics" : {
102+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
103+
"_dd.profiling.enabled" : 0,
104+
"_dd.trace_span_attribute_schema" : 0,
105+
"process_id" : ${content_metrics_process_id}
106+
},
107+
"name" : "junit.test_session",
108+
"resource" : "cucumber-junit-5",
109+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
110+
"start" : ${content_start_3},
111+
"test_session_id" : ${content_test_session_id}
112+
},
113+
"type" : "test_session_end",
114+
"version" : 1
115+
}, {
116+
"content" : {
117+
"duration" : ${content_duration_4},
118+
"error" : 0,
119+
"meta" : {
120+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
121+
"component" : "junit",
122+
"dummy_ci_tag" : "dummy_ci_tag_value",
123+
"env" : "none",
124+
"library_version" : ${content_meta_library_version},
125+
"span.kind" : "test_module_end",
126+
"test.framework" : "cucumber",
127+
"test.framework_version" : ${content_meta_test_framework_version},
128+
"test.module" : "cucumber-junit-5",
129+
"test.status" : "fail",
130+
"test.type" : "test",
131+
"test_session.name" : "session-name"
132+
},
133+
"metrics" : {
134+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
135+
},
136+
"name" : "junit.test_module",
137+
"resource" : "cucumber-junit-5",
138+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
139+
"start" : ${content_start_4},
140+
"test_module_id" : ${content_test_module_id},
141+
"test_session_id" : ${content_test_session_id}
142+
},
143+
"type" : "test_module_end",
144+
"version" : 1
145+
} ]

‎dd-java-agent/instrumentation/selenium/src/test/resources/test-succeed-multiple/coverages.ftl

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)