Skip to content

Commit

Permalink
Do strict JSON comparison when running CI Vis instrumentation tests l…
Browse files Browse the repository at this point in the history
…ocally
  • Loading branch information
nikita-tkachenko-datadog committed Feb 3, 2025
1 parent c80ed84 commit 9e2a709
Show file tree
Hide file tree
Showing 31 changed files with 450 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.Option
import com.jayway.jsonpath.ReadContext
import com.jayway.jsonpath.WriteContext
import datadog.trace.api.Config
import datadog.trace.civisibility.ci.CIProviderInfoFactory
import datadog.trace.civisibility.ci.GitLabInfo
import datadog.trace.civisibility.ci.GithubActionsInfo
import datadog.trace.civisibility.ci.env.CiEnvironment
import datadog.trace.civisibility.ci.env.CiEnvironmentImpl
import freemarker.core.Environment
import freemarker.core.InvalidReferenceException
import freemarker.template.Template
Expand Down Expand Up @@ -87,18 +93,23 @@ abstract class CiVisibilityTestUtils {
replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"")
}

def environment = System.getenv()
def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null
def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE

def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events)
def actualEvents = JSON_MAPPER.writeValueAsString(events)

try {
JSONAssert.assertEquals(expectedEvents, actualEvents, JSONCompareMode.LENIENT)
JSONAssert.assertEquals(expectedEvents, actualEvents, comparisonMode)
} catch (AssertionError e) {
throw new org.opentest4j.AssertionFailedError("Events mismatch", expectedEvents, actualEvents, e)
}

def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages)
def actualCoverages = JSON_MAPPER.writeValueAsString(coverages)
try {
JSONAssert.assertEquals(expectedCoverages, actualCoverages, JSONCompareMode.LENIENT)
JSONAssert.assertEquals(expectedCoverages, actualCoverages, comparisonMode)
} catch (AssertionError e) {
throw new org.opentest4j.AssertionFailedError("Coverages mismatch", expectedCoverages, actualCoverages, e)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {

where:
testcaseName | success | features | retriedTests
"test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-no-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
Expand Down Expand Up @@ -148,13 +148,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
assertSpansData(testcaseName)

where:
testcaseName | features | quarantined | known
testcaseName | features | quarantined | known
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
] | []
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[ {
"files" : [ {
"filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature"
} ],
"span_id" : ${content_span_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id}
} ]
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
[ {
"content" : {
"duration" : ${content_duration},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"library_version" : ${content_meta_library_version},
"span.kind" : "test_suite_end",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.status" : "fail",
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
},
"name" : "junit.test_suite",
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id}
},
"type" : "test_suite_end",
"version" : 1
}, {
"content" : {
"duration" : ${content_duration_2},
"error" : 1,
"meta" : {
"_dd.profiling.ctx" : "test",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"error.message" : ${content_meta_error_message},
"error.stack" : ${content_meta_error_stack},
"error.type" : "java.lang.AssertionError",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
"span.kind" : "test",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.name" : "Addition",
"test.status" : "fail",
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"test.traits" : "{\"category\":[\"foo\"]}",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
"_dd.profiling.enabled" : 0,
"_dd.trace_span_attribute_schema" : 0,
"process_id" : ${content_metrics_process_id}
},
"name" : "junit.test",
"parent_id" : ${content_parent_id},
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"span_id" : ${content_span_id},
"start" : ${content_start_2},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id},
"trace_id" : ${content_trace_id}
},
"type" : "test",
"version" : 2
}, {
"content" : {
"duration" : ${content_duration_3},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
"_dd.profiling.ctx" : "test",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
"span.kind" : "test_session_end",
"test.command" : "cucumber-junit-5",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.status" : "fail",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
"_dd.profiling.enabled" : 0,
"_dd.trace_span_attribute_schema" : 0,
"process_id" : ${content_metrics_process_id}
},
"name" : "junit.test_session",
"resource" : "cucumber-junit-5",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start_3},
"test_session_id" : ${content_test_session_id}
},
"type" : "test_session_end",
"version" : 1
}, {
"content" : {
"duration" : ${content_duration_4},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"library_version" : ${content_meta_library_version},
"span.kind" : "test_module_end",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.status" : "fail",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
},
"name" : "junit.test_module",
"resource" : "cucumber-junit-5",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start_4},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id}
},
"type" : "test_module_end",
"version" : 1
} ]
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "gradle",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "gradle",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "gradle",
Expand Down Expand Up @@ -55,6 +56,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
"_dd.test.is_user_provided_service" : "true",
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "gradle",
"env" : "integration-test",
Expand Down Expand Up @@ -103,6 +105,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down Expand Up @@ -131,6 +134,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_4},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down Expand Up @@ -159,6 +163,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_5},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down Expand Up @@ -187,6 +192,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_6},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand Down Expand Up @@ -234,6 +240,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_7},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand Down Expand Up @@ -288,6 +295,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_8},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand Down Expand Up @@ -344,6 +352,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_9},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand Down Expand Up @@ -400,6 +409,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_10},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand Down Expand Up @@ -456,6 +466,7 @@
"error" : 1,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_11},
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit",
Expand All @@ -474,6 +485,7 @@
"span.kind" : "test",
"test.framework" : "junit4",
"test.framework_version" : "4.10",
"test.has_failed_all_retries" : "true",
"test.is_retry" : "true",
"test.module" : ":test",
"test.name" : "test_failed",
Expand Down Expand Up @@ -512,6 +524,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_12},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down Expand Up @@ -540,6 +553,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_13},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down Expand Up @@ -568,6 +582,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_14},
"_dd.test.is_user_provided_service" : "true",
"env" : "integration-test",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
Expand Down
Loading

0 comments on commit 9e2a709

Please sign in to comment.