add inferred proxy distributed tracing and error case tests for AWS API Gateway

manifests/cpp.yml

@@ -143,8 +143,7 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: irrelevant (library does not implement OpenTelemetry)
   otel/:

manifests/dotnet.yml

@@ -437,8 +437,7 @@ tests/:
       Test_DsmSQS: v2.48.0
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   otel/:

manifests/golang.yml

@@ -566,8 +566,7 @@ tests/:
         "*": irrelevant
         net-http: missing_feature (Endpoint not implemented)
         net-http-orchestrion: missing_feature (Endpoint not implemented)
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   otel/:

manifests/java.yml

@@ -1650,8 +1650,7 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         "*": irrelevant
         spring-boot: bug (AIDM-325)
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_mongo.py:
       Test_Mongo: bug (APMAPI-729)
     test_otel_drop_in.py:

manifests/nodejs.yml

@@ -870,6 +870,14 @@ tests/:
         '*': irrelevant
         express4: *ref_5_26_0
         express5: *ref_5_26_0
+      Test_AWS_API_Gateway_Inferred_Span_Creation_With_Distributed_Context:
+        '*': irrelevant
+        express4: *ref_5_26_0
+        express5: *ref_5_26_0
+      Test_AWS_API_Gateway_Inferred_Span_Creation_With_Error:
+        '*': irrelevant
+        express4: *ref_5_26_0
+        express5: *ref_5_26_0
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   otel/:

manifests/php.yml

@@ -411,8 +411,7 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   otel/:

manifests/python.yml

@@ -797,7 +797,15 @@ tests/:
         '*': irrelevant
         flask-poc: v2.8.0
     test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+      Test_AWS_API_Gateway_Inferred_Span_Creation:
+        '*': irrelevant
+        flask-poc: v3.1.0
+      Test_AWS_API_Gateway_Inferred_Span_Creation_With_Distributed_Context:
+        '*': irrelevant
+        flask-poc: v3.1.0
+      Test_AWS_API_Gateway_Inferred_Span_Creation_With_Error:
+        '*': irrelevant
+        flask-poc: v3.1.0
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   otel/:

manifests/ruby.yml

@@ -437,8 +437,7 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         rails70: missing_feature (Endpoint not implemented)
-    test_inferred_proxy.py:
-      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
+    test_inferred_proxy.py: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

tests/integrations/test_inferred_proxy.py

@@ -5,13 +5,18 @@
 from utils.tools import logger
 
 
+DISTRIBUTED_TRACE_ID = 1
+DISTRIBUTED_PARENT_ID = 2
+DISTRIBUTED_SAMPLING_PRIORITY = 2
+
+
 @features.aws_api_gateway_inferred_span_creation
 @scenarios.integrations
 class Test_AWS_API_Gateway_Inferred_Span_Creation:
-    """Verify DSM context is extracted using dd-pathway-ctx-base64"""
+    """Verify AWS API Gateway inferred spans are created when a web server receives specific headers."""
 
-    start_time = round(time.time() * 1e3)
-    start_time_ns = start_time * 1e6
+    start_time = round(time.time() * 1000)
+    start_time_ns = start_time * 1000000
 
     def setup_api_gateway_inferred_span_creation(self):
         headers = {
@@ -28,14 +33,84 @@ def setup_api_gateway_inferred_span_creation(self):
     def test_api_gateway_inferred_span_creation(self):
         assert self.r.text == "ok"
 
-        span = get_span(interfaces.library)
+        span = get_span(interfaces.library, "GET /api/data")
+
+        assert span is not None, "API Gateway inferred span should have been created but was not found!"
+
+        assert_api_gateway_span(self, span, path="/api/data", status_code="200")
+
+
+@features.aws_api_gateway_inferred_span_creation
+@scenarios.integrations
+class Test_AWS_API_Gateway_Inferred_Span_Creation_With_Distributed_Context:
+    """Verify AWS API Gateway inferred spans are created when a web server receives specific headers and
+    distributed context.
+    """
+
+    start_time = round(time.time() * 1000)
+    start_time_ns = start_time * 1000000
+
+    def setup_api_gateway_inferred_span_creation_with_distributed_context(self):
+        headers = {
+            "x-dd-proxy-request-time-ms": str(self.start_time),  # in ms
+            "x-dd-proxy-path": "/api/data/distributed",
+            "x-dd-proxy-httpmethod": "GET",
+            "x-dd-proxy-domain-name": "system-tests-api-gateway.com",
+            "x-dd-proxy-stage": "staging",
+            "x-dd-proxy": "aws-apigateway",
+            "x-datadog-trace-id": str(DISTRIBUTED_TRACE_ID),
+            "x-datadog-parent-id": str(DISTRIBUTED_PARENT_ID),
+            "x-datadog-origin": "rum",
+            "x-datadog-sampling-priority": str(DISTRIBUTED_SAMPLING_PRIORITY),
+        }
+
+        self.r = weblog.get("/inferred-proxy/span-creation?status_code=200", headers=headers, timeout=60)
+
+    def test_api_gateway_inferred_span_creation_with_distributed_context(self):
+        assert self.r.text == "ok"
+
+        span = get_span(interfaces.library, "GET /api/data/distributed")
+
+        assert span is not None, "API Gateway inferred span should have been created but was not found!"
+
+        assert_api_gateway_span(self, span, path="/api/data/distributed", status_code="200", is_distributed=True)
+
+
+@features.aws_api_gateway_inferred_span_creation
+@scenarios.integrations
+class Test_AWS_API_Gateway_Inferred_Span_Creation_With_Error:
+    """Verify AWS API Gateway inferred spans are created when a web server receives specific headers and
+    an error.
+    """
+
+    start_time = round(time.time() * 1000)
+    start_time_ns = start_time * 1000000
+
+    def setup_api_gateway_inferred_span_creation_error(self):
+        headers = {
+            "x-dd-proxy-request-time-ms": str(self.start_time),  # in ms
+            "x-dd-proxy-path": "/api/data/error",
+            "x-dd-proxy-httpmethod": "GET",
+            "x-dd-proxy-domain-name": "system-tests-api-gateway.com",
+            "x-dd-proxy-stage": "staging",
+            "x-dd-proxy": "aws-apigateway",
+        }
+
+        self.r = weblog.get("/inferred-proxy/span-creation?status_code=500", headers=headers, timeout=60)
+
+    def test_api_gateway_inferred_span_creation_error(self):
+        assert self.r.text == "ok"
+
+        span = get_span(interfaces.library, "GET /api/data/error")
 
         assert span is not None, "API Gateway inferred span should have been created but was not found!"
 
-        assert_api_gateway_span(self, span)
+        assert_api_gateway_span(
+            self, span, path="/api/data/error", status_code="500", is_distributed=False, is_error=True
+        )
 
 
-def get_span(interface):
+def get_span(interface, resource):
     logger.debug(f"Trying to find API Gateway span for interface: {interface}")
 
     for data, trace in interface.get_traces():
@@ -46,14 +121,17 @@ def get_span(interface):
             if span["name"] != "aws.apigateway":
                 continue
 
+            if span["resource"] != resource:
+                continue
+
             logger.debug(f"Span found in {data['log_filename']}:\n{json.dumps(span, indent=2)}")
             return span
 
     logger.debug("No span found")
     return None
 
 
-def assert_api_gateway_span(testCase, span):
+def assert_api_gateway_span(testCase, span, path, status_code, is_distributed=False, is_error=False):
     assert span["name"] == "aws.apigateway", "Inferred AWS API Gateway span name should be 'aws.apigateway'"
 
     # Assertions to check if the span data contains the required keys and values.
@@ -62,26 +140,48 @@ def assert_api_gateway_span(testCase, span):
         "component" in span["meta"]
     ), "Inferred AWS API Gateway span meta should contain 'component' equal to 'aws-apigateway'"
     assert span["meta"]["component"] == "aws-apigateway", "Expected component to be 'aws-apigateway'"
-    assert "service" in span["meta"], "Inferred AWS API Gateway span meta should contain 'service'"
 
-    assert (
-        span["meta"]["service"] == "system-tests-api-gateway.com"
-    ), "Inferred AWS API Gateway span expected service should equal 'system-tests-api-gateway.com'"
+    if span["meta"]["language"] == "javascript":
+        assert "service" in span["meta"], "Inferred AWS API Gateway span meta should contain 'service'"
+        assert (
+            span["meta"]["service"] == "system-tests-api-gateway.com"
+        ), "Inferred AWS API Gateway span expected service should equal 'system-tests-api-gateway.com'"
+    else:
+        assert "service" in span, "Inferred AWS API Gateway span should contain 'service'"
+        assert (
+            span["service"] == "system-tests-api-gateway.com"
+        ), "Inferred AWS API Gateway span expected service should equal 'system-tests-api-gateway.com'"
     assert "span.kind" in span["meta"], "Inferred AWS API Gateway span meta should contain 'span.kind'"
     assert (
         span["meta"]["span.kind"] == "internal"
     ), "Inferred AWS API Gateway span meta span.kind should equal 'internal'"
+    assert "stage" in span["meta"], "Inferred AWS API Gateway span meta should contain 'stage'"
+    assert span["meta"]["stage"] == "staging", "Inferred AWS API Gateway span meta expected stage to be 'staging'"
+    assert "start" in span, "Inferred AWS API Gateway span should have 'startTime'"
+
+    # assert on HTTP tags
     assert "http.method" in span["meta"], "Inferred AWS API Gateway span meta should contain 'http.method'"
     assert span["meta"]["http.method"] == "GET", "Inferred AWS API Gateway span meta expected HTTP method to be 'GET'"
     assert "http.url" in span["meta"], "Inferred AWS API Gateway span eta should contain 'http.url'"
     assert (
-        span["meta"]["http.url"] == "system-tests-api-gateway.com/api/data"
-    ), "Inferred AWS API Gateway span meta expected HTTP URL to be 'system-tests-api-gateway.com/api/data'"
-    assert "stage" in span["meta"], "Inferred AWS API Gateway span meta should contain 'stage'"
-    assert span["meta"]["stage"] == "staging", "Inferred AWS API Gateway span meta expected stage to be 'staging'"
-    assert "start" in span, "Inferred AWS API Gateway span should have 'startTime'"
+        span["meta"]["http.url"] == "system-tests-api-gateway.com" + path
+    ), f"Inferred AWS API Gateway span meta expected HTTP URL to be 'system-tests-api-gateway.com{path}'"
+    assert "http.status_code" in span["meta"], "Inferred AWS API Gateway span eta should contain 'http.status_code'"
+    assert (
+        span["meta"]["http.status_code"] == status_code
+    ), f"Inferred AWS API Gateway span meta expected HTTP Status Code of '{status_code}'"
 
     if not interfaces.library.replay:
+        # round the start time since we get some inconsistent errors due to how the agent rounds start times.
         assert (
-            span["start"] == testCase.start_time_ns
+            round(span["start"], -6) == testCase.start_time_ns
         ), f"Inferred AWS API Gateway span startTime should equal expected '{testCase.start_time_ns!s}''"
+
+    if is_distributed:
+        assert span["trace_id"] == DISTRIBUTED_TRACE_ID
+        assert span["parent_id"] == DISTRIBUTED_PARENT_ID
+        assert span["metrics"]["_sampling_priority_v1"] == DISTRIBUTED_SAMPLING_PRIORITY
+
+    if is_error:
+        assert span["error"] == 1
+        assert span["meta"]["http.status_code"] == "500"

utils/build/docker/python/flask/app.py

@@ -1608,3 +1608,14 @@ def otel_drop_in_default_propagator_inject():
     opentelemetry.propagate.inject(result, opentelemetry.context.get_current())
 
     return jsonify(result)
+
+
+@app.route("/inferred-proxy/span-creation", methods=["GET"])
+def inferred_proxy_span_creation():
+    headers = flask_request.args.get("headers", {})
+    status = int(flask_request.args.get("status_code", "200"))
+
+    logging.info("Received an API Gateway request")
+    logging.info("Request headers: " + str(headers))
+
+    return Response("ok", status=status)