Genesis LLO Support in ADOT SDK (#361)

## What does this pull request do?
Adds support to handle LLO from third-party instrumentation SDKs in ADOT
SDK.

The following SDKs are supported:
- Traceloop/Openllmetry
- OpenInference
- OpenLit

Note: OTel dependencies in ADOT SDK have been loosened as a short-term
workaround to support the various conflicting dependency requirements of
third-party instrumentation SDKs.

## Test plan
Built this custom ADOT SDK into various sample apps and exported the
span and logs data to the OTLP X-Ray and Logs endpoint, respectively, to
validate the LLO extraction and transformation to Gen AI Events.

Configurations tested:
- LangChain + Traceloop/Openllmetry
- LangChan + OpenInference
- LangChain + OpenLit
- CrewAI + Traceloop/Openllmetry
- CrewAI + OpenInference
- CrewAI + OpenLit

Environment variable configuration:
```
λ  env OTEL_METRICS_EXPORTER=none \
       OTEL_TRACES_EXPORTER=otlp \
       OTEL_LOGS_EXPORTER=otlp \
       OTEL_PYTHON_DISTRO=aws_distro \
       OTEL_PYTHON_CONFIGURATOR=aws_configurator \
       OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
       OTEL_EXPORTER_OTLP_LOGS_HEADERS="x-aws-log-group=test,x-aws-log-stream=default" \
       OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://xray.us-east-1.amazonaws.com/v1/traces \
       OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.us-east-1.amazonaws.com/v1/logs \
       OTEL_RESOURCE_ATTRIBUTES="service.name=langchain-app" \
       OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED="true" \
       OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="true" \
       OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="http,sqlalchemy,psycopg2,pymysql,sqlite3,aiopg,asyncpg,mysql_connector,botocore,boto3,urllib3,requests,starlette" \
       AGENT_OBSERVABILITY_ENABLED="true" \
       python app.py
```

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: yiyuan-he

aws-opentelemetry-distro/pyproject.toml

@@ -25,61 +25,61 @@ classifiers = [
 ]
 
 dependencies = [
-  "opentelemetry-api == 1.27.0",
-  "opentelemetry-sdk == 1.27.0",
-  "opentelemetry-exporter-otlp-proto-grpc == 1.27.0",
-  "opentelemetry-exporter-otlp-proto-http == 1.27.0",
-  "opentelemetry-propagator-b3 == 1.27.0",
-  "opentelemetry-propagator-jaeger == 1.27.0",
-  "opentelemetry-exporter-otlp-proto-common == 1.27.0",
-  "opentelemetry-sdk-extension-aws == 2.0.2",
-  "opentelemetry-propagator-aws-xray == 1.0.1",
-  "opentelemetry-distro == 0.48b0",
-  "opentelemetry-propagator-ot-trace == 0.48b0",
-  "opentelemetry-instrumentation == 0.48b0",
-  "opentelemetry-instrumentation-aws-lambda == 0.48b0",
-  "opentelemetry-instrumentation-aio-pika == 0.48b0",
-  "opentelemetry-instrumentation-aiohttp-client == 0.48b0",
-  "opentelemetry-instrumentation-aiopg == 0.48b0",
-  "opentelemetry-instrumentation-asgi == 0.48b0",
-  "opentelemetry-instrumentation-asyncpg == 0.48b0",
-  "opentelemetry-instrumentation-boto == 0.48b0",
-  "opentelemetry-instrumentation-boto3sqs == 0.48b0",
-  "opentelemetry-instrumentation-botocore == 0.48b0",
-  "opentelemetry-instrumentation-celery == 0.48b0",
-  "opentelemetry-instrumentation-confluent-kafka == 0.48b0",
-  "opentelemetry-instrumentation-dbapi == 0.48b0",
-  "opentelemetry-instrumentation-django == 0.48b0",
-  "opentelemetry-instrumentation-elasticsearch == 0.48b0",
-  "opentelemetry-instrumentation-falcon == 0.48b0",
-  "opentelemetry-instrumentation-fastapi == 0.48b0",
-  "opentelemetry-instrumentation-flask == 0.48b0",
-  "opentelemetry-instrumentation-grpc == 0.48b0",
-  "opentelemetry-instrumentation-httpx == 0.48b0",
-  "opentelemetry-instrumentation-jinja2 == 0.48b0",
-  "opentelemetry-instrumentation-kafka-python == 0.48b0",
-  "opentelemetry-instrumentation-logging == 0.48b0",
-  "opentelemetry-instrumentation-mysql == 0.48b0",
-  "opentelemetry-instrumentation-mysqlclient == 0.48b0",
-  "opentelemetry-instrumentation-pika == 0.48b0",
-  "opentelemetry-instrumentation-psycopg2 == 0.48b0",
-  "opentelemetry-instrumentation-pymemcache == 0.48b0",
-  "opentelemetry-instrumentation-pymongo == 0.48b0",
-  "opentelemetry-instrumentation-pymysql == 0.48b0",
-  "opentelemetry-instrumentation-pyramid == 0.48b0",
-  "opentelemetry-instrumentation-redis == 0.48b0",
-  "opentelemetry-instrumentation-remoulade == 0.48b0",
-  "opentelemetry-instrumentation-requests == 0.48b0",
-  "opentelemetry-instrumentation-sqlalchemy == 0.48b0",
-  "opentelemetry-instrumentation-sqlite3 == 0.48b0",
-  "opentelemetry-instrumentation-starlette == 0.48b0",
-  "opentelemetry-instrumentation-system-metrics == 0.48b0",
-  "opentelemetry-instrumentation-tornado == 0.48b0",
-  "opentelemetry-instrumentation-tortoiseorm == 0.48b0",
-  "opentelemetry-instrumentation-urllib == 0.48b0",
-  "opentelemetry-instrumentation-urllib3 == 0.48b0",
-  "opentelemetry-instrumentation-wsgi == 0.48b0",
-  "opentelemetry-instrumentation-cassandra == 0.48b0",
+  "opentelemetry-api >= 1.29.0",
+  "opentelemetry-sdk >= 1.29.0",
+  "opentelemetry-exporter-otlp-proto-grpc >= 1.29.0",
+  "opentelemetry-exporter-otlp-proto-http >= 1.29.0",
+  "opentelemetry-propagator-b3 >= 1.29.0",
+  "opentelemetry-propagator-jaeger >= 1.29.0",
+  "opentelemetry-exporter-otlp-proto-common >= 1.29.0",
+  "opentelemetry-sdk-extension-aws >= 2.0.2",
+  "opentelemetry-propagator-aws-xray >= 1.0.1",
+  "opentelemetry-distro >= 0.50b0",
+  "opentelemetry-propagator-ot-trace >= 0.50b0",
+  "opentelemetry-instrumentation >= 0.50b0",
+  "opentelemetry-instrumentation-aws-lambda >= 0.50b0",
+  "opentelemetry-instrumentation-aio-pika >= 0.50b0",
+  "opentelemetry-instrumentation-aiohttp-client >= 0.50b0",
+  "opentelemetry-instrumentation-aiopg >= 0.50b0",
+  "opentelemetry-instrumentation-asgi >= 0.50b0",
+  "opentelemetry-instrumentation-asyncpg >= 0.50b0",
+  "opentelemetry-instrumentation-boto >= 0.50b0",
+  "opentelemetry-instrumentation-boto3sqs >= 0.50b0",
+  "opentelemetry-instrumentation-botocore >= 0.50b0",
+  "opentelemetry-instrumentation-celery >= 0.50b0",
+  "opentelemetry-instrumentation-confluent-kafka >= 0.50b0",
+  "opentelemetry-instrumentation-dbapi >= 0.50b0",
+  "opentelemetry-instrumentation-django >= 0.50b0",
+  "opentelemetry-instrumentation-elasticsearch >= 0.50b0",
+  "opentelemetry-instrumentation-falcon >= 0.50b0",
+  "opentelemetry-instrumentation-fastapi >= 0.50b0",
+  "opentelemetry-instrumentation-flask >= 0.50b0",
+  "opentelemetry-instrumentation-grpc >= 0.50b0",
+  "opentelemetry-instrumentation-httpx >= 0.50b0",
+  "opentelemetry-instrumentation-jinja2 >= 0.50b0",
+  "opentelemetry-instrumentation-kafka-python >= 0.50b0",
+  "opentelemetry-instrumentation-logging >= 0.50b0",
+  "opentelemetry-instrumentation-mysql >= 0.50b0",
+  "opentelemetry-instrumentation-mysqlclient >= 0.50b0",
+  "opentelemetry-instrumentation-pika >= 0.50b0",
+  "opentelemetry-instrumentation-psycopg2 >= 0.50b0",
+  "opentelemetry-instrumentation-pymemcache >= 0.50b0",
+  "opentelemetry-instrumentation-pymongo >= 0.50b0",
+  "opentelemetry-instrumentation-pymysql >= 0.50b0",
+  "opentelemetry-instrumentation-pyramid >= 0.50b0",
+  "opentelemetry-instrumentation-redis >= 0.50b0",
+  "opentelemetry-instrumentation-remoulade >= 0.50b0",
+  "opentelemetry-instrumentation-requests >= 0.50b0",
+  "opentelemetry-instrumentation-sqlalchemy >= 0.50b0",
+  "opentelemetry-instrumentation-sqlite3 >= 0.50b0",
+  "opentelemetry-instrumentation-starlette >= 0.50b0",
+  "opentelemetry-instrumentation-system-metrics >= 0.50b0",
+  "opentelemetry-instrumentation-tornado >= 0.50b0",
+  "opentelemetry-instrumentation-tortoiseorm >= 0.50b0",
+  "opentelemetry-instrumentation-urllib >= 0.50b0",
+  "opentelemetry-instrumentation-urllib3 >= 0.50b0",
+  "opentelemetry-instrumentation-wsgi >= 0.50b0",
+  "opentelemetry-instrumentation-cassandra >= 0.50b0",
 ]
 
 [project.optional-dependencies]

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_utils.py

@@ -1,13 +1,15 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import os
 import sys
 from logging import Logger, getLogger
 
 import pkg_resources
 
 _logger: Logger = getLogger(__name__)
 
+AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
 
 def is_installed(req: str) -> bool:
     """Is the given required package installed?"""
@@ -21,3 +23,6 @@ def is_installed(req: str) -> bool:
         _logger.debug("Skipping instrumentation patch: package %s, exception: %s", req, exc)
         return False
     return True
+
+def is_agent_observability_enabled() -> bool:
+    return os.environ.get(AGENT_OBSERVABILITY_ENABLED, "false").lower() == "true"

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -10,6 +10,7 @@
 from typing_extensions import override
 
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_LOCAL_SERVICE
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
 from amazon.opentelemetry.distro._aws_resource_attribute_configurator import get_service_attribute
 from amazon.opentelemetry.distro.always_record_sampler import AlwaysRecordSampler
 from amazon.opentelemetry.distro.attribute_propagating_span_processor_builder import (
@@ -27,7 +28,7 @@
 from amazon.opentelemetry.distro.sampler.aws_xray_remote_sampler import AwsXRayRemoteSampler
 from amazon.opentelemetry.distro.scope_based_exporter import ScopeBasedPeriodicExportingMetricReader
 from amazon.opentelemetry.distro.scope_based_filtering_view import ScopeBasedRetainingView
-from opentelemetry._logs import set_logger_provider
+from opentelemetry._logs import set_logger_provider, get_logger_provider
 from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHttpOTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
@@ -91,6 +92,8 @@
 OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"
 OTEL_EXPORTER_OTLP_LOGS_HEADERS = "OTEL_EXPORTER_OTLP_LOGS_HEADERS"
 
+AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
+
 AWS_TRACES_OTLP_ENDPOINT_PATTERN = r"https://xray\.([a-z0-9-]+)\.amazonaws\.com/v1/traces$"
 AWS_LOGS_OTLP_ENDPOINT_PATTERN = r"https://logs\.([a-z0-9-]+)\.amazonaws\.com/v1/logs$"
 
@@ -160,16 +163,17 @@ def _initialize_components():
     sampler_name = _get_sampler()
     sampler = _custom_import_sampler(sampler_name, resource)
 
+    logging_enabled = os.getenv(_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "false")
+    if logging_enabled.strip().lower() == "true":
+        _init_logging(log_exporters, resource)
+
     _init_tracing(
         exporters=trace_exporters,
         id_generator=id_generator,
         sampler=sampler,
         resource=resource,
     )
     _init_metrics(metric_exporters, resource)
-    logging_enabled = os.getenv(_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED, "false")
-    if logging_enabled.strip().lower() == "true":
-        _init_logging(log_exporters, resource)
 
 
 def _init_logging(
@@ -359,7 +363,15 @@ def _customize_span_exporter(span_exporter: SpanExporter, resource: Resource) ->
         _logger.info("Detected using AWS OTLP Traces Endpoint.")
 
         if isinstance(span_exporter, OTLPSpanExporter):
-            span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint)
+            if is_agent_observability_enabled():
+                logs_endpoint = os.getenv(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT)
+                logs_exporter = OTLPAwsLogExporter(endpoint=logs_endpoint)
+                span_exporter = OTLPAwsSpanExporter(
+                    endpoint=traces_endpoint,
+                    logger_provider=get_logger_provider()
+                )
+            else:
+                span_exporter = OTLPAwsSpanExporter(endpoint=traces_endpoint)
 
         else:
             _logger.warning(

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/traces/otlp_aws_span_exporter.py

@@ -1,11 +1,18 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import Dict, Optional
+from typing import Dict, Optional, Sequence
 
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
+from amazon.opentelemetry.distro.llo_handler import LLOHandler
+from amazon.opentelemetry.distro._utils import is_agent_observability_enabled
+from opentelemetry.sdk._logs import LoggerProvider
 from opentelemetry.exporter.otlp.proto.http import Compression
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+
+AGENT_OBSERVABILITY_ENABLED = "AGENT_OBSERVABILITY_ENABLED"
 
 
 class OTLPAwsSpanExporter(OTLPSpanExporter):
@@ -18,9 +25,13 @@ def __init__(
         headers: Optional[Dict[str, str]] = None,
         timeout: Optional[int] = None,
         compression: Optional[Compression] = None,
+        logger_provider: Optional[LoggerProvider] = None
     ):
         self._aws_region = None
 
+        if logger_provider:
+            self._llo_handler = LLOHandler(logger_provider)
+
         if endpoint:
             self._aws_region = endpoint.split(".")[1]
 
@@ -35,3 +46,10 @@ def __init__(
             compression,
             session=AwsAuthSession(aws_region=self._aws_region, service="xray"),
         )
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        if is_agent_observability_enabled():
+            llo_processed_spans = self._llo_handler.process_spans(spans)
+            return super().export(llo_processed_spans)
+
+        return super().export(spans)