[Feature] ✨ adding BACKEND support for OpenTelemetry (OTEL) (mage-ai#4476)

* 🔧 docker-compose changes

* 📝 mint updates

* ✨ otel integration with mage-ai server

* ⬆️ fixing requirements

* 🔧 fixing deps and versions

* 💄 adding missing dep in setup.py

* ♻️ cleanup

* 📝 adding docs

Author: mfreeman451

docker-compose.yml

@@ -19,6 +19,9 @@ x-server_settings: &server_settings
     - ECS_TASK_DEFINITION=$ECS_TASK_DEFINITION
     - ENABLE_NEW_RELIC=$ENABLE_NEW_RELIC
     - ENABLE_PROMETHEUS=$ENABLE_PROMETHEUS
+    - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}
+    - OTEL_EXPORTER_OTLP_HTTP_ENDPOINT=${OTEL_EXPORTER_OTLP_HTTP_ENDPOINT}
+    - OTEL_PYTHON_TORNADO_EXCLUDED_URLS=$OTEL_PYTHON_TORNADO_EXCLUDED_URLS
     - ENV=dev
     - GCP_PROJECT_ID=$GCP_PROJECT_ID
     - GCP_REGION=$GCP_REGION

docs/integrations/observability/opentelemetry.mdx

@@ -0,0 +1,58 @@
+---
+title: "Monitoring with OpenTelemetry"
+sidebarTitle: "OpenTelemetry"
+---
+
+## Overview
+
+This guide details how to monitor your `mage_ai` application using OpenTelemetry. OpenTelemetry provides a unified approach to collecting telemetry data, such as traces and metrics, which are essential for understanding your application's performance and behavior.
+
+## Prerequisites
+
+Ensure the following prerequisites are in place before proceeding:
+- An OpenTelemetry Collector is set up and configured to receive telemetry data.
+- The `mage_ai` application is ready for instrumentation with OpenTelemetry.
+
+## Configuring OpenTelemetry in `mage_ai`
+
+### Setting Environment Variables
+
+Configure the OpenTelemetry Exporters in your application by setting the following environment variables:
+
+1. **OTLP HTTP Exporter**:
+   Use `OTEL_EXPORTER_OTLP_HTTP_ENDPOINT` to specify the HTTP endpoint of your OpenTelemetry Collector.
+
+   Example:
+   ```bash
+   export OTEL_EXPORTER_OTLP_HTTP_ENDPOINT="http://192.168.1.56:3418/v1/traces"
+   ```
+
+2. **OTLP Exporter**:
+
+The OTEL_EXPORTER_OTLP_ENDPOINT environment variable sets a general collector endpoint, which can be used for both gRPC and HTTP connections.
+
+Example:
+
+```bash
+export OTEL_EXPORTER_OTLP_ENDPOINT="192.168.1.56:3417"
+```
+
+### Instrumentation in the Application
+
+1. **SQLAlchemy Instrumentation**:
+The application uses the SQLAlchemyInstrumentor from OpenTelemetry to instrument database operations. This generates telemetry data like traces for database interactions.
+
+2. **Tornado Instrumentation**:
+To monitor HTTP server operations, the application integrates OpenTelemetry's Tornado instrumentation. This collects valuable data related to HTTP requests and server performance.
+
+## Telemetry Data Collection and Analysis
+
+With the environment variables set and the application running, OpenTelemetry will start collecting telemetry data based on the defined instrumentation.
+
+1. **Data Collection**:
+The application will send the collected telemetry data, including traces and metrics, to the specified OpenTelemetry Collector endpoint.
+
+2. **Analyzing Telemetry Data**:
+For effective visualization and analysis, you can connect your OpenTelemetry Collector to backend tools that support OpenTelemetry data (like Grafana or Jaeger).
+
+

docs/mint.json

@@ -433,7 +433,8 @@
                     "integrations/observability/metaplane",
                     "integrations/observability/newrelic",
                     "integrations/observability/sentry",
-                    "integrations/observability/prometheus"
+                    "integrations/observability/prometheus",
+                    "integrations/observability/opentelemetry"
                   ]
                 },
                 {

mage_ai/orchestration/db/__init__.py

@@ -23,6 +23,9 @@
     pool_pre_ping=True,
 )
 
+# Only import if OpenTelemetry is enabled
+if os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 if is_test():
     db_connection_url = f'sqlite:///{TEST_DB}'
@@ -50,6 +53,10 @@
     db_kwargs['connect_args']['options'] = '-c timezone=utc'
 
 try:
+    # if OpenTelemetry is enabled, instrument SQLAlchemy
+    if os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+        SQLAlchemyInstrumentor().instrument(enable_commenter=True, commenter_options={})
+
     engine = create_engine(
         db_connection_url,
         **db_kwargs,

mage_ai/server/server.py

@@ -86,6 +86,7 @@
     ENABLE_PROMETHEUS,
     LDAP_ADMIN_USERNAME,
     OAUTH2_APPLICATION_CLIENT_ID,
+    OTEL_EXPORTER_OTLP_ENDPOINT,
     REDIS_URL,
     REQUESTS_BASE_PATH,
     REQUIRE_USER_AUTHENTICATION,
@@ -344,6 +345,43 @@ def make_app(template_dir: str = None, update_routes: bool = False):
         (r'/version-control', MainPageHandler),
     ]
 
+    if ENABLE_PROMETHEUS or OTEL_EXPORTER_OTLP_ENDPOINT:
+        from opentelemetry.instrumentation.tornado import TornadoInstrumentor
+        TornadoInstrumentor().instrument()
+        logger.info('OpenTelemetry instrumentation enabled.')
+
+    if OTEL_EXPORTER_OTLP_ENDPOINT:
+        logger.info(f'OTEL_EXPORTER_OTLP_ENDPOINT: {OTEL_EXPORTER_OTLP_ENDPOINT}')
+
+        from opentelemetry import trace
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter,
+        )
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+        service_name = "mage-ai-server"
+        resource = Resource(attributes={
+            "service.name": service_name,
+        })
+
+        # Set up a TracerProvider and attach an OTLP exporter to it
+        trace.set_tracer_provider(TracerProvider(resource=resource))
+        tracer_provider = trace.get_tracer_provider()
+
+        # Configure OTLP exporter
+        otlp_exporter = OTLPSpanExporter(
+            # Endpoint of your OpenTelemetry Collector
+            endpoint=OTEL_EXPORTER_OTLP_ENDPOINT,
+            # Use insecure channel if your collector does not support TLS
+            insecure=True
+        )
+
+        # Attach the OTLP exporter to the TracerProvider
+        span_processor = BatchSpanProcessor(otlp_exporter)
+        tracer_provider.add_span_processor(span_processor)
+
     if ENABLE_PROMETHEUS:
         from opentelemetry import metrics
         from opentelemetry.exporter.prometheus import PrometheusMetricReader

mage_ai/settings/__init__.py

@@ -109,6 +109,13 @@ def get_bool_value(value: str) -> bool:
 # If enabled, the /metrics route will expose Tornado server metrics
 ENABLE_PROMETHEUS = get_bool_value(os.getenv('ENABLE_PROMETHEUS', 'False'))
 
+# OpenTelemetry configuration
+OTEL_EXPORTER_OTLP_ENDPOINT = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', None)
+OTEL_EXPORTER_OTLP_HTTP_ENDPOINT = os.getenv('OTEL_EXPORTER_HTTP_OTLP_ENDPOINT', None)
+OTEL_PYTHON_TORNADO_EXCLUDED_URLS = (
+    os.getenv('OTEL_PYTHON_TORNADO_EXCLUDED_URLS') or '/api/statuses'
+)
+
 DEFAULT_LOCALHOST_URL = 'http://localhost:6789'
 MAGE_PUBLIC_HOST = os.getenv('MAGE_PUBLIC_HOST') or DEFAULT_LOCALHOST_URL
 
@@ -164,6 +171,8 @@ def get_bool_value(value: str) -> bool:
     'SCHEDULER_TRIGGER_INTERVAL',
     'REQUIRE_USER_PERMISSIONS',
     'ENABLE_PROMETHEUS',
+    'OTEL_EXPORTER_OTLP_ENDPOINT',
+    'OTEL_EXPORTER_OTLP_HTTP_ENDPOINT',
     'OKTA_DOMAIN_URL',
     'OKTA_CLIENT_ID',
     'OKTA_CLIENT_SECRET',

requirements.txt

@@ -67,8 +67,10 @@ kubernetes>=28.1.0
 langchain>=0.0.222; python_version >= '3.8'
 mysql-connector-python~=8.2.0
 openai>=0.27.8, <1.0.0
-opentelemetry-exporter-prometheus==0.41b0
-opentelemetry-instrumentation-tornado==0.41b0
+opentelemetry-exporter-prometheus==0.43b0
+opentelemetry-instrumentation-tornado==0.43b0
+opentelemetry-exporter-otlp==1.22.0
+opentelemetry-instrumentation-sqlalchemy==0.43b0
 oracledb==1.3.1
 pinotdb==5.1.0
 prometheus_client>=0.18.0

scripts/dev.sh

@@ -42,6 +42,21 @@ case $key in
     shift # past argument
     shift # past value
     ;;
+    --otel_exporter_otlp_endpoint)
+    OTEL_EXPORTER_OTLP_ENDPOINT="$3"
+    shift # past argument
+    shift # past value
+    ;;
+    --otel_exporter_otlp_http_endpoint)
+    OTEL_EXPORTER_OTLP_HTTP_ENDPOINT="$3"
+    shift # past argument
+    shift # past value
+    ;;
+    --otel_python_tornado_excluded_urls)
+    OTEL_PYTHON_TORNADO_EXCLUDED_URLS="$3"
+    shift # past argument
+    shift # past value
+    ;;
     --huggingface_api)
     HUGGINGFACE_API="$3"
     shift # past argument
@@ -143,6 +158,9 @@ export ECS_TASK_DEFINITION=$ECS_TASK_DEFINITION
 export ECS_CONTAINER_NAME=$ECS_CONTAINER_NAME
 export ENABLE_NEW_RELIC=$ENABLE_NEW_RELIC
 export ENABLE_PROMETHEUS=$ENABLE_PROMETHEUS
+export OTEL_EXPORTER_OTLP_ENDPOINT=$OTEL_EXPORTER_OTLP_ENDPOINT
+export OTEL_EXPORTER_OTLP_HTTP_ENDPOINT=$OTEL_EXPORTER_OTLP_HTTP_ENDPOINT
+export OTEL_PYTHON_TORNADO_EXCLUDED_URLS=$OTEL_PYTHON_TORNADO_EXCLUDED_URLS
 
 export GCP_PROJECT_ID=$GCP_PROJECT_ID
 export GCP_PATH_TO_CREDENTIALS=$GCP_PATH_TO_CREDENTIALS

setup.py

@@ -171,8 +171,10 @@ def readme():
             'nkeys~=0.1.0',
             'openai>=0.27.8, <1.0.0',
             'opensearch-py==2.0.0',
-            'opentelemetry-exporter-prometheus==0.41b0',
-            'opentelemetry-instrumentation-tornado==0.41b0',
+            'opentelemetry-exporter-prometheus==0.43b0',
+            'opentelemetry-instrumentation-tornado~=0.43b0',
+            'opentelemetry-exporter-otlp~=1.22.0',
+            'opentelemetry-instrumentation-sqlalchemy~=0.42b0',
             'oracledb==1.3.1',
             'pika==1.3.1',
             'pinotdb==5.1.0',