Refactor test llmobs writer out to utils, add release note

Author: Yun-Kim

releasenotes/notes/feat-llmobs-evals-agent-proxy-55e35060f1aa2555.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    LLM Observability: Introduces support for submitting custom evaluations to LLM Observability via agent proxy mode.

tests/contrib/botocore/test_bedrock_llmobs.py

@@ -12,8 +12,7 @@
 from ddtrace.contrib.internal.botocore.patch import unpatch
 from ddtrace.llmobs import LLMObs
 from ddtrace.llmobs import LLMObs as llmobs_service
-from ddtrace.llmobs._constants import AGENTLESS_BASE_URL
-from ddtrace.llmobs._writer import LLMObsSpanWriter
+from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
 from ddtrace.trace import Pin
 from tests.contrib.botocore.bedrock_utils import _MOCK_RESPONSE_DATA
 from tests.contrib.botocore.bedrock_utils import _MODELS
@@ -22,20 +21,12 @@
 from tests.contrib.botocore.bedrock_utils import bedrock_converse_args_with_system_and_tool
 from tests.contrib.botocore.bedrock_utils import create_bedrock_converse_request
 from tests.contrib.botocore.bedrock_utils import get_request_vcr
+from tests.llmobs._utils import TestLLMObsSpanWriter
 from tests.llmobs._utils import _expected_llmobs_llm_span_event
 from tests.utils import DummyTracer
 from tests.utils import override_global_config
 
 
-class TestLLMObsSpanWriter(LLMObsSpanWriter):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.events = []
-
-    def enqueue(self, event):
-        self.events.append(event)
-
-
 @pytest.fixture(scope="session")
 def request_vcr():
     yield get_request_vcr()
@@ -58,7 +49,7 @@ def aws_credentials():
 
 
 @pytest.fixture
-def boto3(aws_credentials, mock_llmobs_span_writer, ddtrace_global_config):
+def boto3(aws_credentials, llmobs_span_writer, ddtrace_global_config):
     global_config = {"_dd_api_key": "<not-a-real-api_key>"}
     global_config.update(ddtrace_global_config)
     with override_global_config(global_config):
@@ -93,21 +84,9 @@ def bedrock_client_proxy(boto3):
     yield bedrock_client
 
 
-@pytest.fixture
-def mock_llmobs_span_writer():
-    patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter")
-    try:
-        LLMObsSpanWriterMock = patcher.start()
-        m = mock.MagicMock()
-        LLMObsSpanWriterMock.return_value = m
-        yield m
-    finally:
-        patcher.stop()
-
-
 @pytest.fixture
 def llmobs_span_writer():
-    agentless_url = "{}.{}".format(AGENTLESS_BASE_URL, "datad0g.com")
+    agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, "datad0g.com")
     yield TestLLMObsSpanWriter(is_agentless=True, _agentless_url=agentless_url, interval=1.0, timeout=1.0)
 
 
@@ -750,15 +729,15 @@ def test_llmobs_error(self, ddtrace_global_config, bedrock_client, mock_tracer,
         )
 
     @pytest.mark.skipif(BOTO_VERSION < (1, 34, 131), reason="Converse API not available until botocore 1.34.131")
-    def test_llmobs_converse(cls, bedrock_client, mock_llmobs_span_writer, request_vcr, mock_tracer, llmobs_events):
+    def test_llmobs_converse(cls, bedrock_client, request_vcr, mock_tracer, llmobs_events):
         request_params = create_bedrock_converse_request(**bedrock_converse_args_with_system_and_tool)
         with request_vcr.use_cassette("bedrock_converse.yaml"):
             response = bedrock_client.converse(**request_params)
 
         span = mock_tracer.pop_traces()[0][0]
         assert len(llmobs_events) == 1
 
-        llmobs_events[0] == _expected_llmobs_llm_span_event(
+        assert llmobs_events[0] == _expected_llmobs_llm_span_event(
             span,
             model_name="claude-3-sonnet-20240229-v1:0",
             model_provider="anthropic",
@@ -793,9 +772,7 @@ def test_llmobs_converse(cls, bedrock_client, mock_llmobs_span_writer, request_v
         )
 
     @pytest.mark.skipif(BOTO_VERSION < (1, 34, 131), reason="Converse API not available until botocore 1.34.131")
-    def test_llmobs_converse_error(
-        self, bedrock_client, mock_llmobs_span_writer, request_vcr, mock_tracer, llmobs_events
-    ):
+    def test_llmobs_converse_error(self, bedrock_client, request_vcr, mock_tracer, llmobs_events):
         """Test error handling for the Bedrock Converse API."""
         import botocore
 
@@ -825,9 +802,7 @@ def test_llmobs_converse_error(
         )
 
     @pytest.mark.skipif(BOTO_VERSION < (1, 34, 131), reason="Converse API not available until botocore 1.34.131")
-    def test_llmobs_converse_stream(
-        cls, bedrock_client, mock_llmobs_span_writer, request_vcr, mock_tracer, llmobs_events
-    ):
+    def test_llmobs_converse_stream(cls, bedrock_client, request_vcr, mock_tracer, llmobs_events):
         output_msg = ""
         request_params = create_bedrock_converse_request(**bedrock_converse_args_with_system_and_tool)
         with request_vcr.use_cassette("bedrock_converse_stream.yaml"):
@@ -840,7 +815,7 @@ def test_llmobs_converse_stream(
         span = mock_tracer.pop_traces()[0][0]
         assert len(llmobs_events) == 1
 
-        llmobs_events[0] == _expected_llmobs_llm_span_event(
+        assert llmobs_events[0] == _expected_llmobs_llm_span_event(
             span,
             model_name="claude-3-sonnet-20240229-v1:0",
             model_provider="anthropic",

tests/contrib/langgraph/conftest.py

@@ -10,11 +10,11 @@
 from ddtrace.contrib.internal.langgraph.patch import patch
 from ddtrace.contrib.internal.langgraph.patch import unpatch
 from ddtrace.llmobs import LLMObs as llmobs_service
-from ddtrace.llmobs._constants import AGENTLESS_BASE_URL
-from ddtrace.llmobs._writer import LLMObsSpanWriter
+from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
 from ddtrace.trace import Pin
 from tests.utils import DummyTracer
 from tests.utils import override_global_config
+from tests.llmobs._utils import TestLLMObsSpanWriter
 
 
 DATADOG_SITE = "datad0g.com"
@@ -41,18 +41,9 @@ def default_global_config():
     return {"_dd_api_key": "<not-a-real-api_key>", "_llmobs_ml_app": "unnamed-ml-app", "service": "tests.llmobs"}
 
 
-class TestLLMObsSpanWriter(LLMObsSpanWriter):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.events = []
-
-    def enqueue(self, event):
-        self.events.append(event)
-
-
 @pytest.fixture
 def llmobs_span_writer():
-    agentless_url = "{}.{}".format(AGENTLESS_BASE_URL, DATADOG_SITE)
+    agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, DATADOG_SITE)
     yield TestLLMObsSpanWriter(is_agentless=True, _agentless_url=agentless_url, interval=1.0, timeout=1.0)
 
 

tests/llmobs/_utils.py

@@ -14,6 +14,7 @@
 from ddtrace.llmobs._constants import INTEGRATION
 from ddtrace.llmobs._utils import _get_span_name
 from ddtrace.llmobs._writer import LLMObsEvaluationMetricEvent
+from ddtrace.llmobs._writer import LLMObsSpanWriter
 from ddtrace.trace import Span
 
 
@@ -798,3 +799,15 @@ def _expected_span_link(span_event, link_from, link_to):
         "span_id": span_event["span_id"],
         "attributes": {"from": link_from, "to": link_to},
     }
+
+
+class TestLLMObsSpanWriter(LLMObsSpanWriter):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.events = []
+
+    def enqueue(self, event):
+        self.events.append(event)
+
+    def events(self):
+        return self.events

tests/llmobs/conftest.py

@@ -11,7 +11,7 @@
 from ddtrace.internal.utils.http import Response
 from ddtrace.llmobs import LLMObs as llmobs_service
 from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator
-from ddtrace.llmobs._writer import LLMObsSpanWriter
+from tests.llmobs._utils import TestLLMObsSpanWriter
 from tests.llmobs._utils import logs_vcr
 from tests.utils import DummyTracer
 from tests.utils import override_env
@@ -197,19 +197,6 @@ def llmobs_env():
     }
 
 
-class TestLLMObsSpanWriter(LLMObsSpanWriter):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._events = []
-
-    def enqueue(self, event):
-        self._events.append(event)
-        super().enqueue(event)
-
-    def events(self):
-        return self._events
-
-
 @pytest.fixture
 def llmobs_span_writer(_llmobs_backend):
     url, _ = _llmobs_backend