Skip to content

Commit f561ec9

Browse files
Yun-KimYun-Kim
committed
fmt
1 parent 835bf84 commit f561ec9

7 files changed

+44
-37
lines changed

‎ddtrace/llmobs/_writer.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
from ddtrace.internal import forksafe
1919
from ddtrace.internal.logger import get_logger
2020
from ddtrace.internal.periodic import PeriodicService
21-
from ddtrace.internal.utils.http import get_connection
2221
from ddtrace.internal.utils.http import verify_url
2322
from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter
2423
from ddtrace.llmobs import _telemetry as telemetry
@@ -237,7 +236,7 @@ def _get_connection(self):
237236
def _url(self) -> str:
238237
return "{}{}".format(self._intake, self._endpoint)
239238

240-
def _data(self, events: List[Any]) -> Dict[str, Any]:
239+
def _data(self, events):
241240
"""Return payload containing events to be encoded and submitted to LLM Observability."""
242241
raise NotImplementedError
243242

‎tests/llmobs/conftest.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import mock
99
import pytest
1010

11-
from ddtrace.internal.utils.http import Response
1211
from ddtrace.llmobs import LLMObs as llmobs_service
1312
from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator
1413
from tests.llmobs._utils import TestLLMObsSpanWriter

‎tests/llmobs/test_llmobs_eval_metric_writer.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def test_buffer_limit(mock_writer_logs):
6969
@pytest.mark.vcr_logs
7070
def test_send_metric_bad_api_key(mock_writer_logs):
7171
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
72-
"datad0g.com", "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
72+
"datad0g.com", "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
7373
)
7474
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
7575
llmobs_eval_metric_writer.periodic()
@@ -123,7 +123,12 @@ def test_send_score_metric(mock_writer_logs):
123123
@pytest.mark.vcr_logs
124124
def test_send_timed_events(mock_writer_logs):
125125
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
126-
site="datad0g.com", api_key=DD_API_KEY, interval=0.01, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
126+
site="datad0g.com",
127+
api_key=DD_API_KEY,
128+
interval=0.01,
129+
timeout=1,
130+
is_agentless=True,
131+
_agentless_url=AGENTLESS_URL,
127132
)
128133
llmobs_eval_metric_writer.start()
129134
mock_writer_logs.reset_mock()
@@ -151,9 +156,7 @@ def test_send_multiple_events(mock_writer_logs):
151156
llmobs_eval_metric_writer.enqueue(_score_metric_event())
152157
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
153158
llmobs_eval_metric_writer.periodic()
154-
mock_writer_logs.debug.assert_called_with(
155-
"sent %d LLMObs %s events to %s", 2, "evaluation_metric", INTAKE_ENDPOINT
156-
)
159+
mock_writer_logs.debug.assert_called_with("sent %d LLMObs %s events to %s", 2, "evaluation_metric", INTAKE_ENDPOINT)
157160

158161

159162
def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess):

‎tests/llmobs/test_llmobs_ragas_evaluators.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
1-
import os
2-
31
import mock
42
import pytest
53

64
from ddtrace.llmobs._evaluators.ragas.answer_relevancy import RagasAnswerRelevancyEvaluator
75
from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator
86
from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator
9-
from ddtrace.llmobs._evaluators.runner import EvaluatorRunner
107
from ddtrace.trace import Span
118
from tests.llmobs._utils import _expected_llmobs_llm_span_event
129
from tests.llmobs._utils import _expected_ragas_answer_relevancy_spans

‎tests/llmobs/test_llmobs_span_agent_writer.py

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import pytest
21
import time
32

43
import mock
@@ -21,13 +20,17 @@
2120

2221

2322
def test_writer_start(mock_writer_logs):
24-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False)
23+
llmobs_span_writer = LLMObsSpanWriter(
24+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False
25+
)
2526
llmobs_span_writer.start()
2627
mock_writer_logs.debug.assert_has_calls([mock.call("started %r to %r", "LLMObsSpanWriter", AGENT_PROXY_URL)])
2728

2829

2930
def test_buffer_limit(mock_writer_logs):
30-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False)
31+
llmobs_span_writer = LLMObsSpanWriter(
32+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False
33+
)
3134
for _ in range(1001):
3235
llmobs_span_writer.enqueue({})
3336
mock_writer_logs.warning.assert_called_with(
@@ -37,7 +40,9 @@ def test_buffer_limit(mock_writer_logs):
3740

3841
@mock.patch("ddtrace.llmobs._writer.LLMObsSpanWriter._send_payload")
3942
def test_flush_queue_when_event_cause_queue_to_exceed_payload_limit(mock_send_payload, mock_writer_logs):
40-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False)
43+
llmobs_span_writer = LLMObsSpanWriter(
44+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False
45+
)
4146
llmobs_span_writer.enqueue(_large_event())
4247
llmobs_span_writer.enqueue(_large_event())
4348
llmobs_span_writer.enqueue(_large_event())
@@ -57,7 +62,9 @@ def test_flush_queue_when_event_cause_queue_to_exceed_payload_limit(mock_send_pa
5762

5863
@mock.patch("ddtrace.llmobs._writer.LLMObsSpanWriter._send_payload")
5964
def test_truncating_oversized_events(mock_send_payload, mock_writer_logs):
60-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False)
65+
llmobs_span_writer = LLMObsSpanWriter(
66+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1000, timeout=1, is_agentless=False
67+
)
6168
llmobs_span_writer.enqueue(_oversized_llm_event())
6269
llmobs_span_writer.enqueue(_oversized_retrieval_event())
6370
llmobs_span_writer.enqueue(_oversized_workflow_event())
@@ -72,23 +79,29 @@ def test_truncating_oversized_events(mock_send_payload, mock_writer_logs):
7279

7380
@mock.patch("ddtrace.llmobs._writer.BaseLLMObsWriter._send_payload")
7481
def test_send_completion_event(mock_send_payload, mock_writer_logs):
75-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False)
82+
llmobs_span_writer = LLMObsSpanWriter(
83+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False
84+
)
7685
llmobs_span_writer.enqueue(_completion_event())
7786
llmobs_span_writer.periodic()
7887
mock_writer_logs.debug.assert_has_calls([mock.call("encoded %d LLMObs %s events to be sent", 1, "span")])
7988

8089

8190
@mock.patch("ddtrace.llmobs._writer.BaseLLMObsWriter._send_payload")
8291
def test_send_chat_completion_event(mock_send_payload, mock_writer_logs):
83-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False)
92+
llmobs_span_writer = LLMObsSpanWriter(
93+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=False
94+
)
8495
llmobs_span_writer.enqueue(_chat_completion_event())
8596
llmobs_span_writer.periodic()
8697
mock_writer_logs.debug.assert_has_calls([mock.call("encoded %d LLMObs %s events to be sent", 1, "span")])
8798

8899

89100
@mock.patch("ddtrace.llmobs._writer.BaseLLMObsWriter._send_payload")
90101
def test_send_timed_events(mock_send_payload, mock_writer_logs):
91-
llmobs_span_writer = LLMObsSpanWriter(site=DATADOG_SITE, api_key=DD_API_KEY, interval=0.05, timeout=1, is_agentless=False)
102+
llmobs_span_writer = LLMObsSpanWriter(
103+
site=DATADOG_SITE, api_key=DD_API_KEY, interval=0.05, timeout=1, is_agentless=False
104+
)
92105
llmobs_span_writer.start()
93106
mock_writer_logs.reset_mock()
94107

‎tests/llmobs/test_llmobs_span_agentless_writer.py

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import pytest
21
import os
32
import time
43

54
import mock
5+
import pytest
66

77
from ddtrace.llmobs._writer import LLMObsSpanWriter
88
from tests.llmobs._utils import _chat_completion_event
@@ -62,15 +62,9 @@ def test_truncating_oversized_events(mock_writer_logs):
6262
llmobs_span_writer.enqueue(_oversized_workflow_event())
6363
mock_writer_logs.warning.assert_has_calls(
6464
[
65-
mock.call(
66-
"dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400724
67-
),
68-
mock.call(
69-
"dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400464
70-
),
71-
mock.call(
72-
"dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400445
73-
),
65+
mock.call("dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400724),
66+
mock.call("dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400464),
67+
mock.call("dropping event input/output because its size (%d) exceeds the event size limit (1MB)", 1400445),
7468
]
7569
)
7670

@@ -93,22 +87,26 @@ def test_send_chat_completion_event(mock_writer_logs):
9387

9488
@pytest.mark.vcr_logs
9589
def test_send_completion_bad_api_key(mock_writer_logs):
96-
llmobs_span_writer = LLMObsSpanWriter(DATADOG_SITE, "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=INTAKE_URL)
90+
llmobs_span_writer = LLMObsSpanWriter(
91+
DATADOG_SITE, "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=INTAKE_URL
92+
)
9793
llmobs_span_writer.enqueue(_completion_event())
9894
llmobs_span_writer.periodic()
9995
mock_writer_logs.error.assert_called_with(
100-
'failed to send %d LLMObs %s events to %s, got response code %d, status: %s',
96+
"failed to send %d LLMObs %s events to %s, got response code %d, status: %s",
10197
1,
102-
'span',
103-
'https://llmobs-intake.datad0g.com/api/v2/llmobs',
98+
"span",
99+
"https://llmobs-intake.datad0g.com/api/v2/llmobs",
104100
403,
105101
b'{"errors":[{"status":"403","title":"Forbidden","detail":"API key is invalid"}]}',
106102
)
107103

108104

109105
@pytest.mark.vcr_logs
110106
def test_send_timed_events(mock_writer_logs):
111-
llmobs_span_writer = LLMObsSpanWriter(DATADOG_SITE, DD_API_KEY, 0.01, 1, is_agentless=True, _agentless_url=INTAKE_URL)
107+
llmobs_span_writer = LLMObsSpanWriter(
108+
DATADOG_SITE, DD_API_KEY, 0.01, 1, is_agentless=True, _agentless_url=INTAKE_URL
109+
)
112110
llmobs_span_writer.start()
113111
mock_writer_logs.reset_mock()
114112

‎tests/llmobs/test_llmobs_span_encoder.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import json
2-
import pytest
32

4-
import mock
3+
import pytest
54

6-
import ddtrace
75
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
86
from tests.llmobs._utils import TestLLMObsSpanWriter
97
from tests.llmobs._utils import _chat_completion_event

0 commit comments

Comments
 (0)