Skip to content

Commit abd46cb

Browse files
Yun-KimYun-Kim
committed
Make site/agentless_url an internal detail
1 parent 3e8c16a commit abd46cb

11 files changed

+66
-121
lines changed

‎ddtrace/llmobs/_llmobs.py

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@
3333
from ddtrace.internal.utils.formats import parse_tags_str
3434
from ddtrace.llmobs import _constants as constants
3535
from ddtrace.llmobs import _telemetry as telemetry
36-
from ddtrace.llmobs._constants import AGENTLESS_EVAL_BASE_URL
37-
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
3836
from ddtrace.llmobs._constants import ANNOTATIONS_CONTEXT_ID
3937
from ddtrace.llmobs._constants import DECORATOR
4038
from ddtrace.llmobs._constants import DISPATCH_ON_LLM_TOOL_CHOICE
@@ -106,20 +104,16 @@ def __init__(self, tracer=None):
106104
self.tracer = tracer or ddtrace.tracer
107105
self._llmobs_context_provider = LLMObsContextProvider()
108106
self._llmobs_span_writer = LLMObsSpanWriter(
109-
site=config._dd_site,
110-
api_key=config._dd_api_key,
111107
interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)),
112108
timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)),
109+
site=config._dd_site,
113110
is_agentless=config._llmobs_agentless_enabled,
114-
_agentless_url="%s.%s" % (AGENTLESS_SPAN_BASE_URL, config._dd_site),
115111
)
116112
self._llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
117-
site=config._dd_site,
118-
api_key=config._dd_api_key,
119113
interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)),
120114
timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)),
115+
site=config._dd_site,
121116
is_agentless=config._llmobs_agentless_enabled,
122-
_agentless_url="%s.%s" % (AGENTLESS_EVAL_BASE_URL, config._dd_site),
123117
)
124118
self._evaluator_runner = EvaluatorRunner(
125119
interval=float(os.getenv("_DD_LLMOBS_EVALUATOR_INTERVAL", 1.0)),

‎ddtrace/llmobs/_writer.py

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import http.client as httplib
1515

1616
import ddtrace
17+
from ddtrace import config
1718
from ddtrace.internal import agent
1819
from ddtrace.internal import forksafe
1920
from ddtrace.internal.logger import get_logger
@@ -99,26 +100,26 @@ class BaseLLMObsWriter(PeriodicService):
99100

100101
def __init__(
101102
self,
102-
site: str,
103-
api_key: str,
104103
interval: float,
105104
timeout: float,
105+
site: str = "",
106+
api_key: str = "",
106107
is_agentless: bool = True,
107108
_agentless_url: str = "",
108109
) -> None:
109110
super(BaseLLMObsWriter, self).__init__(interval=interval)
110111
self._lock = forksafe.RLock()
111-
self._buffer = [] # type: List[Union[LLMObsSpanEvent, LLMObsEvaluationMetricEvent]]
112-
self._buffer_size = 0
113-
self._buffer_limit = 1000
114-
self._timeout = timeout # type: float
115-
self._api_key = api_key or "" # type: str
116-
self._endpoint = "" # type: str
117-
self._site = site # type: str
118-
self._intake = "" # type: str
119-
self._headers = {"Content-Type": "application/json"}
120-
self._agentless = is_agentless
121-
self._agentless_url = _agentless_url
112+
self._buffer: List[Union[LLMObsSpanEvent, LLMObsEvaluationMetricEvent]] = []
113+
self._buffer_size: int = 0
114+
self._buffer_limit: int = 1000
115+
self._timeout: float = timeout
116+
self._api_key: str = api_key or config._dd_api_key
117+
self._endpoint: str = ""
118+
self._site: str = site or config._dd_site
119+
self._intake: str = ""
120+
self._headers: Dict[str, str] = {"Content-Type": "application/json"}
121+
self._agentless: bool = is_agentless
122+
self._agentless_url: str = _agentless_url
122123
if is_agentless:
123124
self._headers["DD-API-KEY"] = self._api_key
124125
else:
@@ -258,18 +259,19 @@ class LLMObsEvalMetricWriter(BaseLLMObsWriter):
258259

259260
def __init__(
260261
self,
261-
site: str,
262-
api_key: str,
263262
interval: float,
264263
timeout: float,
264+
site: str = "",
265+
api_key: str = "",
265266
is_agentless: bool = True,
266267
_agentless_url: str = "",
267268
) -> None:
269+
# TODO: NEED TO FIX TEST WRITER CLASSES
268270
super(LLMObsEvalMetricWriter, self).__init__(
269-
site, api_key, interval, timeout, is_agentless=is_agentless, _agentless_url=_agentless_url
271+
interval, timeout, site, api_key, is_agentless=is_agentless, _agentless_url=_agentless_url
270272
)
271273
if self._agentless:
272-
self._intake = _agentless_url or AGENTLESS_EVAL_BASE_URL
274+
self._intake = _agentless_url or "{}.{}".format(AGENTLESS_EVAL_BASE_URL, self._site)
273275
self._endpoint = AGENTLESS_EVAL_ENDPOINT
274276
else:
275277
self._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_EVAL_SUBDOMAIN_HEADER_VALUE
@@ -289,18 +291,18 @@ class LLMObsSpanWriter(BaseLLMObsWriter):
289291

290292
def __init__(
291293
self,
292-
site: str,
293-
api_key: str,
294294
interval: float,
295295
timeout: float,
296+
site: str = "",
297+
api_key: str = "",
296298
is_agentless: bool = True,
297299
_agentless_url: str = "",
298300
) -> None:
299301
super(LLMObsSpanWriter, self).__init__(
300-
site, api_key, interval, timeout, is_agentless=is_agentless, _agentless_url=_agentless_url
302+
interval, timeout, site, api_key, is_agentless=is_agentless, _agentless_url=_agentless_url
301303
)
302304
if self._agentless:
303-
self._intake = _agentless_url or AGENTLESS_SPAN_BASE_URL
305+
self._intake = _agentless_url or "{}.{}".format(AGENTLESS_SPAN_BASE_URL, self._site)
304306
self._endpoint = AGENTLESS_SPAN_ENDPOINT
305307
else:
306308
self._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_SPAN_SUBDOMAIN_HEADER_VALUE

‎tests/contrib/botocore/test_bedrock_llmobs.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@
2727
from tests.utils import override_global_config
2828

2929

30-
DATADOG_SITE = "datad0g.com"
31-
32-
3330
@pytest.fixture(scope="session")
3431
def request_vcr():
3532
yield get_request_vcr()
@@ -89,9 +86,7 @@ def bedrock_client_proxy(boto3):
8986

9087
@pytest.fixture
9188
def llmobs_span_writer():
92-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, DATADOG_SITE)
93-
api_key = "<not-a-real-key>"
94-
yield TestLLMObsSpanWriter(DATADOG_SITE, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
89+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
9590

9691

9792
@pytest.fixture

‎tests/contrib/langgraph/conftest.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@
1717
from tests.utils import override_global_config
1818

1919

20-
DATADOG_SITE = "datad0g.com"
21-
22-
2320
@pytest.fixture
2421
def mock_tracer():
2522
yield DummyTracer()
@@ -42,9 +39,7 @@ def default_global_config():
4239

4340
@pytest.fixture
4441
def llmobs_span_writer():
45-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, DATADOG_SITE)
46-
api_key = "<not-a-real-key>"
47-
yield TestLLMObsSpanWriter(DATADOG_SITE, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
42+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
4843

4944

5045
@pytest.fixture

‎tests/contrib/openai_agents/conftest.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,7 @@ def mock_tracer_chat_completions(agents, openai, mock_tracer):
192192

193193
@pytest.fixture
194194
def llmobs_span_writer():
195-
dd_site = "datad0g.com"
196-
api_key = "<not-a-real-key>"
197-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, dd_site)
198-
yield TestLLMObsSpanWriter(dd_site, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
195+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
199196

200197

201198
@pytest.fixture

‎tests/llmobs/conftest.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,7 @@ def llmobs_env():
179179
@pytest.fixture
180180
def llmobs_span_writer(_llmobs_backend):
181181
url, _ = _llmobs_backend
182-
site = "datad0g.com"
183-
api_key = "<test-key>"
184-
yield TestLLMObsSpanWriter(site, api_key, 1.0, 1.0, is_agentless=True, _agentless_url=url)
182+
yield TestLLMObsSpanWriter(1.0, 1.0, api_key="<test-key>", is_agentless=True, _agentless_url=url)
185183

186184

187185
class LLMObsServer(BaseHTTPRequestHandler):

‎tests/llmobs/test_llmobs_eval_metric_writer.py

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ def _score_metric_event():
4848

4949
def test_writer_start(mock_writer_logs):
5050
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
51-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
51+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
5252
)
5353
llmobs_eval_metric_writer.start()
5454
mock_writer_logs.debug.assert_has_calls([mock.call("started %r to %r", "LLMObsEvalMetricWriter", INTAKE_ENDPOINT)])
@@ -57,7 +57,7 @@ def test_writer_start(mock_writer_logs):
5757

5858
def test_buffer_limit(mock_writer_logs):
5959
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
60-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
60+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
6161
)
6262
for _ in range(1001):
6363
llmobs_eval_metric_writer.enqueue({})
@@ -69,7 +69,7 @@ def test_buffer_limit(mock_writer_logs):
6969
@pytest.mark.vcr_logs
7070
def test_send_metric_bad_api_key(mock_writer_logs):
7171
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
72-
"datad0g.com", "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
72+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
7373
)
7474
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
7575
llmobs_eval_metric_writer.periodic()
@@ -83,10 +83,9 @@ def test_send_metric_bad_api_key(mock_writer_logs):
8383
)
8484

8585

86-
@pytest.mark.vcr_logs
8786
def test_send_metric_no_api_key(mock_writer_logs):
8887
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
89-
"datad0g.com", "", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
88+
interval=1, timeout=1, site=DD_SITE, api_key="", is_agentless=True
9089
)
9190
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
9291
llmobs_eval_metric_writer.periodic()
@@ -99,7 +98,7 @@ def test_send_metric_no_api_key(mock_writer_logs):
9998
@pytest.mark.vcr_logs
10099
def test_send_categorical_metric(mock_writer_logs):
101100
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
102-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
101+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
103102
)
104103
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
105104
llmobs_eval_metric_writer.periodic()
@@ -111,7 +110,7 @@ def test_send_categorical_metric(mock_writer_logs):
111110
@pytest.mark.vcr_logs
112111
def test_send_score_metric(mock_writer_logs):
113112
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
114-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
113+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
115114
)
116115
llmobs_eval_metric_writer.enqueue(_score_metric_event())
117116
llmobs_eval_metric_writer.periodic()
@@ -123,12 +122,7 @@ def test_send_score_metric(mock_writer_logs):
123122
@pytest.mark.vcr_logs
124123
def test_send_timed_events(mock_writer_logs):
125124
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
126-
site="datad0g.com",
127-
api_key=DD_API_KEY,
128-
interval=0.01,
129-
timeout=1,
130-
is_agentless=True,
131-
_agentless_url=AGENTLESS_URL,
125+
interval=0.01, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
132126
)
133127
llmobs_eval_metric_writer.start()
134128
mock_writer_logs.reset_mock()
@@ -150,7 +144,7 @@ def test_send_timed_events(mock_writer_logs):
150144
@pytest.mark.vcr_logs
151145
def test_send_multiple_events(mock_writer_logs):
152146
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
153-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
147+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
154148
)
155149
mock_writer_logs.reset_mock()
156150
llmobs_eval_metric_writer.enqueue(_score_metric_event())
@@ -164,19 +158,10 @@ def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess):
164158
pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
165159
if "PYTHONPATH" in env:
166160
pypath.append(env["PYTHONPATH"])
167-
env.update(
168-
{
169-
"DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"),
170-
"DD_SITE": "datad0g.com",
171-
"PYTHONPATH": ":".join(pypath),
172-
"DD_LLMOBS_ML_APP": "unnamed-ml-app",
173-
}
174-
)
161+
env.update({"PYTHONPATH": ":".join(pypath), "DD_LLMOBS_ML_APP": "unnamed-ml-app"})
175162
out, err, status, pid = run_python_code_in_subprocess(
176163
"""
177164
import atexit
178-
import os
179-
import time
180165
181166
from ddtrace.llmobs._writer import LLMObsEvalMetricWriter
182167
from tests.llmobs.test_llmobs_eval_metric_writer import _score_metric_event
@@ -186,7 +171,7 @@ def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess):
186171
ctx.__enter__()
187172
atexit.register(lambda: ctx.__exit__())
188173
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
189-
site="datad0g.com", api_key=os.getenv("DD_API_KEY"), interval=0.01, timeout=1
174+
interval=0.01, timeout=1, site="datad0g.com", api_key="<not-a-real-key>", is_agentless=False
190175
)
191176
llmobs_eval_metric_writer.start()
192177
llmobs_eval_metric_writer.enqueue(_score_metric_event())

‎tests/llmobs/test_llmobs_evaluator_runner.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ def test_evaluator_runner_buffer_limit(mock_evaluator_logs):
4141
)
4242

4343

44+
@pytest.mark.vcr_logs
4445
def test_evaluator_runner_periodic_enqueues_eval_metric(mock_llmobs_eval_metric_writer, active_evaluator_runner):
4546
active_evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN)
4647
active_evaluator_runner.periodic()
@@ -57,6 +58,7 @@ def test_evaluator_runner_stopped_does_not_enqueue_metric(llmobs, mock_llmobs_ev
5758
assert mock_llmobs_eval_metric_writer.enqueue.call_count == 0
5859

5960

61+
@pytest.mark.vcr_logs
6062
def test_evaluator_runner_timed_enqueues_eval_metric(llmobs, mock_llmobs_eval_metric_writer, active_evaluator_runner):
6163
active_evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN)
6264

@@ -95,7 +97,9 @@ def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subproces
9597
pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
9698
if "PYTHONPATH" in env:
9799
pypath.append(env["PYTHONPATH"])
98-
env.update({"PYTHONPATH": ":".join(pypath), "_DD_LLMOBS_EVALUATOR_INTERVAL": "5"})
100+
env.update(
101+
{"PYTHONPATH": ":".join(pypath), "_DD_LLMOBS_EVALUATOR_INTERVAL": "0.01", "_DD_LLMOBS_WRITER_INTERVAL": "0.01"}
102+
)
99103
out, err, status, pid = run_python_code_in_subprocess(
100104
"""
101105
from ddtrace.llmobs import LLMObs

0 commit comments

Comments
 (0)