Skip to content

Commit b3b960b

Browse files
Yun-KimYun-Kim
committed
Make site/agentless_url an internal detail
1 parent 3e8c16a commit b3b960b

11 files changed

+68
-124
lines changed

‎ddtrace/llmobs/_llmobs.py

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@
3333
from ddtrace.internal.utils.formats import parse_tags_str
3434
from ddtrace.llmobs import _constants as constants
3535
from ddtrace.llmobs import _telemetry as telemetry
36-
from ddtrace.llmobs._constants import AGENTLESS_EVAL_BASE_URL
37-
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
3836
from ddtrace.llmobs._constants import ANNOTATIONS_CONTEXT_ID
3937
from ddtrace.llmobs._constants import DECORATOR
4038
from ddtrace.llmobs._constants import DISPATCH_ON_LLM_TOOL_CHOICE
@@ -106,20 +104,18 @@ def __init__(self, tracer=None):
106104
self.tracer = tracer or ddtrace.tracer
107105
self._llmobs_context_provider = LLMObsContextProvider()
108106
self._llmobs_span_writer = LLMObsSpanWriter(
109-
site=config._dd_site,
110-
api_key=config._dd_api_key,
111107
interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)),
112108
timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)),
109+
site=config._dd_site,
110+
api_key=config._dd_api_key,
113111
is_agentless=config._llmobs_agentless_enabled,
114-
_agentless_url="%s.%s" % (AGENTLESS_SPAN_BASE_URL, config._dd_site),
115112
)
116113
self._llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
117-
site=config._dd_site,
118-
api_key=config._dd_api_key,
119114
interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)),
120115
timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)),
116+
site=config._dd_site,
117+
api_key=config._dd_api_key,
121118
is_agentless=config._llmobs_agentless_enabled,
122-
_agentless_url="%s.%s" % (AGENTLESS_EVAL_BASE_URL, config._dd_site),
123119
)
124120
self._evaluator_runner = EvaluatorRunner(
125121
interval=float(os.getenv("_DD_LLMOBS_EVALUATOR_INTERVAL", 1.0)),

‎ddtrace/llmobs/_writer.py

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import http.client as httplib
1515

1616
import ddtrace
17+
from ddtrace import config
1718
from ddtrace.internal import agent
1819
from ddtrace.internal import forksafe
1920
from ddtrace.internal.logger import get_logger
@@ -99,26 +100,26 @@ class BaseLLMObsWriter(PeriodicService):
99100

100101
def __init__(
101102
self,
102-
site: str,
103-
api_key: str,
104103
interval: float,
105104
timeout: float,
105+
site: str = "",
106+
api_key: str = "",
106107
is_agentless: bool = True,
107108
_agentless_url: str = "",
108109
) -> None:
109110
super(BaseLLMObsWriter, self).__init__(interval=interval)
110111
self._lock = forksafe.RLock()
111-
self._buffer = [] # type: List[Union[LLMObsSpanEvent, LLMObsEvaluationMetricEvent]]
112-
self._buffer_size = 0
113-
self._buffer_limit = 1000
114-
self._timeout = timeout # type: float
115-
self._api_key = api_key or "" # type: str
116-
self._endpoint = "" # type: str
117-
self._site = site # type: str
118-
self._intake = "" # type: str
119-
self._headers = {"Content-Type": "application/json"}
120-
self._agentless = is_agentless
121-
self._agentless_url = _agentless_url
112+
self._buffer: List[Union[LLMObsSpanEvent, LLMObsEvaluationMetricEvent]] = []
113+
self._buffer_size: int = 0
114+
self._buffer_limit: int = 1000
115+
self._timeout: float = timeout
116+
self._api_key: str = api_key or config._dd_api_key
117+
self._endpoint: str = ""
118+
self._site: str = site or config._dd_site
119+
self._intake: str = ""
120+
self._headers: Dict[str, str] = {"Content-Type": "application/json"}
121+
self._agentless: bool = is_agentless
122+
self._agentless_url: str = _agentless_url
122123
if is_agentless:
123124
self._headers["DD-API-KEY"] = self._api_key
124125
else:
@@ -258,18 +259,19 @@ class LLMObsEvalMetricWriter(BaseLLMObsWriter):
258259

259260
def __init__(
260261
self,
261-
site: str,
262-
api_key: str,
263262
interval: float,
264263
timeout: float,
264+
site: str = "",
265+
api_key: str = "",
265266
is_agentless: bool = True,
266267
_agentless_url: str = "",
267268
) -> None:
269+
# TODO: NEED TO FIX TEST WRITER CLASSES
268270
super(LLMObsEvalMetricWriter, self).__init__(
269-
site, api_key, interval, timeout, is_agentless=is_agentless, _agentless_url=_agentless_url
271+
interval, timeout, site, api_key, is_agentless=is_agentless, _agentless_url=_agentless_url
270272
)
271273
if self._agentless:
272-
self._intake = _agentless_url or AGENTLESS_EVAL_BASE_URL
274+
self._intake = _agentless_url or "{}.{}".format(AGENTLESS_EVAL_BASE_URL, self._site)
273275
self._endpoint = AGENTLESS_EVAL_ENDPOINT
274276
else:
275277
self._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_EVAL_SUBDOMAIN_HEADER_VALUE
@@ -289,18 +291,18 @@ class LLMObsSpanWriter(BaseLLMObsWriter):
289291

290292
def __init__(
291293
self,
292-
site: str,
293-
api_key: str,
294294
interval: float,
295295
timeout: float,
296+
site: str = "",
297+
api_key: str = "",
296298
is_agentless: bool = True,
297299
_agentless_url: str = "",
298300
) -> None:
299301
super(LLMObsSpanWriter, self).__init__(
300-
site, api_key, interval, timeout, is_agentless=is_agentless, _agentless_url=_agentless_url
302+
interval, timeout, site, api_key, is_agentless=is_agentless, _agentless_url=_agentless_url
301303
)
302304
if self._agentless:
303-
self._intake = _agentless_url or AGENTLESS_SPAN_BASE_URL
305+
self._intake = _agentless_url or "{}.{}".format(AGENTLESS_SPAN_BASE_URL, self._site)
304306
self._endpoint = AGENTLESS_SPAN_ENDPOINT
305307
else:
306308
self._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_SPAN_SUBDOMAIN_HEADER_VALUE

‎tests/contrib/botocore/test_bedrock_llmobs.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
from ddtrace.contrib.internal.botocore.patch import unpatch
1313
from ddtrace.llmobs import LLMObs
1414
from ddtrace.llmobs import LLMObs as llmobs_service
15-
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
1615
from ddtrace.trace import Pin
1716
from tests.contrib.botocore.bedrock_utils import _MOCK_RESPONSE_DATA
1817
from tests.contrib.botocore.bedrock_utils import _MODELS
@@ -27,9 +26,6 @@
2726
from tests.utils import override_global_config
2827

2928

30-
DATADOG_SITE = "datad0g.com"
31-
32-
3329
@pytest.fixture(scope="session")
3430
def request_vcr():
3531
yield get_request_vcr()
@@ -89,9 +85,7 @@ def bedrock_client_proxy(boto3):
8985

9086
@pytest.fixture
9187
def llmobs_span_writer():
92-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, DATADOG_SITE)
93-
api_key = "<not-a-real-key>"
94-
yield TestLLMObsSpanWriter(DATADOG_SITE, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
88+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
9589

9690

9791
@pytest.fixture

‎tests/contrib/langgraph/conftest.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,12 @@
1010
from ddtrace.contrib.internal.langgraph.patch import patch
1111
from ddtrace.contrib.internal.langgraph.patch import unpatch
1212
from ddtrace.llmobs import LLMObs as llmobs_service
13-
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
1413
from ddtrace.trace import Pin
1514
from tests.llmobs._utils import TestLLMObsSpanWriter
1615
from tests.utils import DummyTracer
1716
from tests.utils import override_global_config
1817

1918

20-
DATADOG_SITE = "datad0g.com"
21-
22-
2319
@pytest.fixture
2420
def mock_tracer():
2521
yield DummyTracer()
@@ -42,9 +38,7 @@ def default_global_config():
4238

4339
@pytest.fixture
4440
def llmobs_span_writer():
45-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, DATADOG_SITE)
46-
api_key = "<not-a-real-key>"
47-
yield TestLLMObsSpanWriter(DATADOG_SITE, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
41+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
4842

4943

5044
@pytest.fixture

‎tests/contrib/openai_agents/conftest.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from ddtrace.contrib.internal.openai_agents.patch import patch
1111
from ddtrace.contrib.internal.openai_agents.patch import unpatch
1212
from ddtrace.llmobs import LLMObs as llmobs_service
13-
from ddtrace.llmobs._constants import AGENTLESS_SPAN_BASE_URL
1413
from ddtrace.llmobs._writer import LLMObsSpanWriter
1514
from ddtrace.trace import Pin
1615
from tests.utils import DummyTracer
@@ -192,10 +191,7 @@ def mock_tracer_chat_completions(agents, openai, mock_tracer):
192191

193192
@pytest.fixture
194193
def llmobs_span_writer():
195-
dd_site = "datad0g.com"
196-
api_key = "<not-a-real-key>"
197-
agentless_url = "{}.{}".format(AGENTLESS_SPAN_BASE_URL, dd_site)
198-
yield TestLLMObsSpanWriter(dd_site, api_key, 1.0, 5.0, is_agentless=True, _agentless_url=agentless_url)
194+
yield TestLLMObsSpanWriter(1.0, 5.0, "datad0g.com", "<not-a-real-key>", is_agentless=True)
199195

200196

201197
@pytest.fixture

‎tests/llmobs/conftest.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,7 @@ def llmobs_env():
179179
@pytest.fixture
180180
def llmobs_span_writer(_llmobs_backend):
181181
url, _ = _llmobs_backend
182-
site = "datad0g.com"
183-
api_key = "<test-key>"
184-
yield TestLLMObsSpanWriter(site, api_key, 1.0, 1.0, is_agentless=True, _agentless_url=url)
182+
yield TestLLMObsSpanWriter(1.0, 1.0, api_key="<test-key>", is_agentless=True, _agentless_url=url)
185183

186184

187185
class LLMObsServer(BaseHTTPRequestHandler):

‎tests/llmobs/test_llmobs_eval_metric_writer.py

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ def _score_metric_event():
4848

4949
def test_writer_start(mock_writer_logs):
5050
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
51-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
51+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
5252
)
5353
llmobs_eval_metric_writer.start()
5454
mock_writer_logs.debug.assert_has_calls([mock.call("started %r to %r", "LLMObsEvalMetricWriter", INTAKE_ENDPOINT)])
@@ -57,7 +57,7 @@ def test_writer_start(mock_writer_logs):
5757

5858
def test_buffer_limit(mock_writer_logs):
5959
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
60-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
60+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
6161
)
6262
for _ in range(1001):
6363
llmobs_eval_metric_writer.enqueue({})
@@ -69,7 +69,7 @@ def test_buffer_limit(mock_writer_logs):
6969
@pytest.mark.vcr_logs
7070
def test_send_metric_bad_api_key(mock_writer_logs):
7171
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
72-
"datad0g.com", "<bad-api-key>", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
72+
interval=1, timeout=1, site=DD_SITE, api_key="<bad-api-key>", is_agentless=True
7373
)
7474
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
7575
llmobs_eval_metric_writer.periodic()
@@ -83,10 +83,9 @@ def test_send_metric_bad_api_key(mock_writer_logs):
8383
)
8484

8585

86-
@pytest.mark.vcr_logs
8786
def test_send_metric_no_api_key(mock_writer_logs):
8887
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
89-
"datad0g.com", "", 1, 1, is_agentless=True, _agentless_url=AGENTLESS_URL
88+
interval=1, timeout=1, site=DD_SITE, api_key="", is_agentless=True
9089
)
9190
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
9291
llmobs_eval_metric_writer.periodic()
@@ -99,7 +98,7 @@ def test_send_metric_no_api_key(mock_writer_logs):
9998
@pytest.mark.vcr_logs
10099
def test_send_categorical_metric(mock_writer_logs):
101100
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
102-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
101+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
103102
)
104103
llmobs_eval_metric_writer.enqueue(_categorical_metric_event())
105104
llmobs_eval_metric_writer.periodic()
@@ -111,7 +110,7 @@ def test_send_categorical_metric(mock_writer_logs):
111110
@pytest.mark.vcr_logs
112111
def test_send_score_metric(mock_writer_logs):
113112
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
114-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
113+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
115114
)
116115
llmobs_eval_metric_writer.enqueue(_score_metric_event())
117116
llmobs_eval_metric_writer.periodic()
@@ -123,12 +122,7 @@ def test_send_score_metric(mock_writer_logs):
123122
@pytest.mark.vcr_logs
124123
def test_send_timed_events(mock_writer_logs):
125124
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
126-
site="datad0g.com",
127-
api_key=DD_API_KEY,
128-
interval=0.01,
129-
timeout=1,
130-
is_agentless=True,
131-
_agentless_url=AGENTLESS_URL,
125+
interval=0.01, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
132126
)
133127
llmobs_eval_metric_writer.start()
134128
mock_writer_logs.reset_mock()
@@ -150,7 +144,7 @@ def test_send_timed_events(mock_writer_logs):
150144
@pytest.mark.vcr_logs
151145
def test_send_multiple_events(mock_writer_logs):
152146
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
153-
site="datad0g.com", api_key=DD_API_KEY, interval=1, timeout=1, is_agentless=True, _agentless_url=AGENTLESS_URL
147+
interval=1, timeout=1, site=DD_SITE, api_key=DD_API_KEY, is_agentless=True
154148
)
155149
mock_writer_logs.reset_mock()
156150
llmobs_eval_metric_writer.enqueue(_score_metric_event())
@@ -164,19 +158,10 @@ def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess):
164158
pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
165159
if "PYTHONPATH" in env:
166160
pypath.append(env["PYTHONPATH"])
167-
env.update(
168-
{
169-
"DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"),
170-
"DD_SITE": "datad0g.com",
171-
"PYTHONPATH": ":".join(pypath),
172-
"DD_LLMOBS_ML_APP": "unnamed-ml-app",
173-
}
174-
)
161+
env.update({"PYTHONPATH": ":".join(pypath), "DD_LLMOBS_ML_APP": "unnamed-ml-app"})
175162
out, err, status, pid = run_python_code_in_subprocess(
176163
"""
177164
import atexit
178-
import os
179-
import time
180165
181166
from ddtrace.llmobs._writer import LLMObsEvalMetricWriter
182167
from tests.llmobs.test_llmobs_eval_metric_writer import _score_metric_event
@@ -186,7 +171,7 @@ def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess):
186171
ctx.__enter__()
187172
atexit.register(lambda: ctx.__exit__())
188173
llmobs_eval_metric_writer = LLMObsEvalMetricWriter(
189-
site="datad0g.com", api_key=os.getenv("DD_API_KEY"), interval=0.01, timeout=1
174+
interval=0.01, timeout=1, site="datad0g.com", api_key="<not-a-real-key>", is_agentless=False
190175
)
191176
llmobs_eval_metric_writer.start()
192177
llmobs_eval_metric_writer.enqueue(_score_metric_event())

‎tests/llmobs/test_llmobs_evaluator_runner.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ def test_evaluator_runner_buffer_limit(mock_evaluator_logs):
4141
)
4242

4343

44+
@pytest.mark.vcr_logs
4445
def test_evaluator_runner_periodic_enqueues_eval_metric(mock_llmobs_eval_metric_writer, active_evaluator_runner):
4546
active_evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN)
4647
active_evaluator_runner.periodic()
@@ -57,6 +58,7 @@ def test_evaluator_runner_stopped_does_not_enqueue_metric(llmobs, mock_llmobs_ev
5758
assert mock_llmobs_eval_metric_writer.enqueue.call_count == 0
5859

5960

61+
@pytest.mark.vcr_logs
6062
def test_evaluator_runner_timed_enqueues_eval_metric(llmobs, mock_llmobs_eval_metric_writer, active_evaluator_runner):
6163
active_evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN)
6264

@@ -95,7 +97,9 @@ def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subproces
9597
pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
9698
if "PYTHONPATH" in env:
9799
pypath.append(env["PYTHONPATH"])
98-
env.update({"PYTHONPATH": ":".join(pypath), "_DD_LLMOBS_EVALUATOR_INTERVAL": "5"})
100+
env.update(
101+
{"PYTHONPATH": ":".join(pypath), "_DD_LLMOBS_EVALUATOR_INTERVAL": "0.01", "_DD_LLMOBS_WRITER_INTERVAL": "0.01"}
102+
)
99103
out, err, status, pid = run_python_code_in_subprocess(
100104
"""
101105
from ddtrace.llmobs import LLMObs

0 commit comments

Comments
 (0)