From 8e4b9f89b02d015d36a0e136e21e5031fff966a3 Mon Sep 17 00:00:00 2001 From: Nathan Lee Date: Fri, 18 Oct 2024 18:42:25 +0100 Subject: [PATCH 1/6] fix(integrations): Add partial json support to streams Adds partial json to the content block when streaming. --- sentry_sdk/integrations/anthropic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 87e69a3113..048cac4c88 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -21,8 +21,7 @@ raise DidNotEnable("Anthropic not installed") if TYPE_CHECKING: - from typing import Any, AsyncIterator, Iterator - from sentry_sdk.tracing import Span + pass class AnthropicIntegration(Integration): @@ -106,6 +105,8 @@ def _collect_ai_data(event, input_tokens, output_tokens, content_blocks): elif event.type == "content_block_delta": if hasattr(event.delta, "text"): content_blocks.append(event.delta.text) + elif hasattr(event.delta, "partial_json"): + content_blocks.append(event.delta.partial_json) elif event.type == "content_block_stop": pass elif event.type == "message_delta": From 576bb7dd8fd66d0d20af559b771c3c1a4f934a1b Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 25 Oct 2024 09:13:49 +0000 Subject: [PATCH 2/6] commit back type checks --- sentry_sdk/integrations/anthropic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 048cac4c88..14ec39f8b2 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -21,7 +21,8 @@ raise DidNotEnable("Anthropic not installed") if TYPE_CHECKING: - pass + from typing import Any, AsyncIterator, Iterator + from sentry_sdk.tracing import Span class AnthropicIntegration(Integration): From 8cc1ef36580758d47427b8fadf3992210b2762c4 Mon Sep 17 00:00:00 2001 From: Nathan Lee Date: Sun, 27 Oct 2024 20:15:10 +0000 Subject: [PATCH 3/6] add more tests around collecting ai data --- .../integrations/anthropic/test_anthropic.py | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 8ce12e70f5..280bee6896 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -10,7 +10,7 @@ async def __call__(self, *args, **kwargs): import pytest -from anthropic import AsyncAnthropic, Anthropic, AnthropicError, AsyncStream, Stream +from anthropic import Anthropic, AnthropicError, AsyncAnthropic, AsyncStream, Stream from anthropic.types import MessageDeltaUsage, TextDelta, Usage from anthropic.types.content_block_delta_event import ContentBlockDeltaEvent from anthropic.types.content_block_start_event import ContentBlockStartEvent @@ -19,6 +19,7 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_delta_event import MessageDeltaEvent from anthropic.types.message_start_event import MessageStartEvent +from sentry_sdk.integrations.anthropic import _add_ai_data_to_span, _collect_ai_data from sentry_sdk.utils import package_version try: @@ -757,3 +758,59 @@ async def test_span_origin_async(sentry_init, capture_events): assert event["contexts"]["trace"]["origin"] == "manual" assert event["spans"][0]["origin"] == "auto.ai.anthropic" + + +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="Versions <0.27.0 do not include InputJSONDelta.", +) +def test_collect_ai_data_with_input_json_delta(): + event = ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="test", type="input_json_delta"), + index=0, + type="content_block_delta", + ) + + input_tokens = 10 + output_tokens = 20 + content_blocks = [] + + new_input_tokens, new_output_tokens, new_content_blocks = _collect_ai_data( + event, input_tokens, output_tokens, content_blocks + ) + + assert new_input_tokens == input_tokens + assert new_output_tokens == output_tokens + assert new_content_blocks == ["test"] + + +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="Versions <0.27.0 do not include InputJSONDelta.", +) +def test_add_ai_data_to_span_with_input_json_delta(sentry_init): + sentry_init( + integrations=[AnthropicIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + with start_transaction(name="test") as transaction: + span = transaction.start_span() + integration = AnthropicIntegration() + + _add_ai_data_to_span( + span, + integration, + input_tokens=10, + output_tokens=20, + content_blocks=["{'test': 'data'}", "'more': 'json'"], + ) + + assert span.get_data(SPANDATA.AI_RESPONSES) == [ + {"type": "text", "text": "{'test': 'data''more': 'json'}"} + ] + assert span.get_data("ai.streaming") is True + assert span.get_measurement("ai_prompt_tokens_used")["value"] == 10 + assert span.get_measurement("ai_completion_tokens_used")["value"] == 20 + assert span.get_measurement("ai_total_tokens_used")["value"] == 30 From 1b6f72c6ee5993b59a28f9389106eda3ad7a90df Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 15 Nov 2024 12:43:48 +0000 Subject: [PATCH 4/6] fix: add tests for unsupported versions of anthropic --- .../integrations/anthropic/test_anthropic.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 280bee6896..b1c2bdca43 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,5 +1,7 @@ from unittest import mock +from sentry_sdk.integrations import DidNotEnable + try: from unittest.mock import AsyncMock except ImportError: @@ -518,9 +520,8 @@ def test_streaming_create_message_with_input_json_delta( if send_default_pii and include_prompts: assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"text": "", "type": "text"} - ] # we do not record InputJSONDelta because it could contain PII - + {"text": "{'location': 'San Francisco, CA'}", "type": "text"} + ] else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] assert SPANDATA.AI_RESPONSES not in span["data"] @@ -814,3 +815,22 @@ def test_add_ai_data_to_span_with_input_json_delta(sentry_init): assert span.get_measurement("ai_prompt_tokens_used")["value"] == 10 assert span.get_measurement("ai_completion_tokens_used")["value"] == 20 assert span.get_measurement("ai_total_tokens_used")["value"] == 30 + + +def test_unsupported_anthropic_version(sentry_init): + with mock.patch( + "sentry_sdk.integrations.anthropic.package_version", return_value=(0, 15, 0) + ): + with pytest.raises(DidNotEnable): + sentry_init( + integrations=[AnthropicIntegration()], + traces_sample_rate=1.0, + ) + + +def test_no_version_info(sentry_init): + with mock.patch( + "sentry_sdk.integrations.anthropic.package_version", return_value=None + ): + with pytest.raises(DidNotEnable): + sentry_init(integrations=[AnthropicIntegration()]) From edaa5d30d4f2c5ce8a5390833b7c25215e88d651 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 20 Feb 2025 16:30:39 +0100 Subject: [PATCH 5/6] Fixed tests --- .../integrations/anthropic/test_anthropic.py | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index b1c2bdca43..8d3bb856d9 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -45,7 +45,7 @@ async def __call__(self, *args, **kwargs): except ImportError: from anthropic.types.content_block import ContentBlock as TextBlock -from sentry_sdk import start_transaction +from sentry_sdk import start_transaction, start_span from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.anthropic import AnthropicIntegration @@ -656,8 +656,8 @@ async def test_streaming_create_message_with_input_json_delta_async( if send_default_pii and include_prompts: assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"text": "", "type": "text"} - ] # we do not record InputJSONDelta because it could contain PII + {"text": "{'location': 'San Francisco, CA'}", "type": "text"} + ] else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] @@ -796,8 +796,8 @@ def test_add_ai_data_to_span_with_input_json_delta(sentry_init): send_default_pii=True, ) - with start_transaction(name="test") as transaction: - span = transaction.start_span() + with start_transaction(name="test"): + span = start_span() integration = AnthropicIntegration() _add_ai_data_to_span( @@ -805,32 +805,13 @@ def test_add_ai_data_to_span_with_input_json_delta(sentry_init): integration, input_tokens=10, output_tokens=20, - content_blocks=["{'test': 'data'}", "'more': 'json'"], + content_blocks=["{'test': 'data',", "'more': 'json'}"], ) - assert span.get_data(SPANDATA.AI_RESPONSES) == [ - {"type": "text", "text": "{'test': 'data''more': 'json'}"} + assert span._data.get(SPANDATA.AI_RESPONSES) == [ + {"type": "text", "text": "{'test': 'data','more': 'json'}"} ] - assert span.get_data("ai.streaming") is True - assert span.get_measurement("ai_prompt_tokens_used")["value"] == 10 - assert span.get_measurement("ai_completion_tokens_used")["value"] == 20 - assert span.get_measurement("ai_total_tokens_used")["value"] == 30 - - -def test_unsupported_anthropic_version(sentry_init): - with mock.patch( - "sentry_sdk.integrations.anthropic.package_version", return_value=(0, 15, 0) - ): - with pytest.raises(DidNotEnable): - sentry_init( - integrations=[AnthropicIntegration()], - traces_sample_rate=1.0, - ) - - -def test_no_version_info(sentry_init): - with mock.patch( - "sentry_sdk.integrations.anthropic.package_version", return_value=None - ): - with pytest.raises(DidNotEnable): - sentry_init(integrations=[AnthropicIntegration()]) + assert span._data.get("ai.streaming") is True + assert span._measurements.get("ai_prompt_tokens_used")["value"] == 10 + assert span._measurements.get("ai_completion_tokens_used")["value"] == 20 + assert span._measurements.get("ai_total_tokens_used")["value"] == 30 From 018cb057eb5b4ceeddc6f12a7abbd81039642224 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 20 Feb 2025 16:34:48 +0100 Subject: [PATCH 6/6] Make linter happy] --- tests/integrations/anthropic/test_anthropic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 8d3bb856d9..7f6622a1ba 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,6 +1,5 @@ from unittest import mock -from sentry_sdk.integrations import DidNotEnable try: from unittest.mock import AsyncMock