From 29438900678ce76cda72636a52bdddb942c061eb Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Thu, 30 Jan 2025 15:57:31 -0500 Subject: [PATCH 1/5] Correctly parse env variable as it can't be a set --- ddtrace/contrib/internal/botocore/patch.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ddtrace/contrib/internal/botocore/patch.py b/ddtrace/contrib/internal/botocore/patch.py index 61353e7b34e..bd1f815032a 100644 --- a/ddtrace/contrib/internal/botocore/patch.py +++ b/ddtrace/contrib/internal/botocore/patch.py @@ -113,7 +113,10 @@ def _load_dynamodb_primary_key_names_for_tables() -> Dict[str, Set[str]]: os.getenv("DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_TAGS", 758) ), # RFC defined default limit - spans are limited past 1000 "payload_tagging_services": set( - os.getenv("DD_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES", default={"s3", "sns", "sqs", "kinesis", "eventbridge"}) + service.strip() + for service in os.getenv("DD_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES", "s3,sns,sqs,kinesis,eventbridge").split( + "," + ) ), }, ) From 7a2c74a744560fb321488087d3b92dc8b4a219d9 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Thu, 6 Feb 2025 23:49:27 -0500 Subject: [PATCH 2/5] add changelog --- ...ectly-parse-payload_tagging_services-794cdf0f340975e6.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/correctly-parse-payload_tagging_services-794cdf0f340975e6.yaml diff --git a/releasenotes/notes/correctly-parse-payload_tagging_services-794cdf0f340975e6.yaml b/releasenotes/notes/correctly-parse-payload_tagging_services-794cdf0f340975e6.yaml new file mode 100644 index 00000000000..b6e4c5758cd --- /dev/null +++ b/releasenotes/notes/correctly-parse-payload_tagging_services-794cdf0f340975e6.yaml @@ -0,0 +1,4 @@ +fixes: + - | + configurations: This fix resolves an issue where DD_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES env variable was not parsed correctly + From cf7882086aca7c982333e76240064efd17bce408 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 7 Feb 2025 16:27:39 -0500 Subject: [PATCH 3/5] Add DynamoDB payload tagging test --- tests/contrib/botocore/test.py | 50 ++++ ...oreTest.test_dynamodb_payload_tagging.json | 275 ++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 tests/snapshots/tests.contrib.botocore.test.BotocoreTest.test_dynamodb_payload_tagging.json diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index cb1a06bec1c..8736e0e5c4d 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -4093,6 +4093,56 @@ def test_aws_payload_tagging_s3_invalid_config(self): with pytest.raises(Exception): s3.list_objects(bucket="mybucket") + @pytest.mark.snapshot(ignores=snapshot_ignores) + @pytest.mark.skipif( + PYTHON_VERSION_INFO < (3, 8), + reason="Skipping for older py versions whose latest supported moto versions don't have the right dynamodb api", + ) + @mock_dynamodb + def test_dynamodb_payload_tagging(self): + with self.override_config( + "botocore", dict(payload_tagging_request="all", payload_tagging_response="all", payload_tagging_services="s3,dynamodb") + ): + ddb = self.session.create_client("dynamodb", region_name="us-west-2") + pin = Pin(service=self.TEST_SERVICE) + pin._tracer = self.tracer + pin.onto(ddb) + + with self.override_config("botocore", dict(instrument_internals=True)): + ddb.create_table( + TableName="foobar", + AttributeDefinitions=[{"AttributeName": "myattr", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "myattr", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + ddb.put_item(TableName="foobar", Item={"myattr": {"S": "baz"}}) + ddb.get_item(TableName="foobar", Key={"myattr": {"S": "baz"}}) + + spans = self.get_spans() + assert spans + span = spans[0] + assert len(spans) == 6 + assert_is_measured(span) + assert span.get_tag("aws.operation") == "CreateTable" + assert span.get_tag("component") == "botocore" + assert span.get_tag("span.kind"), "client" + assert_span_http_status_code(span, 200) + assert span.service == "test-botocore-tracing.dynamodb" + assert span.resource == "dynamodb.createtable" + + span = spans[1] + assert span.name == "botocore.parsers.parse" + assert span.get_tag("component") == "botocore" + assert span.get_tag("span.kind"), "client" + assert span.service == "test-botocore-tracing.dynamodb" + assert span.resource == "botocore.parsers.parse" + + span = spans[2] + assert span.get_tag("aws.operation") == "PutItem" + # Since the dynamodb_primary_key_names_for_tables isn't configured, we + # cannot create span pointers for this item. + assert not span._links + @pytest.mark.skip(reason="broken during period of skipping on main branch") @pytest.mark.snapshot(ignores=snapshot_ignores) @mock_s3 diff --git a/tests/snapshots/tests.contrib.botocore.test.BotocoreTest.test_dynamodb_payload_tagging.json b/tests/snapshots/tests.contrib.botocore.test.BotocoreTest.test_dynamodb_payload_tagging.json new file mode 100644 index 00000000000..69c87ff588e --- /dev/null +++ b/tests/snapshots/tests.contrib.botocore.test.BotocoreTest.test_dynamodb_payload_tagging.json @@ -0,0 +1,275 @@ +[[ + { + "name": "dynamodb.command", + "service": "test-botocore-tracing.dynamodb", + "resource": "dynamodb.getitem", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "http", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "67a6776f00000000", + "aws.agent": "botocore", + "aws.dynamodb.table_name": "foobar", + "aws.operation": "GetItem", + "aws.region": "us-west-2", + "aws.request.body.Key.myattr.S": "baz", + "aws.request.body.TableName": "foobar", + "aws.requestid": "1pGBOmkOZ1BdOWIHeWnkX96CO0n93XaZSqtfcgRZJhe6hrREgPCs", + "aws.response.body.HTTPHeaders.date": "Fri, 07 Feb 2025 21:13:20 GMT", + "aws.response.body.HTTPHeaders.server": "amazon.com", + "aws.response.body.HTTPHeaders.x-amz-crc32": "357125523", + "aws.response.body.HTTPHeaders.x-amzn-requestid": "1pGBOmkOZ1BdOWIHeWnkX96CO0n93XaZSqtfcgRZJhe6hrREgPCs", + "aws.response.body.HTTPStatusCode": "200", + "aws.response.body.RequestId": "1pGBOmkOZ1BdOWIHeWnkX96CO0n93XaZSqtfcgRZJhe6hrREgPCs", + "aws.response.body.RetryAttempts": "0", + "aws_service": "dynamodb", + "component": "botocore", + "http.status_code": "200", + "language": "python", + "region": "us-west-2", + "runtime-id": "ef4281e602a34aa3b78873db2aa0da91", + "span.kind": "client", + "tablename": "foobar" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 7842, + "retry_attempts": 0 + }, + "duration": 898096696, + "start": 1738962799557241364 + }, + { + "name": "botocore.parsers.parse", + "service": "test-botocore-tracing.dynamodb", + "resource": "botocore.parsers.parse", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "component": "botocore", + "span.kind": "client" + }, + "duration": 328440, + "start": 1738962800007520310 + }], +[ + { + "name": "dynamodb.command", + "service": "test-botocore-tracing.dynamodb", + "resource": "dynamodb.createtable", + "trace_id": 1, + "span_id": 1, + "parent_id": 0, + "type": "http", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "67a6776d00000000", + "aws.agent": "botocore", + "aws.dynamodb.table_name": "foobar", + "aws.operation": "CreateTable", + "aws.region": "us-west-2", + "aws.request.body.AttributeDefinitions.0.AttributeName": "myattr", + "aws.request.body.AttributeDefinitions.0.AttributeType": "S", + "aws.request.body.BillingMode": "PAY_PER_REQUEST", + "aws.request.body.KeySchema.0.AttributeName": "myattr", + "aws.request.body.KeySchema.0.KeyType": "HASH", + "aws.request.body.TableName": "foobar", + "aws.requestid": "s5qfbhhfkr4Go9yZWDL5eSDNflqO9AY7R1PtdbeHgZSnABZRXWwc", + "aws.response.body.HTTPHeaders.date": "Fri, 07 Feb 2025 21:13:17 GMT", + "aws.response.body.HTTPHeaders.server": "amazon.com", + "aws.response.body.HTTPHeaders.x-amz-crc32": "725802381", + "aws.response.body.HTTPHeaders.x-amzn-requestid": "s5qfbhhfkr4Go9yZWDL5eSDNflqO9AY7R1PtdbeHgZSnABZRXWwc", + "aws.response.body.HTTPStatusCode": "200", + "aws.response.body.RequestId": "s5qfbhhfkr4Go9yZWDL5eSDNflqO9AY7R1PtdbeHgZSnABZRXWwc", + "aws.response.body.RetryAttempts": "0", + "aws_service": "dynamodb", + "component": "botocore", + "http.status_code": "200", + "language": "python", + "region": "us-west-2", + "runtime-id": "ef4281e602a34aa3b78873db2aa0da91", + "span.kind": "client", + "tablename": "foobar" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 7842, + "retry_attempts": 0 + }, + "duration": 1169517214, + "start": 1738962797488686721 + }, + { + "name": "botocore.parsers.parse", + "service": "test-botocore-tracing.dynamodb", + "resource": "botocore.parsers.parse", + "trace_id": 1, + "span_id": 2, + "parent_id": 1, + "type": "", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "component": "botocore", + "span.kind": "client" + }, + "duration": 680298, + "start": 1738962798196385705 + }], +[ + { + "name": "dynamodb.command", + "service": "test-botocore-tracing.dynamodb", + "resource": "dynamodb.putitem", + "trace_id": 2, + "span_id": 1, + "parent_id": 0, + "type": "http", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "67a6776e00000000", + "aws.agent": "botocore", + "aws.dynamodb.table_name": "foobar", + "aws.operation": "PutItem", + "aws.region": "us-west-2", + "aws.request.body.Item.myattr.S": "baz", + "aws.request.body.TableName": "foobar", + "aws.requestid": "l8KoXDDLWxK9lyqrrMRmtbdenA29koyXlCZEOlqgXgh46RkAzTiV", + "aws.response.body.HTTPHeaders.date": "Fri, 07 Feb 2025 21:13:19 GMT", + "aws.response.body.HTTPHeaders.server": "amazon.com", + "aws.response.body.HTTPHeaders.x-amz-crc32": "2745614147", + "aws.response.body.HTTPHeaders.x-amzn-requestid": "l8KoXDDLWxK9lyqrrMRmtbdenA29koyXlCZEOlqgXgh46RkAzTiV", + "aws.response.body.HTTPStatusCode": "200", + "aws.response.body.RequestId": "l8KoXDDLWxK9lyqrrMRmtbdenA29koyXlCZEOlqgXgh46RkAzTiV", + "aws.response.body.RetryAttempts": "0", + "aws_service": "dynamodb", + "component": "botocore", + "http.status_code": "200", + "language": "python", + "region": "us-west-2", + "runtime-id": "ef4281e602a34aa3b78873db2aa0da91", + "span.kind": "client", + "tablename": "foobar" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 7842, + "retry_attempts": 0 + }, + "duration": 897531031, + "start": 1738962798659199416 + }, + { + "name": "botocore.parsers.parse", + "service": "test-botocore-tracing.dynamodb", + "resource": "botocore.parsers.parse", + "trace_id": 2, + "span_id": 2, + "parent_id": 1, + "type": "", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "component": "botocore", + "span.kind": "client" + }, + "duration": 188248, + "start": 1738962799115125322 + }], +[ + { + "name": "sqs.command", + "service": "aws.sqs", + "resource": "sqs.listqueues", + "trace_id": 3, + "span_id": 1, + "parent_id": 0, + "type": "http", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "67a6776d00000000", + "aws.agent": "botocore", + "aws.operation": "ListQueues", + "aws.region": "us-east-1", + "aws.requestid": "8BQ748A97HZTB3DQNICVX52K4H4KRCK9BWKBM3QX1FE0F9DJ4481", + "aws_service": "sqs", + "component": "botocore", + "http.status_code": "200", + "language": "python", + "region": "us-east-1", + "runtime-id": "ef4281e602a34aa3b78873db2aa0da91", + "span.kind": "client" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 7842, + "retry_attempts": 0 + }, + "duration": 17043485, + "start": 1738962797437241544 + }], +[ + { + "name": "sqs.command", + "service": "aws.sqs", + "resource": "sqs.createqueue", + "trace_id": 4, + "span_id": 1, + "parent_id": 0, + "type": "http", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "67a6776d00000000", + "aws.agent": "botocore", + "aws.operation": "CreateQueue", + "aws.region": "us-east-1", + "aws.requestid": "P5XR375SOHFTGCNG78FP0M00O7J8TR5Y01KWQXDNSAC2MR17BVSN", + "aws.sqs.queue_name": "Test", + "aws_service": "sqs", + "component": "botocore", + "http.status_code": "200", + "language": "python", + "queuename": "Test", + "region": "us-east-1", + "runtime-id": "ef4281e602a34aa3b78873db2aa0da91", + "span.kind": "client" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 7842, + "retry_attempts": 0 + }, + "duration": 9204293, + "start": 1738962797455089485 + }]] From 1b10e6f44e49844c774a95173d638fc2d39b0f15 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 7 Feb 2025 16:31:45 -0500 Subject: [PATCH 4/5] lint --- tests/contrib/botocore/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index 8736e0e5c4d..eca45cdc510 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -4101,7 +4101,8 @@ def test_aws_payload_tagging_s3_invalid_config(self): @mock_dynamodb def test_dynamodb_payload_tagging(self): with self.override_config( - "botocore", dict(payload_tagging_request="all", payload_tagging_response="all", payload_tagging_services="s3,dynamodb") + "botocore", + dict(payload_tagging_request="all", payload_tagging_response="all", payload_tagging_services="s3,dynamodb"), ): ddb = self.session.create_client("dynamodb", region_name="us-west-2") pin = Pin(service=self.TEST_SERVICE) From bf0dcb6593b80dd1c786be40b7769963ca220b44 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 7 Feb 2025 17:34:38 -0500 Subject: [PATCH 5/5] Ignore crc32 header in botocore tests --- tests/contrib/botocore/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index eca45cdc510..3375a21c407 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -64,6 +64,7 @@ "meta.aws.response.body.HTTPHeaders.content-length", "meta.aws.response.body.HTTPHeaders.x-amzn-requestid", "meta.error.stack", + "meta.aws.response.body.HTTPHeaders.x-amz-crc32", ]