diff --git a/.riot/requirements/1761702.txt b/.riot/requirements/1761702.txt deleted file mode 100644 index e6ca01ea565..00000000000 --- a/.riot/requirements/1761702.txt +++ /dev/null @@ -1,23 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1761702.in -# -attrs==23.1.0 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -hypothesis==6.45.0 -iniconfig==2.0.0 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -yaaredis==3.0.0 diff --git a/.riot/requirements/1cc7b0e.txt b/.riot/requirements/1cc7b0e.txt deleted file mode 100644 index adb8f71e30b..00000000000 --- a/.riot/requirements/1cc7b0e.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1cc7b0e.in -# -attrs==23.1.0 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -hypothesis==6.45.0 -importlib-metadata==7.0.0 -iniconfig==2.0.0 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -yaaredis==2.0.4 -zipp==3.17.0 diff --git a/.riot/requirements/1f1e9b4.txt b/.riot/requirements/1f1e9b4.txt deleted file mode 100644 index d9b9189f22a..00000000000 --- a/.riot/requirements/1f1e9b4.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1f1e9b4.in -# -attrs==23.1.0 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -hypothesis==6.45.0 -importlib-metadata==7.0.0 -iniconfig==2.0.0 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -yaaredis==3.0.0 -zipp==3.17.0 diff --git a/.riot/requirements/9b8251b.txt b/.riot/requirements/9b8251b.txt deleted file mode 100644 index c6c4004b105..00000000000 --- a/.riot/requirements/9b8251b.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/9b8251b.in -# -attrs==23.1.0 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -hypothesis==6.45.0 -importlib-metadata==7.0.0 -iniconfig==2.0.0 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -yaaredis==3.0.0 -zipp==3.17.0 diff --git a/.riot/requirements/fda8aa6.txt b/.riot/requirements/fda8aa6.txt deleted file mode 100644 index efa619edb1e..00000000000 --- a/.riot/requirements/fda8aa6.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/fda8aa6.in -# -attrs==23.1.0 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -hypothesis==6.45.0 -importlib-metadata==7.0.0 -iniconfig==2.0.0 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -pytest==7.4.3 -pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -yaaredis==2.0.4 -zipp==3.17.0 diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py index f7cd86b416e..8ccfa55edfd 100644 --- a/ddtrace/_monkey.py +++ b/ddtrace/_monkey.py @@ -95,7 +95,6 @@ "pyodbc": True, "fastapi": True, "dogpile_cache": True, - "yaaredis": True, "asyncpg": True, "aws_lambda": True, # patch only in AWS Lambda environments "azure_functions": True, diff --git a/ddtrace/appsec/__init__.py b/ddtrace/appsec/__init__.py index 05d1a852710..6b5758a95c2 100644 --- a/ddtrace/appsec/__init__.py +++ b/ddtrace/appsec/__init__.py @@ -1,5 +1,6 @@ +# this module must not load any other unsafe appsec module directly + from ddtrace.internal import core -from ddtrace.settings.asm import config as asm_config _APPSEC_TO_BE_LOADED = True @@ -28,7 +29,9 @@ def load_iast(): def load_common_appsec_modules(): """Lazily load the common module patches.""" - if (asm_config._ep_enabled and asm_config._asm_enabled) or asm_config._iast_enabled: + from ddtrace.settings.asm import config as asm_config + + if asm_config._load_modules: from ddtrace.appsec._common_module_patches import patch_common_modules patch_common_modules() diff --git a/ddtrace/appsec/_asm_request_context.py b/ddtrace/appsec/_asm_request_context.py index d8f258d43a7..bd6c8b21a2a 100644 --- a/ddtrace/appsec/_asm_request_context.py +++ b/ddtrace/appsec/_asm_request_context.py @@ -15,9 +15,6 @@ from ddtrace.appsec._constants import APPSEC from ddtrace.appsec._constants import EXPLOIT_PREVENTION from ddtrace.appsec._constants import SPAN_DATA_NAMES -from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled -from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._utils import add_context_log from ddtrace.appsec._utils import get_triggers from ddtrace.internal import core @@ -28,6 +25,16 @@ from ddtrace.trace import Span +if asm_config._iast_enabled: + from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled + from ddtrace.appsec._iast._taint_tracking import OriginType + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject +else: + + def is_iast_request_enabled() -> bool: + return False + + if TYPE_CHECKING: from ddtrace.appsec._ddwaf import DDWaf_info from ddtrace.appsec._ddwaf import DDWaf_result diff --git a/ddtrace/appsec/_common_module_patches.py b/ddtrace/appsec/_common_module_patches.py index 8c834b80e6f..ac3c2c4e775 100644 --- a/ddtrace/appsec/_common_module_patches.py +++ b/ddtrace/appsec/_common_module_patches.py @@ -44,6 +44,10 @@ def is_iast_request_enabled() -> bool: def patch_common_modules(): global _is_patched + # ensure that the subprocess patch is applied even after one click activation + subprocess_patch.patch() + subprocess_patch.add_str_callback(_RASP_SYSTEM, wrapped_system_5542593D237084A7) + subprocess_patch.add_lst_callback(_RASP_POPEN, popen_FD233052260D8B4D) if _is_patched: return # for testing purposes, we need to update is_iast_request_enabled @@ -60,10 +64,6 @@ def is_iast_request_enabled() -> bool: try_wrap_function_wrapper("urllib.request", "OpenerDirector.open", wrapped_open_ED4CF71136E15EBF) try_wrap_function_wrapper("_io", "BytesIO.read", wrapped_read_F3E51D71B4EC16EF) try_wrap_function_wrapper("_io", "StringIO.read", wrapped_read_F3E51D71B4EC16EF) - # ensure that the subprocess patch is applied even after one click activation - subprocess_patch.patch() - subprocess_patch.add_str_callback(_RASP_SYSTEM, wrapped_system_5542593D237084A7) - subprocess_patch.add_lst_callback(_RASP_POPEN, popen_FD233052260D8B4D) core.on("asm.block.dbapi.execute", execute_4C9BAC8E228EB347) if asm_config._iast_enabled: from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink diff --git a/ddtrace/appsec/_constants.py b/ddtrace/appsec/_constants.py index 2172548205b..454483fcf17 100644 --- a/ddtrace/appsec/_constants.py +++ b/ddtrace/appsec/_constants.py @@ -1,3 +1,5 @@ +# this module must not load any other unsafe appsec module directly + import os from re import Match import sys diff --git a/ddtrace/appsec/_iast/taint_sinks/xss.py b/ddtrace/appsec/_iast/taint_sinks/xss.py index 6f7d263f7c2..73350faac44 100644 --- a/ddtrace/appsec/_iast/taint_sinks/xss.py +++ b/ddtrace/appsec/_iast/taint_sinks/xss.py @@ -64,7 +64,15 @@ def patch(): ) _set_metric_iast_instrumented_sink(VULN_XSS) - _set_metric_iast_instrumented_sink(VULN_XSS) + # Even when starting the application with `ddtrace-run ddtrace-run`, `jinja2.FILTERS` is created before this patch + # function executes. Therefore, we update the in-memory object with the newly patched version. + try: + from jinja2.filters import FILTERS + from jinja2.filters import do_mark_safe + + FILTERS["safe"] = do_mark_safe + except (ImportError, KeyError): + pass def unpatch(): diff --git a/ddtrace/appsec/_utils.py b/ddtrace/appsec/_utils.py index 79f8f8b5311..e4dbae7a27f 100644 --- a/ddtrace/appsec/_utils.py +++ b/ddtrace/appsec/_utils.py @@ -1,3 +1,5 @@ +# this module must not load any other unsafe appsec module directly + import logging import sys from typing import Any @@ -5,6 +7,7 @@ from ddtrace.appsec._constants import API_SECURITY from ddtrace.appsec._constants import APPSEC +from ddtrace.appsec._constants import SPAN_DATA_NAMES from ddtrace.internal._unpatched import unpatched_json_loads from ddtrace.internal.compat import to_unicode from ddtrace.internal.logger import get_logger @@ -21,7 +24,6 @@ def parse_response_body(raw_body): import xmltodict from ddtrace.appsec import _asm_request_context - from ddtrace.appsec._constants import SPAN_DATA_NAMES from ddtrace.contrib.internal.trace_utils import _get_header_value_case_insensitive if not raw_body: diff --git a/ddtrace/contrib/_yaaredis.py b/ddtrace/contrib/_yaaredis.py deleted file mode 100644 index 65917b03c29..00000000000 --- a/ddtrace/contrib/_yaaredis.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -The yaaredis integration traces yaaredis requests. - - -Enabling -~~~~~~~~ - -The yaaredis integration is enabled automatically when using -:ref:`ddtrace-run` or :ref:`import ddtrace.auto`. - -Or use :func:`patch()` to manually enable the integration:: - - from ddtrace import patch - patch(yaaredis=True) - - -Global Configuration -~~~~~~~~~~~~~~~~~~~~ - -.. py:data:: ddtrace.config.yaaredis["service"] - - The service name reported by default for yaaredis traces. - - This option can also be set with the ``DD_YAAREDIS_SERVICE`` environment - variable. - - Default: ``"redis"`` - -.. py:data:: ddtrace.config.yaaredis["cmd_max_length"] - - Max allowable size for the yaaredis command span tag. - Anything beyond the max length will be replaced with ``"..."``. - - This option can also be set with the ``DD_YAAREDIS_CMD_MAX_LENGTH`` environment - variable. - - Default: ``1000`` - -.. py:data:: ddtrace.config.aredis["resource_only_command"] - - The span resource will only include the command executed. To include all - arguments in the span resource, set this value to ``False``. - - This option can also be set with the ``DD_REDIS_RESOURCE_ONLY_COMMAND`` environment - variable. - - Default: ``True`` - - -Instance Configuration -~~~~~~~~~~~~~~~~~~~~~~ - -To configure particular yaaredis instances use the :class:`Pin ` API:: - - import yaaredis - from ddtrace.trace import Pin - - client = yaaredis.StrictRedis(host="localhost", port=6379) - - # Override service name for this instance - Pin.override(client, service="my-custom-queue") - - # Traces reported for this client will now have "my-custom-queue" - # as the service name. - async def example(): - await client.get("my-key") -""" diff --git a/ddtrace/contrib/internal/httplib/patch.py b/ddtrace/contrib/internal/httplib/patch.py index 79a8ea2816f..7db5d59d31c 100644 --- a/ddtrace/contrib/internal/httplib/patch.py +++ b/ddtrace/contrib/internal/httplib/patch.py @@ -5,7 +5,6 @@ import wrapt from ddtrace import config -from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request_asm from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND from ddtrace.contrib import trace_utils @@ -77,12 +76,14 @@ def _wrap_getresponse(func, instance, args, kwargs): def _call_asm_wrap(func, instance, *args, **kwargs): + from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request_asm + _wrap_request_asm(func, instance, args, kwargs) def _wrap_request(func, instance, args, kwargs): # Use any attached tracer if available, otherwise use the global tracer - if asm_config._iast_enabled or asm_config._asm_enabled: + if asm_config._iast_enabled or (asm_config._asm_enabled and asm_config._ep_enabled): func_to_call = functools.partial(_call_asm_wrap, func, instance) else: func_to_call = func diff --git a/ddtrace/contrib/internal/langchain/patch.py b/ddtrace/contrib/internal/langchain/patch.py index 58c635dc46f..8b5b25a581e 100644 --- a/ddtrace/contrib/internal/langchain/patch.py +++ b/ddtrace/contrib/internal/langchain/patch.py @@ -1090,29 +1090,6 @@ def unpatch(): delattr(langchain, "_datadog_integration") -def taint_outputs(instance, inputs, outputs): - from ddtrace.appsec._iast._metrics import _set_iast_error_metric - from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject - - try: - ranges = None - for key in filter(lambda x: x in inputs, instance.input_keys): - input_val = inputs.get(key) - if input_val: - ranges = get_tainted_ranges(input_val) - if ranges: - break - - if ranges: - source = ranges[0].source - for key in filter(lambda x: x in outputs, instance.output_keys): - output_value = outputs[key] - outputs[key] = taint_pyobject(output_value, source.name, source.value, source.origin) - except Exception as e: - _set_iast_error_metric("IAST propagation error. langchain taint_outputs. {}".format(e)) - - def taint_parser_output(func, instance, args, kwargs): from ddtrace.appsec._iast._metrics import _set_iast_error_metric from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges diff --git a/ddtrace/contrib/internal/mysql/patch.py b/ddtrace/contrib/internal/mysql/patch.py index 6425bd33766..0aad999e546 100644 --- a/ddtrace/contrib/internal/mysql/patch.py +++ b/ddtrace/contrib/internal/mysql/patch.py @@ -4,8 +4,6 @@ import wrapt from ddtrace import config -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.contrib.dbapi import TracedConnection from ddtrace.contrib.internal.trace_utils import _convert_to_string from ddtrace.ext import db @@ -51,6 +49,9 @@ def patch(): mysql.connector.Connect = mysql.connector.connect if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION + _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION) mysql.connector._datadog_patch = True diff --git a/ddtrace/contrib/internal/mysqldb/patch.py b/ddtrace/contrib/internal/mysqldb/patch.py index cde0f58629f..fe9c62bbd5e 100644 --- a/ddtrace/contrib/internal/mysqldb/patch.py +++ b/ddtrace/contrib/internal/mysqldb/patch.py @@ -4,8 +4,6 @@ from wrapt import wrap_function_wrapper as _w from ddtrace import config -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.constants import _SPAN_MEASURED_KEY from ddtrace.constants import SPAN_KIND from ddtrace.contrib.dbapi import TracedConnection @@ -67,6 +65,9 @@ def patch(): _w("MySQLdb", "connect", _connect) if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION + _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION) diff --git a/ddtrace/contrib/internal/pytest/_plugin_v2.py b/ddtrace/contrib/internal/pytest/_plugin_v2.py index 79435f94576..a9736374114 100644 --- a/ddtrace/contrib/internal/pytest/_plugin_v2.py +++ b/ddtrace/contrib/internal/pytest/_plugin_v2.py @@ -60,6 +60,7 @@ from ddtrace.internal.test_visibility.api import InternalTestSession from ddtrace.internal.test_visibility.api import InternalTestSuite from ddtrace.internal.test_visibility.coverage_lines import CoverageLines +from ddtrace.settings.asm import config as asm_config from ddtrace.vendor.debtcollector import deprecate @@ -574,9 +575,10 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial def pytest_terminal_summary(terminalreporter, exitstatus, config): """Report flaky or failed tests""" try: - from ddtrace.appsec._iast._pytest_plugin import print_iast_report + if asm_config._iast_enabled: + from ddtrace.appsec._iast._pytest_plugin import print_iast_report - print_iast_report(terminalreporter) + print_iast_report(terminalreporter) except Exception: # noqa: E722 log.debug("Encountered error during code security summary", exc_info=True) diff --git a/ddtrace/contrib/internal/pytest/plugin.py b/ddtrace/contrib/internal/pytest/plugin.py index 6ac5df3442b..0908878e0f0 100644 --- a/ddtrace/contrib/internal/pytest/plugin.py +++ b/ddtrace/contrib/internal/pytest/plugin.py @@ -17,13 +17,16 @@ import pytest from ddtrace import config -from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast # noqa:F401 from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 from ddtrace.contrib.internal.pytest._utils import _extract_span from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_itr from ddtrace.settings.asm import config as asm_config +if asm_config._iast_enabled: + from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast # noqa:F401 + + # pytest default settings config._add( "pytest", diff --git a/ddtrace/contrib/internal/requests/patch.py b/ddtrace/contrib/internal/requests/patch.py index eab51c2c0a4..a885e5575de 100644 --- a/ddtrace/contrib/internal/requests/patch.py +++ b/ddtrace/contrib/internal/requests/patch.py @@ -4,9 +4,6 @@ from wrapt import wrap_function_wrapper as _w from ddtrace import config -from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.contrib.internal.trace_utils import unwrap as _u from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool @@ -46,10 +43,16 @@ def patch(): _w("requests", "Session.send", _wrap_send) # IAST needs to wrap this function because `Session.send` is too late - _w("requests", "Session.request", _wrap_request) + if asm_config._load_modules: + from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request + + _w("requests", "Session.request", _wrap_request) Pin(_config=config.requests).onto(requests.Session) if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SSRF + _set_metric_iast_instrumented_sink(VULN_SSRF) @@ -59,5 +62,13 @@ def unpatch(): return requests.__datadog_patch = False - _u(requests.Session, "request") - _u(requests.Session, "send") + try: + _u(requests.Session, "request") + except AttributeError: + # It was not patched + pass + try: + _u(requests.Session, "send") + except AttributeError: + # It was not patched + pass diff --git a/ddtrace/contrib/internal/sqlalchemy/patch.py b/ddtrace/contrib/internal/sqlalchemy/patch.py index 916cc53daa4..c6d6df476c1 100644 --- a/ddtrace/contrib/internal/sqlalchemy/patch.py +++ b/ddtrace/contrib/internal/sqlalchemy/patch.py @@ -1,8 +1,6 @@ import sqlalchemy from wrapt import wrap_function_wrapper as _w -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.contrib.internal.trace_utils import unwrap from ddtrace.settings.asm import config as asm_config @@ -24,6 +22,9 @@ def patch(): _w("sqlalchemy.engine", "create_engine", _wrap_create_engine) if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION + _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION) diff --git a/ddtrace/contrib/internal/sqlite3/patch.py b/ddtrace/contrib/internal/sqlite3/patch.py index 03c79789661..68ee5779983 100644 --- a/ddtrace/contrib/internal/sqlite3/patch.py +++ b/ddtrace/contrib/internal/sqlite3/patch.py @@ -5,8 +5,6 @@ import wrapt from ddtrace import config -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.contrib.dbapi import FetchTracedCursor from ddtrace.contrib.dbapi import TracedConnection from ddtrace.contrib.dbapi import TracedCursor @@ -47,6 +45,9 @@ def patch(): sqlite3.dbapi2.connect = wrapped if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION + _set_metric_iast_instrumented_sink(VULN_SQL_INJECTION) diff --git a/ddtrace/contrib/internal/subprocess/patch.py b/ddtrace/contrib/internal/subprocess/patch.py index 2d66edd4737..1ffc1f2d733 100644 --- a/ddtrace/contrib/internal/subprocess/patch.py +++ b/ddtrace/contrib/internal/subprocess/patch.py @@ -58,7 +58,7 @@ def del_lst_callback(name: str): def patch() -> List[str]: - if not (asm_config._asm_enabled or asm_config._iast_enabled): + if not asm_config._load_modules: return [] patched: List[str] = [] @@ -66,7 +66,7 @@ def patch() -> List[str]: import subprocess # nosec should_patch_system = not trace_utils.iswrapped(os.system) - should_patch_fork = not trace_utils.iswrapped(os.fork) + should_patch_fork = (not trace_utils.iswrapped(os.fork)) if hasattr(os, "fork") else False spawnvef = getattr(os, "_spawnvef", None) should_patch_spawnvef = spawnvef is not None and not trace_utils.iswrapped(spawnvef) @@ -316,10 +316,11 @@ def unpatch() -> None: import os # nosec import subprocess # nosec - trace_utils.unwrap(os, "system") - trace_utils.unwrap(os, "_spawnvef") - trace_utils.unwrap(subprocess.Popen, "__init__") - trace_utils.unwrap(subprocess.Popen, "wait") + for obj, attr in [(os, "system"), (os, "_spawnvef"), (subprocess.Popen, "__init__"), (subprocess.Popen, "wait")]: + try: + trace_utils.unwrap(obj, attr) + except AttributeError: + pass SubprocessCmdLine._clear_cache() @@ -327,7 +328,7 @@ def unpatch() -> None: @trace_utils.with_traced_module def _traced_ossystem(module, pin, wrapped, instance, args, kwargs): try: - if asm_config._bypass_instrumentation_for_waf: + if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled): return wrapped(*args, **kwargs) if isinstance(args[0], str): for callback in _STR_CALLBACKS.values(): @@ -351,6 +352,8 @@ def _traced_ossystem(module, pin, wrapped, instance, args, kwargs): @trace_utils.with_traced_module def _traced_fork(module, pin, wrapped, instance, args, kwargs): + if not (asm_config._asm_enabled or asm_config._iast_enabled): + return wrapped(*args, **kwargs) try: with pin.tracer.trace(COMMANDS.SPAN_NAME, resource="fork", span_type=SpanTypes.SYSTEM) as span: span.set_tag(COMMANDS.EXEC, ["os.fork"]) @@ -366,6 +369,8 @@ def _traced_fork(module, pin, wrapped, instance, args, kwargs): @trace_utils.with_traced_module def _traced_osspawn(module, pin, wrapped, instance, args, kwargs): + if not (asm_config._asm_enabled or asm_config._iast_enabled): + return wrapped(*args, **kwargs) try: mode, file, func_args, _, _ = args if isinstance(func_args, (list, tuple, str)): @@ -395,7 +400,7 @@ def _traced_osspawn(module, pin, wrapped, instance, args, kwargs): @trace_utils.with_traced_module def _traced_subprocess_init(module, pin, wrapped, instance, args, kwargs): try: - if asm_config._bypass_instrumentation_for_waf: + if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled): return wrapped(*args, **kwargs) cmd_args = args[0] if len(args) else kwargs["args"] if isinstance(cmd_args, (list, tuple, str)): @@ -429,7 +434,7 @@ def _traced_subprocess_init(module, pin, wrapped, instance, args, kwargs): @trace_utils.with_traced_module def _traced_subprocess_wait(module, pin, wrapped, instance, args, kwargs): try: - if asm_config._bypass_instrumentation_for_waf: + if asm_config._bypass_instrumentation_for_waf or not (asm_config._asm_enabled or asm_config._iast_enabled): return wrapped(*args, **kwargs) binary = core.get_item("subprocess_popen_binary") diff --git a/ddtrace/contrib/internal/urllib/patch.py b/ddtrace/contrib/internal/urllib/patch.py index ed5e2891f06..1ba279fb20a 100644 --- a/ddtrace/contrib/internal/urllib/patch.py +++ b/ddtrace/contrib/internal/urllib/patch.py @@ -2,9 +2,6 @@ from wrapt import wrap_function_wrapper as _w -from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_open -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.contrib.internal.trace_utils import unwrap as _u from ddtrace.settings.asm import config as asm_config @@ -20,8 +17,15 @@ def patch(): return urllib.request.__datadog_patch = True - _w("urllib.request", "urlopen", _wrap_open) + if asm_config._load_modules: + from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_open + + _w("urllib.request", "urlopen", _wrap_open) + if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SSRF + _set_metric_iast_instrumented_sink(VULN_SSRF) diff --git a/ddtrace/contrib/internal/urllib3/patch.py b/ddtrace/contrib/internal/urllib3/patch.py index 6c10526c125..7c5d6adc28d 100644 --- a/ddtrace/contrib/internal/urllib3/patch.py +++ b/ddtrace/contrib/internal/urllib3/patch.py @@ -4,9 +4,6 @@ from wrapt import wrap_function_wrapper as _w from ddtrace import config -from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND from ddtrace.contrib import trace_utils @@ -54,14 +51,20 @@ def patch(): urllib3.__datadog_patch = True _w("urllib3", "connectionpool.HTTPConnectionPool.urlopen", _wrap_urlopen) - if hasattr(urllib3, "_request_methods"): - _w("urllib3._request_methods", "RequestMethods.request", _wrap_request) - else: - # Old version before https://github.com/urllib3/urllib3/pull/2398 - _w("urllib3.request", "RequestMethods.request", _wrap_request) + if asm_config._load_modules: + from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_request + + if hasattr(urllib3, "_request_methods"): + _w("urllib3._request_methods", "RequestMethods.request", _wrap_request) + else: + # Old version before https://github.com/urllib3/urllib3/pull/2398 + _w("urllib3.request", "RequestMethods.request", _wrap_request) Pin().onto(urllib3.connectionpool.HTTPConnectionPool) if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SSRF + _set_metric_iast_instrumented_sink(VULN_SSRF) diff --git a/ddtrace/contrib/internal/webbrowser/patch.py b/ddtrace/contrib/internal/webbrowser/patch.py index 1387df37ac9..1f90e9cf9aa 100644 --- a/ddtrace/contrib/internal/webbrowser/patch.py +++ b/ddtrace/contrib/internal/webbrowser/patch.py @@ -2,9 +2,6 @@ from wrapt import wrap_function_wrapper as _w -from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_open -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink -from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.contrib.internal.trace_utils import unwrap as _u from ddtrace.settings.asm import config as asm_config @@ -20,9 +17,15 @@ def patch(): return webbrowser.__datadog_patch = True - _w("webbrowser", "open", _wrap_open) + if asm_config._load_modules: + from ddtrace.appsec._common_module_patches import wrapped_request_D8CB81E472AF98A2 as _wrap_open + + _w("webbrowser", "open", _wrap_open) if asm_config._iast_enabled: + from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink + from ddtrace.appsec._iast.constants import VULN_SSRF + _set_metric_iast_instrumented_sink(VULN_SSRF) diff --git a/ddtrace/contrib/internal/yaaredis/patch.py b/ddtrace/contrib/internal/yaaredis/patch.py deleted file mode 100644 index f9cba77b5bb..00000000000 --- a/ddtrace/contrib/internal/yaaredis/patch.py +++ /dev/null @@ -1,91 +0,0 @@ -import os - -import wrapt -import yaaredis - -from ddtrace import config -from ddtrace._trace.utils_redis import _instrument_redis_cmd -from ddtrace._trace.utils_redis import _instrument_redis_execute_pipeline -from ddtrace.contrib.internal.redis_utils import _run_redis_command_async -from ddtrace.internal.schema import schematize_service_name -from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning -from ddtrace.internal.utils.formats import CMD_MAX_LEN -from ddtrace.internal.utils.formats import asbool -from ddtrace.internal.utils.formats import stringify_cache_args -from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.trace import Pin -from ddtrace.vendor.debtcollector import deprecate - - -config._add( - "yaaredis", - dict( - _default_service=schematize_service_name("redis"), - cmd_max_length=int(os.getenv("DD_YAAREDIS_CMD_MAX_LENGTH", CMD_MAX_LEN)), - resource_only_command=asbool(os.getenv("DD_REDIS_RESOURCE_ONLY_COMMAND", True)), - ), -) - - -def get_version(): - # type: () -> str - return getattr(yaaredis, "__version__", "") - - -def patch(): - """Patch the instrumented methods""" - deprecate( - prefix="The yaaredis module is deprecated.", - message="The yaaredis module is deprecated and will be deleted.", - category=DDTraceDeprecationWarning, - removal_version="3.0.0", - ) - - if getattr(yaaredis, "_datadog_patch", False): - return - yaaredis._datadog_patch = True - - _w = wrapt.wrap_function_wrapper - - _w("yaaredis.client", "StrictRedis.execute_command", traced_execute_command) - _w("yaaredis.client", "StrictRedis.pipeline", traced_pipeline) - _w("yaaredis.pipeline", "StrictPipeline.execute", traced_execute_pipeline) - _w("yaaredis.pipeline", "StrictPipeline.immediate_execute_command", traced_execute_command) - Pin().onto(yaaredis.StrictRedis) - - -def unpatch(): - if getattr(yaaredis, "_datadog_patch", False): - yaaredis._datadog_patch = False - - unwrap(yaaredis.client.StrictRedis, "execute_command") - unwrap(yaaredis.client.StrictRedis, "pipeline") - unwrap(yaaredis.pipeline.StrictPipeline, "execute") - unwrap(yaaredis.pipeline.StrictPipeline, "immediate_execute_command") - - -async def traced_execute_command(func, instance, args, kwargs): - pin = Pin.get_from(instance) - if not pin or not pin.enabled(): - return await func(*args, **kwargs) - - with _instrument_redis_cmd(pin, config.yaaredis, instance, args) as ctx: - return await _run_redis_command_async(ctx=ctx, func=func, args=args, kwargs=kwargs) - - -async def traced_pipeline(func, instance, args, kwargs): - pipeline = await func(*args, **kwargs) - pin = Pin.get_from(instance) - if pin: - pin.onto(pipeline) - return pipeline - - -async def traced_execute_pipeline(func, instance, args, kwargs): - pin = Pin.get_from(instance) - if not pin or not pin.enabled(): - return await func(*args, **kwargs) - - cmds = [stringify_cache_args(c, cmd_max_len=config.yaaredis.cmd_max_length) for c, _ in instance.command_stack] - with _instrument_redis_execute_pipeline(pin, config.yaaredis, cmds, instance): - return await func(*args, **kwargs) diff --git a/ddtrace/internal/appsec/product.py b/ddtrace/internal/appsec/product.py index 126d6d2a04f..e0854ff2a2a 100644 --- a/ddtrace/internal/appsec/product.py +++ b/ddtrace/internal/appsec/product.py @@ -1,4 +1,3 @@ -from ddtrace import config from ddtrace.settings.asm import config as asm_config @@ -10,14 +9,14 @@ def post_preload(): def start(): - if asm_config._asm_enabled or config._remote_config_enabled: + if asm_config._asm_rc_enabled: from ddtrace.appsec._remoteconfiguration import enable_appsec_rc enable_appsec_rc() def restart(join=False): - if asm_config._asm_enabled or config._remote_config_enabled: + if asm_config._asm_rc_enabled: from ddtrace.appsec._remoteconfiguration import _forksafe_appsec_rc _forksafe_appsec_rc() diff --git a/ddtrace/internal/logger.py b/ddtrace/internal/logger.py index 0592afa0a74..6f7b853c3dc 100644 --- a/ddtrace/internal/logger.py +++ b/ddtrace/internal/logger.py @@ -1,180 +1,80 @@ import collections import logging import os -import typing -from typing import Optional # noqa:F401 -from typing import cast # noqa:F401 +from typing import DefaultDict +from typing import Tuple -if typing.TYPE_CHECKING: - from typing import Any # noqa:F401 - from typing import DefaultDict # noqa:F401 - from typing import Tuple # noqa:F401 - - -def get_logger(name): - # type: (str) -> DDLogger +def get_logger(name: str) -> logging.Logger: """ - Retrieve or create a ``DDLogger`` instance. - - This function mirrors the behavior of `logging.getLogger`. + Retrieve or create a ``Logger`` instance with consistent behavior for internal use. - If no logger with the provided name has been fetched before then - a new one is created. + Configure all loggers with a rate limiter filter to prevent excessive logging. - If a previous logger has been created then it is returned. - - DEV: We do not want to mess with `logging.setLoggerClass()` - That will totally mess with the user's loggers, we want - just our own, selective loggers to be DDLoggers - - :param name: The name of the logger to fetch or create - :type name: str - :return: The logger instance - :rtype: ``DDLogger`` """ - # DEV: `logging.Logger.manager` refers to the single root `logging.Manager` instance - # https://github.com/python/cpython/blob/48769a28ad6ef4183508951fa6a378531ace26a4/Lib/logging/__init__.py#L1824-L1826 # noqa:E501 - manager = logging.Logger.manager + logger = logging.getLogger(name) + # addFilter will only add the filter if it is not already present + logger.addFilter(log_filter) + return logger - # If the logger does not exist yet, create it - # DEV: `Manager.loggerDict` is a dict mapping logger name to logger - # DEV: This is a simplified version of `logging.Manager.getLogger` - # https://github.com/python/cpython/blob/48769a28ad6ef4183508951fa6a378531ace26a4/Lib/logging/__init__.py#L1221-L1253 # noqa:E501 - # DEV: _fixupParents could be adding a placeholder, we want to replace it if that's the case - if name in manager.loggerDict: - logger = manager.loggerDict[name] - if isinstance(manager.loggerDict[name], logging.PlaceHolder): - placeholder = logger - logger = DDLogger(name=name) - manager.loggerDict[name] = logger - # DEV: `_fixupChildren` and `_fixupParents` have been around for awhile, - # DEV: but add the `hasattr` guard... just in case. - if hasattr(manager, "_fixupChildren"): - manager._fixupChildren(placeholder, logger) - if hasattr(manager, "_fixupParents"): - manager._fixupParents(logger) - else: - logger = DDLogger(name=name) - manager.loggerDict[name] = logger - if hasattr(manager, "_fixupParents"): - manager._fixupParents(logger) - # Return our logger - return cast(DDLogger, logger) +# Named tuple used for keeping track of a log lines current time bucket and the number of log lines skipped +LoggingBucket = collections.namedtuple("LoggingBucket", ("bucket", "skipped")) +# Dict to keep track of the current time bucket per name/level/pathname/lineno +_buckets: DefaultDict[Tuple[str, int, str, int], LoggingBucket] = collections.defaultdict(lambda: LoggingBucket(0, 0)) +# Allow 1 log record per name/level/pathname/lineno every 60 seconds by default +# Allow configuring via `DD_TRACE_LOGGING_RATE` +# DEV: `DD_TRACE_LOGGING_RATE=0` means to disable all rate limiting +_rate_limit = int(os.getenv("DD_TRACE_LOGGING_RATE", default=60)) -def hasHandlers(self): - # type: (DDLogger) -> bool - """ - See if this logger has any handlers configured. - Loop through all handlers for this logger and its parents in the - logger hierarchy. Return True if a handler was found, else False. - Stop searching up the hierarchy whenever a logger with the "propagate" - attribute set to zero is found - that will be the last logger which - is checked for the existence of handlers. - https://github.com/python/cpython/blob/8f192d12af82c4dc40730bf59814f6a68f68f950/Lib/logging/__init__.py#L1629 +def log_filter(record: logging.LogRecord) -> bool: """ - c = self - rv = False - while c: - if c.handlers: - rv = True - break - if not c.propagate: - break - else: - c = c.parent # type: ignore - return rv + Function used to determine if a log record should be outputted or not (True = output, False = skip). - -class DDLogger(logging.Logger): - """ - Custom rate limited logger used by ``ddtrace`` - - This logger class is used to rate limit the output of - log messages from within the ``ddtrace`` package. + This function will: + - Log all records with a level of ERROR or higher with telemetry + - Rate limit log records based on the logger name, record level, filename, and line number """ + if record.levelno >= logging.ERROR: + # avoid circular import + from ddtrace.internal import telemetry - # Named tuple used for keeping track of a log lines current time bucket and the number of log lines skipped - LoggingBucket = collections.namedtuple("LoggingBucket", ("bucket", "skipped")) - - def __init__(self, *args, **kwargs): - # type: (*Any, **Any) -> None - """Constructor for ``DDLogger``""" - super(DDLogger, self).__init__(*args, **kwargs) - - # Dict to keep track of the current time bucket per name/level/pathname/lineno - self.buckets = collections.defaultdict( - lambda: DDLogger.LoggingBucket(0, 0) - ) # type: DefaultDict[Tuple[str, int, str, int], DDLogger.LoggingBucket] + # currently we only have one error code + full_file_name = os.path.join(record.pathname, record.filename) + telemetry.telemetry_writer.add_error(1, record.msg % record.args, full_file_name, record.lineno) - # Allow 1 log record per name/level/pathname/lineno every 60 seconds by default - # Allow configuring via `DD_TRACE_LOGGING_RATE` - # DEV: `DD_TRACE_LOGGING_RATE=0` means to disable all rate limiting - rate_limit = os.getenv("DD_TRACE_LOGGING_RATE", default=None) - - if rate_limit is not None: - self.rate_limit = int(rate_limit) - else: - self.rate_limit = 60 - - def handle(self, record): - # type: (logging.LogRecord) -> None - """ - Function used to call the handlers for a log line. - - This implementation will first determine if this log line should - be logged or rate limited, and then call the base ``logging.Logger.handle`` - function if it should be logged - - DEV: This method has all of it's code inlined to reduce on functions calls - - :param record: The log record being logged - :type record: ``logging.LogRecord`` - """ - if record.levelno >= logging.ERROR: - # avoid circular import - from ddtrace.internal import telemetry - - # currently we only have one error code - full_file_name = os.path.join(record.pathname, record.filename) - telemetry.telemetry_writer.add_error(1, record.msg % record.args, full_file_name, record.lineno) - - # If rate limiting has been disabled (`DD_TRACE_LOGGING_RATE=0`) then apply no rate limit - # If the logging is in debug, then do not apply any limits to any log - if not self.rate_limit or self.getEffectiveLevel() == logging.DEBUG: - super(DDLogger, self).handle(record) - return + logger = logging.getLogger(record.name) + # If rate limiting has been disabled (`DD_TRACE_LOGGING_RATE=0`) then apply no rate limit + # If the logger is set to debug, then do not apply any limits to any log + if not _rate_limit or logger.getEffectiveLevel() == logging.DEBUG: + return True # Allow 1 log record by name/level/pathname/lineno every X seconds - # DEV: current unix time / rate (e.g. 300 seconds) = time bucket - # int(1546615098.8404942 / 300) = 515538 - # DEV: LogRecord `created` is a unix timestamp/float - # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10` - current_bucket = int(record.created / self.rate_limit) - - # Limit based on logger name, record level, filename, and line number - # ('ddtrace.writer', 'DEBUG', '../site-packages/ddtrace/writer.py', 137) - # This way each unique log message can get logged at least once per time period - # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10` - key = (record.name, record.levelno, record.pathname, record.lineno) - - # Only log this message if the time bucket has changed from the previous time we ran - logging_bucket = self.buckets[key] - if logging_bucket.bucket != current_bucket: - # Append count of skipped messages if we have skipped some since our last logging - if logging_bucket.skipped: - record.msg = "{}, %s additional messages skipped".format(record.msg) - record.args = record.args + (logging_bucket.skipped,) # type: ignore - + # DEV: current unix time / rate (e.g. 300 seconds) = time bucket + # int(1546615098.8404942 / 300) = 515538 + # DEV: LogRecord `created` is a unix timestamp/float + # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10` + current_bucket = int(record.created / _rate_limit) + # Limit based on logger name, record level, filename, and line number + # ('ddtrace.writer', 'DEBUG', '../site-packages/ddtrace/writer.py', 137) + # This way each unique log message can get logged at least once per time period + # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10` + key = (record.name, record.levelno, record.pathname, record.lineno) + # Only log this message if the time bucket has changed from the previous time we ran + logging_bucket = _buckets[key] + if logging_bucket.bucket != current_bucket: + # Append count of skipped messages if we have skipped some since our last logging + if logging_bucket.skipped: + record.msg = "{}, %s additional messages skipped".format(record.msg) + record.args = record.args + (logging_bucket.skipped,) # type: ignore # Reset our bucket - self.buckets[key] = DDLogger.LoggingBucket(current_bucket, 0) - - # Call the base handle to actually log this record - super(DDLogger, self).handle(record) - else: - # Increment the count of records we have skipped - # DEV: `self.buckets[key]` is a tuple which is immutable so recreate instead - self.buckets[key] = DDLogger.LoggingBucket(logging_bucket.bucket, logging_bucket.skipped + 1) + _buckets[key] = LoggingBucket(current_bucket, 0) + # Actually log this record + return True + # Increment the count of records we have skipped + # DEV: `buckets[key]` is a tuple which is immutable so recreate instead + _buckets[key] = LoggingBucket(logging_bucket.bucket, logging_bucket.skipped + 1) + # Skip this log message + return False diff --git a/ddtrace/internal/writer/writer.py b/ddtrace/internal/writer/writer.py index da3f09a99b1..301d0400c7c 100644 --- a/ddtrace/internal/writer/writer.py +++ b/ddtrace/internal/writer/writer.py @@ -578,9 +578,7 @@ def start(self): try: # appsec remote config should be enabled/started after the global tracer and configs # are initialized - if os.getenv("AWS_LAMBDA_FUNCTION_NAME") is None and ( - asm_config._asm_enabled or config._remote_config_enabled - ): + if asm_config._asm_rc_enabled: from ddtrace.appsec._remoteconfiguration import enable_appsec_rc enable_appsec_rc() diff --git a/ddtrace/settings/asm.py b/ddtrace/settings/asm.py index 4024c13f982..8cb35132d47 100644 --- a/ddtrace/settings/asm.py +++ b/ddtrace/settings/asm.py @@ -17,6 +17,7 @@ from ddtrace.appsec._constants import LOGIN_EVENTS_MODE from ddtrace.appsec._constants import TELEMETRY_INFORMATION_NAME from ddtrace.constants import APPSEC_ENV +from ddtrace.internal.serverless import in_aws_lambda from ddtrace.settings._core import report_telemetry as _report_telemetry @@ -224,15 +225,21 @@ class ASMConfig(Env): def __init__(self): super().__init__() - # Is one click available? - self._eval_asm_can_be_enabled() - if not self._asm_libddwaf_available: + if not self._iast_supported: + self._iast_enabled = False + if not self._asm_libddwaf_available or in_aws_lambda(): self._asm_enabled = False self._asm_can_be_enabled = False self._iast_enabled = False self._api_security_enabled = False - if not self._iast_supported: - self._iast_enabled = False + self._ep_enabled = False + self._auto_user_instrumentation_enabled = False + self._auto_user_instrumentation_local_mode = LOGIN_EVENTS_MODE.DISABLED + self._load_modules = False + self._asm_rc_enabled = False + else: + # Is one click available? + self._eval_asm_can_be_enabled() def reset(self): """For testing purposes, reset the configuration to its default values given current environment variables.""" @@ -240,6 +247,10 @@ def reset(self): def _eval_asm_can_be_enabled(self): self._asm_can_be_enabled = APPSEC_ENV not in os.environ and tracer_config._remote_config_enabled + self._load_modules: bool = bool( + self._iast_enabled or (self._ep_enabled and (self._asm_enabled or self._asm_can_be_enabled)) + ) + self._asm_rc_enabled = (self._asm_enabled and tracer_config._remote_config_enabled) or self._asm_can_be_enabled @property def _api_security_feature_active(self) -> bool: diff --git a/docs/index.rst b/docs/index.rst index d71419dc03e..3526d6a6f18 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -182,8 +182,6 @@ contacting support. +--------------------------------------------------+---------------+----------------+ | :ref:`wsgi` | \* | No | +--------------------------------------------------+---------------+----------------+ -| :ref:`yaaredis` | >= 2.0.0 | Yes | -+--------------------------------------------------+---------------+----------------+ .. [1] Libraries that are automatically instrumented when the diff --git a/docs/integrations.rst b/docs/integrations.rst index d14c03bcfb7..f9601e3aeb0 100644 --- a/docs/integrations.rst +++ b/docs/integrations.rst @@ -505,12 +505,6 @@ Vertica ^^^^^^^ .. automodule:: ddtrace.contrib._vertica -.. _yaaredis: - -yaaredis -^^^^^^^^ -.. automodule:: ddtrace.contrib._yaaredis - .. _wsgi: WSGI diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 913ad7f8319..cc2969bd5c9 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -315,4 +315,3 @@ workflow Wrapt wsgi xfail -yaaredis diff --git a/hatch.toml b/hatch.toml index 74dcba41602..f5d3b99dd28 100644 --- a/hatch.toml +++ b/hatch.toml @@ -374,7 +374,7 @@ dependencies = [ test = [ "uname -a", "pip freeze", - "DD_TRACE_AGENT_URL=\"http://testagent:9126\" DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 DD_IAST_DEDUPLICATION_ENABLED=false python -m pytest -vvv {args:tests/appsec/integrations/flask_tests/}", + "DD_TRACE_AGENT_URL=\"http://testagent:9126\" DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_ENABLED=true DD_IAST_REQUEST_SAMPLING=100 DD_IAST_DEDUPLICATION_ENABLED=false python -m pytest -vvv {args:tests/appsec/integrations/flask_tests/}", ] [[envs.appsec_integrations_flask.matrix]] diff --git a/lib-injection/sources/min_compatible_versions.csv b/lib-injection/sources/min_compatible_versions.csv index c7366036a89..0c61576da1a 100644 --- a/lib-injection/sources/min_compatible_versions.csv +++ b/lib-injection/sources/min_compatible_versions.csv @@ -191,6 +191,5 @@ webtest,0 werkzeug,<1.0 wheel,0 xmltodict,>=0.12 -yaaredis,~=2.0.0 yarl,~=1.0 zeep,0 diff --git a/min_compatible_versions.csv b/min_compatible_versions.csv index 382aec7fd6a..dd91aedc2b4 100644 --- a/min_compatible_versions.csv +++ b/min_compatible_versions.csv @@ -192,6 +192,5 @@ webtest,0 werkzeug,<1.0 wheel,0 xmltodict,>=0.12 -yaaredis,~=2.0.0 yarl,~=1.0 zeep,0 diff --git a/releasenotes/notes/ensure_no_appsec_loading-8ce46c58d6ecf81f.yaml b/releasenotes/notes/ensure_no_appsec_loading-8ce46c58d6ecf81f.yaml new file mode 100644 index 00000000000..b7c34f83779 --- /dev/null +++ b/releasenotes/notes/ensure_no_appsec_loading-8ce46c58d6ecf81f.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + ASM: This ensures that no module from ASM are loaded when ASM is disabled or unavailable. + SCA: This ensures that no module from IAST are loaded when IAST is disabled or unavailable. \ No newline at end of file diff --git a/releasenotes/notes/fix-internal-logging-init-6058c02b527cdf77.yaml b/releasenotes/notes/fix-internal-logging-init-6058c02b527cdf77.yaml new file mode 100644 index 00000000000..132e7efaab5 --- /dev/null +++ b/releasenotes/notes/fix-internal-logging-init-6058c02b527cdf77.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + internal: Fix ``ddtrace`` internal logger initialization mutating an unlocked shared resource. diff --git a/releasenotes/notes/remove-yaaredis-095441532392e12f.yaml b/releasenotes/notes/remove-yaaredis-095441532392e12f.yaml new file mode 100644 index 00000000000..ce5ae6da6c1 --- /dev/null +++ b/releasenotes/notes/remove-yaaredis-095441532392e12f.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + yaaredis: Removes the yaaredis integration from ``ddtrace``. For yaaredis support pin ddtrace to an older version. \ No newline at end of file diff --git a/riotfile.py b/riotfile.py index fe4aece3126..f04523d3057 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2091,25 +2091,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest-randomly": latest, }, ), - Venv( - name="yaaredis", - command="pytest {cmdargs} tests/contrib/yaaredis", - pkgs={ - "pytest-asyncio": "==0.21.1", - "pytest-randomly": latest, - }, - venvs=[ - Venv( - pys=select_pys(min_version="3.8", max_version="3.9"), - pkgs={"yaaredis": ["~=2.0.0", latest]}, - ), - Venv( - # yaaredis added support for Python 3.10 in 3.0 - pys="3.10", - pkgs={"yaaredis": latest}, - ), - ], - ), Venv( name="sanic", command="pytest {cmdargs} tests/contrib/sanic", diff --git a/supported_versions_output.json b/supported_versions_output.json index 06eff37be25..7b158105f97 100644 --- a/supported_versions_output.json +++ b/supported_versions_output.json @@ -305,11 +305,5 @@ "minimum_tracer_supported": "1.71.1", "max_tracer_supported": "1.71.1", "auto-instrumented": true - }, - { - "integration": "yaaredis", - "minimum_tracer_supported": "2.0.4", - "max_tracer_supported": "3.0.0", - "auto-instrumented": true } ] \ No newline at end of file diff --git a/supported_versions_table.csv b/supported_versions_table.csv index edbe73503cd..1f230039d50 100644 --- a/supported_versions_table.csv +++ b/supported_versions_table.csv @@ -49,4 +49,3 @@ tornado *,4.5.3,6.4,False urllib3,1.24.3,2.2.3,False valkey,6.0.0,6.0.2,True vertexai,1.71.1,1.71.1,True -yaaredis,2.0.4,3.0.0,True diff --git a/tests/appsec/integrations/django_tests/conftest.py b/tests/appsec/integrations/django_tests/conftest.py index d047b7acee5..688c09f0ce4 100644 --- a/tests/appsec/integrations/django_tests/conftest.py +++ b/tests/appsec/integrations/django_tests/conftest.py @@ -6,12 +6,13 @@ from ddtrace.appsec._iast import enable_iast_propagation from ddtrace.appsec._iast._patch_modules import patch_iast -from ddtrace.contrib.internal.django.patch import patch +from ddtrace.contrib.internal.django.patch import patch as django_patch from ddtrace.trace import Pin from tests.appsec.iast.conftest import _end_iast_context_and_oce from tests.appsec.iast.conftest import _start_iast_context_and_oce from tests.utils import DummyTracer from tests.utils import TracerSpanContainer +from tests.utils import override_env from tests.utils import override_global_config @@ -26,10 +27,10 @@ def pytest_configure(): _iast_deduplication_enabled=False, _iast_request_sampling=100.0, ) - ): + ), override_env(dict(_DD_IAST_PATCH_MODULES="tests.appsec.integrations")): settings.DEBUG = False patch_iast() - patch() + django_patch() enable_iast_propagation() django.setup() diff --git a/tests/appsec/integrations/django_tests/django_app/urls.py b/tests/appsec/integrations/django_tests/django_app/urls.py index e79b6bee284..6a77fe99808 100644 --- a/tests/appsec/integrations/django_tests/django_app/urls.py +++ b/tests/appsec/integrations/django_tests/django_app/urls.py @@ -46,6 +46,11 @@ def shutdown(request): views.sqli_http_request_parameter_name_post, name="sqli_http_request_parameter_name_post", ), + handler( + "appsec/sqli_query_no_redacted/$", + views.sqli_query_no_redacted, + name="sqli_query_no_redacted", + ), handler( "appsec/sqli_http_request_header_name/$", views.sqli_http_request_header_name, diff --git a/tests/appsec/integrations/django_tests/django_app/views.py b/tests/appsec/integrations/django_tests/django_app/views.py index 685b3c598c2..020f127753b 100644 --- a/tests/appsec/integrations/django_tests/django_app/views.py +++ b/tests/appsec/integrations/django_tests/django_app/views.py @@ -129,6 +129,14 @@ def sqli_http_request_parameter_name_post(request): return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) +def sqli_query_no_redacted(request): + obj = request.GET["q"] + with connection.cursor() as cursor: + # label sqli_query_no_redacted + cursor.execute(f"SELECT * FROM {obj} ORDER BY name") + return HttpResponse("OK", status=200) + + def sqli_http_request_header_name(request): key = [x for x in request.META.keys() if x == "master"][0] diff --git a/tests/appsec/integrations/django_tests/test_django_appsec_iast.py b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py index 657688f5760..2999a286483 100644 --- a/tests/appsec/integrations/django_tests/test_django_appsec_iast.py +++ b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py @@ -168,7 +168,7 @@ def test_django_tainted_user_agent_iast_disabled(client, test_spans, tracer): @pytest.mark.django_db() @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") -def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter(client, test_spans, tracer): +def test_django_sqli_http_request_parameter(client, test_spans, tracer): root_span, response = _aux_appsec_get_root_span( client, test_spans, @@ -309,6 +309,46 @@ def test_django_sqli_http_request_parameter_name_post(client, test_spans, tracer assert loaded["vulnerabilities"][0]["hash"] == hash_value +@pytest.mark.django_db() +@pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") +def test_django_sqli_query_no_redacted(client, test_spans, tracer): + root_span, response = _aux_appsec_get_root_span( + client, + test_spans, + tracer, + url="/appsec/sqli_query_no_redacted/?q=sqlite_master", + ) + + vuln_type = "SQL_INJECTION" + + assert response.status_code == 200 + assert response.content == b"OK" + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + + line, hash_value = get_line_and_hash("sqli_query_no_redacted", vuln_type, filename=TEST_FILE) + + assert loaded["sources"] == [ + { + "name": "q", + "origin": "http.request.parameter", + "value": "sqlite_master", + } + ] + + assert loaded["vulnerabilities"][0]["type"] == vuln_type + assert loaded["vulnerabilities"][0]["evidence"] == { + "valueParts": [ + {"value": "SELECT * FROM "}, + {"source": 0, "value": "sqlite_master"}, + {"value": " ORDER BY name"}, + ] + } + assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE + assert loaded["vulnerabilities"][0]["location"]["line"] == line + assert loaded["vulnerabilities"][0]["hash"] == hash_value + + @pytest.mark.django_db() @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_sqli_http_request_header_value(client, test_spans, tracer): diff --git a/tests/appsec/integrations/fastapi_tests/test_fastapi_appsec_iast.py b/tests/appsec/integrations/fastapi_tests/test_fastapi_appsec_iast.py index 07d8e1c9dc6..ee0b4e041d1 100644 --- a/tests/appsec/integrations/fastapi_tests/test_fastapi_appsec_iast.py +++ b/tests/appsec/integrations/fastapi_tests/test_fastapi_appsec_iast.py @@ -1003,10 +1003,6 @@ async def test_route(request: Request): with override_global_config(dict(_iast_enabled=True, _iast_request_sampling=100.0)): patch_iast({"xss": True}) - from jinja2.filters import FILTERS - from jinja2.filters import do_mark_safe - - FILTERS["safe"] = do_mark_safe _aux_appsec_prepare_tracer(tracer) resp = client.get( "/index.html?iast_queryparam=test1234", diff --git a/tests/appsec/integrations/flask_tests/mini.py b/tests/appsec/integrations/flask_tests/mini.py new file mode 100644 index 00000000000..5254d2ad5bd --- /dev/null +++ b/tests/appsec/integrations/flask_tests/mini.py @@ -0,0 +1,36 @@ +import ddtrace.auto # noqa: F401 + + +"""do not move this import""" + +import os # noqa: E402 +import sys # noqa: E402 + +from flask import Flask # noqa: E402 +import requests # noqa: E402 F401 + +from ddtrace.settings.asm import config as asm_config # noqa: E402 +from ddtrace.version import get_version # noqa: E402 + + +app = Flask(__name__) + + +@app.route("/") +def hello_world(): + res = [] + for m in sys.modules: + if m.startswith("ddtrace.appsec"): + res.append(m) + return { + "appsec": list(sorted(res)), + "asm_config": { + k: getattr(asm_config, k) for k in dir(asm_config) if isinstance(getattr(asm_config, k), (int, bool, float)) + }, + "aws": "AWS_LAMBDA_FUNCTION_NAME" in os.environ, + "version": get_version(), + } + + +if __name__ == "__main__": + app.run(debug=True, port=8475) diff --git a/tests/appsec/integrations/flask_tests/test_appsec_loading_modules.py b/tests/appsec/integrations/flask_tests/test_appsec_loading_modules.py new file mode 100644 index 00000000000..e989ee5d612 --- /dev/null +++ b/tests/appsec/integrations/flask_tests/test_appsec_loading_modules.py @@ -0,0 +1,84 @@ +import json +import os +import pathlib +import subprocess +import time +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.request import urlopen + +import pytest + + +MODULES_ALWAYS_LOADED = ["ddtrace.appsec", "ddtrace.appsec._capabilities", "ddtrace.appsec._constants"] +MODULE_ASM_ONLY = ["ddtrace.appsec._processor", "ddtrace.appsec._ddwaf"] +MODULE_IAST_ONLY = [ + "ddtrace.appsec._iast", + "ddtrace.appsec._iast._taint_tracking._native", + "ddtrace.appsec._iast._stacktrace", +] + + +@pytest.mark.parametrize("appsec_enabled", ["true", "false"]) +@pytest.mark.parametrize("iast_enabled", ["true", None]) +@pytest.mark.parametrize("aws_lambda", ["any", None]) +def test_loading(appsec_enabled, iast_enabled, aws_lambda): + flask_app = pathlib.Path(__file__).parent / "mini.py" + env = os.environ.copy() + if appsec_enabled: + env["DD_APPSEC_ENABLED"] = appsec_enabled + else: + env.pop("DD_APPSEC_ENABLED", None) + if iast_enabled: + env["DD_IAST_ENABLED"] = iast_enabled + else: + env.pop("DD_IAST_ENABLED", None) + if aws_lambda: + env["AWS_LAMBDA_FUNCTION_NAME"] = aws_lambda + else: + env.pop("AWS_LAMBDA_FUNCTION_NAME", None) + + process = subprocess.Popen( + ["python", str(flask_app)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + for i in range(16): + time.sleep(1) + try: + with urlopen("http://localhost:8475") as response: + assert response.status == 200 + payload = response.read().decode() + data = json.loads(payload) + assert "appsec" in data + # appsec is always enabled + for m in MODULES_ALWAYS_LOADED: + assert m in data["appsec"], f"{m} not in {data['appsec']}" + for m in MODULE_ASM_ONLY: + if appsec_enabled == "true" and not aws_lambda: + assert m in data["appsec"], f"{m} not in {data['appsec']}" + else: + assert m not in data["appsec"], f"{m} in {data['appsec']}" + for m in MODULE_IAST_ONLY: + if iast_enabled and not aws_lambda: + assert m in data["appsec"], f"{m} not in {data['appsec']}" + else: + assert m not in data["appsec"], f"{m} in {data['appsec']}" + process.terminate() + process.wait() + break + except HTTPError as e: + process.terminate() + process.wait() + raise AssertionError(e.read().decode()) + except URLError: + continue + except AssertionError: + process.terminate() + process.wait() + raise + else: + process.terminate() + process.wait() + raise AssertionError("Server did not start") diff --git a/tests/appsec/integrations/flask_tests/test_iast_flask.py b/tests/appsec/integrations/flask_tests/test_iast_flask.py index 0d8f7c5b4ad..6aa0558a737 100644 --- a/tests/appsec/integrations/flask_tests/test_iast_flask.py +++ b/tests/appsec/integrations/flask_tests/test_iast_flask.py @@ -51,10 +51,6 @@ def setUp(self): patch_header_injection() patch_xss_injection() patch_json() - from jinja2.filters import FILTERS - from jinja2.filters import do_mark_safe - - FILTERS["safe"] = do_mark_safe super(FlaskAppSecIASTEnabledTestCase, self).setUp() self.tracer._configure(api_version="v0.4", appsec_enabled=True, iast_enabled=True) oce.reconfigure() @@ -1804,18 +1800,19 @@ def test_sqli(): return "OK", 200 - if tuple(map(int, werkzeug_version.split("."))) >= (2, 3): - self.client.set_cookie(domain="localhost", key="sqlite_master", value="sqlite_master3") - else: - self.client.set_cookie(server_name="localhost", key="sqlite_master", value="sqlite_master3") + with override_global_config(dict(_iast_enabled=False)): + if tuple(map(int, werkzeug_version.split("."))) >= (2, 3): + self.client.set_cookie(domain="localhost", key="sqlite_master", value="sqlite_master3") + else: + self.client.set_cookie(server_name="localhost", key="sqlite_master", value="sqlite_master3") - resp = self.client.post("/sqli/cookies/") - assert resp.status_code == 200 + resp = self.client.post("/sqli/cookies/") + assert resp.status_code == 200 - root_span = self.pop_spans()[0] - assert root_span.get_metric(IAST.ENABLED) is None + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) is None - assert root_span.get_tag(IAST.JSON) is None + assert root_span.get_tag(IAST.JSON) is None @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_flask_full_sqli_iast_disabled_http_request_header_getitem(self): diff --git a/tests/contrib/subprocess/test_subprocess.py b/tests/contrib/subprocess/test_subprocess.py index f9084d23db1..ff6b30bb0dd 100644 --- a/tests/contrib/subprocess/test_subprocess.py +++ b/tests/contrib/subprocess/test_subprocess.py @@ -253,7 +253,7 @@ def test_unpatch(tracer): assert span.get_tag(COMMANDS.SHELL) == "dir -l /" unpatch() - with override_global_config(dict(_asm_enabled=True)): + with override_global_config(dict(_ep_enabled=False)): Pin.get_from(os)._clone(tracer=tracer).onto(os) with tracer.trace("os.system_unpatch"): ret = os.system("dir -l /") @@ -273,7 +273,7 @@ def test_unpatch(tracer): def test_ossystem_noappsec(tracer): - with override_global_config(dict(_asm_enabled=False)): + with override_global_config(dict(_ep_enabled=False)): patch() assert not hasattr(os.system, "__wrapped__") assert not hasattr(os._spawnvef, "__wrapped__") diff --git a/tests/contrib/subprocess/test_subprocess_patch.py b/tests/contrib/subprocess/test_subprocess_patch.py index 57778f798c1..471f096fbae 100644 --- a/tests/contrib/subprocess/test_subprocess_patch.py +++ b/tests/contrib/subprocess/test_subprocess_patch.py @@ -19,6 +19,8 @@ class TestSubprocessPatch(PatchTestCase.Base): def __init__(self, *args, **kwargs): asm_config._asm_enabled = True + asm_config._ep_enabled = True + asm_config._load_modules = True super(TestSubprocessPatch, self).__init__(*args, **kwargs) def assert_module_patched(self, subprocess): diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index c49f3e79dbf..2f857e300fd 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -202,8 +202,6 @@ components: - ddtrace/contrib/internal/redis/* - ddtrace/contrib/_aredis.py - ddtrace/contrib/internal/aredis/* - - ddtrace/contrib/_yaaredis.py - - ddtrace/contrib/internal/yaaredis/* - ddtrace/_trace/utils_redis.py - ddtrace/contrib/internal/redis_utils.py - ddtrace/ext/redis.py @@ -1167,20 +1165,6 @@ suites: - tests/snapshots/tests.contrib.wsgi.* runner: riot snapshot: true - yaaredis: - paths: - - '@core' - - '@bootstrap' - - '@contrib' - - '@tracing' - - '@redis' - - tests/contrib/yaaredis/* - - tests/snapshots/tests.contrib.yaaredis.* - pattern: yaaredis$ - runner: riot - services: - - redis - snapshot: true valkey: parallelism: 5 paths: diff --git a/tests/contrib/yaaredis/__init__.py b/tests/contrib/yaaredis/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/contrib/yaaredis/test_yaaredis.py b/tests/contrib/yaaredis/test_yaaredis.py deleted file mode 100644 index df064817aef..00000000000 --- a/tests/contrib/yaaredis/test_yaaredis.py +++ /dev/null @@ -1,237 +0,0 @@ -# -*- encoding: utf-8 -*- -import os -import uuid - -import pytest -from wrapt import ObjectProxy -import yaaredis - -from ddtrace.contrib.internal.yaaredis.patch import patch -from ddtrace.contrib.internal.yaaredis.patch import unpatch -from ddtrace.trace import Pin -from tests.opentracer.utils import init_tracer -from tests.utils import override_config - -from ..config import REDIS_CONFIG - - -@pytest.fixture(autouse=True) -async def traced_yaaredis(): - r = yaaredis.StrictRedis(port=REDIS_CONFIG["port"]) - await r.flushall() - - patch() - try: - yield r - finally: - unpatch() - - r = yaaredis.StrictRedis(port=REDIS_CONFIG["port"]) - await r.flushall() - - -def test_patching(): - """ - When patching yaaredis library - We wrap the correct methods - When unpatching yaaredis library - We unwrap the correct methods - """ - assert isinstance(yaaredis.client.StrictRedis.execute_command, ObjectProxy) - assert isinstance(yaaredis.client.StrictRedis.pipeline, ObjectProxy) - assert isinstance(yaaredis.pipeline.StrictPipeline.execute, ObjectProxy) - assert isinstance(yaaredis.pipeline.StrictPipeline.immediate_execute_command, ObjectProxy) - - unpatch() - - assert not isinstance(yaaredis.client.StrictRedis.execute_command, ObjectProxy) - assert not isinstance(yaaredis.client.StrictRedis.pipeline, ObjectProxy) - assert not isinstance(yaaredis.pipeline.StrictPipeline.execute, ObjectProxy) - assert not isinstance(yaaredis.pipeline.StrictPipeline.immediate_execute_command, ObjectProxy) - - -@pytest.mark.asyncio -async def test_long_command(snapshot_context, traced_yaaredis): - with snapshot_context(): - await traced_yaaredis.mget(*range(1000)) - - -@pytest.mark.asyncio -@pytest.mark.snapshot -async def test_cmd_max_length(traced_yaaredis): - with override_config("yaaredis", dict(cmd_max_length=7)): - await traced_yaaredis.get("here-is-a-long-key") - - -@pytest.mark.skip(reason="No traces sent to the test agent") -@pytest.mark.subprocess(env=dict(DD_YAAREDIS_CMD_MAX_LENGTH="10"), ddtrace_run=True) -@pytest.mark.snapshot -def test_cmd_max_length_env(): - import asyncio - - import yaaredis - - from tests.contrib.config import REDIS_CONFIG - - async def main(): - r = yaaredis.StrictRedis(port=REDIS_CONFIG["port"]) - await r.get("here-is-a-long-key") - - asyncio.run(main()) - - -@pytest.mark.asyncio -async def test_basics(snapshot_context, traced_yaaredis): - with snapshot_context(): - await traced_yaaredis.get("cheese") - - -@pytest.mark.asyncio -async def test_unicode(snapshot_context, traced_yaaredis): - with snapshot_context(): - await traced_yaaredis.get("😐") - - -@pytest.mark.asyncio -async def test_pipeline_traced(snapshot_context, traced_yaaredis): - with snapshot_context(): - p = await traced_yaaredis.pipeline(transaction=False) - await p.set("blah", 32) - await p.rpush("foo", "éé") - await p.hgetall("xxx") - await p.execute() - - -@pytest.mark.asyncio -async def test_pipeline_immediate(snapshot_context, traced_yaaredis): - with snapshot_context(): - p = await traced_yaaredis.pipeline() - await p.set("a", 1) - await p.immediate_execute_command("SET", "a", 1) - await p.execute() - - -@pytest.mark.asyncio -async def test_meta_override(tracer, test_spans, traced_yaaredis): - pin = Pin.get_from(traced_yaaredis) - assert pin is not None - pin._clone(tags={"cheese": "camembert"}, tracer=tracer).onto(traced_yaaredis) - - await traced_yaaredis.get("cheese") - test_spans.assert_trace_count(1) - test_spans.assert_span_count(1) - assert test_spans.spans[0].service == "redis" - assert test_spans.spans[0].get_tag("component") == "yaaredis" - assert test_spans.spans[0].get_tag("span.kind") == "client" - assert test_spans.spans[0].get_tag("db.system") == "redis" - assert "cheese" in test_spans.spans[0].get_tags() and test_spans.spans[0].get_tag("cheese") == "camembert" - - -@pytest.mark.asyncio -async def test_service_name(tracer, test_spans, traced_yaaredis): - service = str(uuid.uuid4()) - Pin._override(traced_yaaredis, service=service, tracer=tracer) - - await traced_yaaredis.set("cheese", "1") - test_spans.assert_trace_count(1) - test_spans.assert_span_count(1) - assert test_spans.spans[0].service == service - - -@pytest.mark.asyncio -async def test_service_name_config(tracer, test_spans, traced_yaaredis): - service = str(uuid.uuid4()) - with override_config("yaaredis", dict(service=service)): - Pin._override(traced_yaaredis, tracer=tracer) - await traced_yaaredis.set("cheese", "1") - test_spans.assert_trace_count(1) - test_spans.assert_span_count(1) - assert test_spans.spans[0].service == service - - -@pytest.mark.asyncio -async def test_opentracing(tracer, snapshot_context, traced_yaaredis): - """Ensure OpenTracing works with redis.""" - - with snapshot_context(): - pin = Pin.get_from(traced_yaaredis) - ot_tracer = init_tracer("redis_svc", pin.tracer) - - with ot_tracer.start_active_span("redis_get"): - await traced_yaaredis.get("cheese") - - -@pytest.mark.parametrize( - "service_schema", - [ - (None, None), - (None, "v0"), - (None, "v1"), - ("mysvc", None), - ("mysvc", "v0"), - ("mysvc", "v1"), - ], -) -@pytest.mark.snapshot() -def test_schematization(ddtrace_run_python_code_in_subprocess, service_schema): - service, schema = service_schema - code = """ -import sys - -import pytest - -from tests.contrib.yaaredis.test_yaaredis import traced_yaaredis - -@pytest.mark.asyncio -async def test_basics(traced_yaaredis): - async for client in traced_yaaredis: - await client.get("cheese") - - -if __name__ == "__main__": - sys.exit(pytest.main(["-x", __file__])) - """ - env = os.environ.copy() - if service: - env["DD_SERVICE"] = service - if schema: - env["DD_TRACE_SPAN_ATTRIBUTE_SCHEMA"] = schema - out, err, status, _ = ddtrace_run_python_code_in_subprocess(code, env=env) - assert status == 0, (err.decode(), out.decode()) - assert err == b"", err.decode() - - -@pytest.mark.subprocess(env=dict(DD_REDIS_RESOURCE_ONLY_COMMAND="false")) -@pytest.mark.snapshot -def test_full_command_in_resource_env(): - import asyncio - - import yaaredis - - import ddtrace - from tests.contrib.config import REDIS_CONFIG - - async def traced_client(): - with ddtrace.tracer.trace("web-request", service="test"): - redis_client = yaaredis.StrictRedis(port=REDIS_CONFIG["port"]) - await redis_client.get("put_key_in_resource") - p = await redis_client.pipeline(transaction=False) - await p.set("pipeline-cmd1", 1) - await p.set("pipeline-cmd2", 2) - await p.execute() - - ddtrace.patch(yaaredis=True) - asyncio.run(traced_client()) - - -@pytest.mark.snapshot -@pytest.mark.asyncio -@pytest.mark.parametrize("use_global_tracer", [True]) -async def test_full_command_in_resource_config(tracer, traced_yaaredis): - with override_config("yaaredis", dict(resource_only_command=False)): - with tracer.trace("web-request", service="test"): - await traced_yaaredis.get("put_key_in_resource") - p = await traced_yaaredis.pipeline(transaction=False) - await p.set("pipeline-cmd1", 1) - await p.set("pipeline-cmd2", 2) - await p.execute() diff --git a/tests/contrib/yaaredis/test_yaaredis_patch.py b/tests/contrib/yaaredis/test_yaaredis_patch.py deleted file mode 100644 index d93247a1faa..00000000000 --- a/tests/contrib/yaaredis/test_yaaredis_patch.py +++ /dev/null @@ -1,31 +0,0 @@ -# This test script was automatically generated by the contrib-patch-tests.py -# script. If you want to make changes to it, you should make sure that you have -# removed the ``_generated`` suffix from the file name, to prevent the content -# from being overwritten by future re-generations. - -from ddtrace.contrib.internal.yaaredis.patch import get_version -from ddtrace.contrib.internal.yaaredis.patch import patch - - -try: - from ddtrace.contrib.internal.yaaredis.patch import unpatch -except ImportError: - unpatch = None -from tests.contrib.patch import PatchTestCase - - -class TestYaaredisPatch(PatchTestCase.Base): - __integration_name__ = "yaaredis" - __module_name__ = "yaaredis" - __patch_func__ = patch - __unpatch_func__ = unpatch - __get_version__ = get_version - - def assert_module_patched(self, yaaredis): - pass - - def assert_not_module_patched(self, yaaredis): - pass - - def assert_not_module_double_patched(self, yaaredis): - pass diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_with_rate.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_with_rate.json deleted file mode 100644 index 2a94b63ab1c..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_with_rate.json +++ /dev/null @@ -1,38 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_dd1.sr.eausr": 0.5, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 2 - }, - "duration": 309666, - "start": 1692650065803420466 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_without_rate.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_without_rate.json deleted file mode 100644 index 40703c30619..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_analytics_without_rate.json +++ /dev/null @@ -1,38 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_dd1.sr.eausr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 2 - }, - "duration": 277750, - "start": 1692650065792499174 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_basics.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_basics.json deleted file mode 100644 index 64a0dc8fcee..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_basics.json +++ /dev/null @@ -1,37 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 2 - }, - "duration": 199667, - "start": 1692650065770875215 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_cmd_max_length.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_cmd_max_length.json deleted file mode 100644 index b39a54205e8..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_cmd_max_length.json +++ /dev/null @@ -1,37 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET here...", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 2 - }, - "duration": 227917, - "start": 1692650065760107632 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_config[True].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_config[True].json deleted file mode 100644 index f453d45ceb2..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_config[True].json +++ /dev/null @@ -1,82 +0,0 @@ -[[ - { - "name": "web-request", - "service": "test", - "resource": "web-request", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "language": "python", - "runtime-id": "8684af00a9414982b4794ddcadcd26ec" - }, - "metrics": { - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "process_id": 78504 - }, - "duration": 1603000, - "start": 1698860084361738000 - }, - { - "name": "redis.command", - "service": "redis", - "resource": "GET put_key_in_resource", - "trace_id": 0, - "span_id": 2, - "parent_id": 1, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "component": "yaaredis", - "db.system": "redis", - "out.host": "localhost", - "redis.raw_command": "GET put_key_in_resource", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "redis.args_length": 2 - }, - "duration": 659000, - "start": 1698860084362141000 - }, - { - "name": "redis.command", - "service": "redis", - "resource": "SET pipeline-cmd1 1\nSET pipeline-cmd2 2", - "trace_id": 0, - "span_id": 3, - "parent_id": 1, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "component": "yaaredis", - "db.system": "redis", - "out.host": "localhost", - "redis.raw_command": "SET pipeline-cmd1 1\nSET pipeline-cmd2 2", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "redis.pipeline_length": 2 - }, - "duration": 442000, - "start": 1698860084362884000 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_env.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_env.json deleted file mode 100644 index 09c08bf56a0..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_full_command_in_resource_env.json +++ /dev/null @@ -1,82 +0,0 @@ -[[ - { - "name": "web-request", - "service": "test", - "resource": "web-request", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "_dd.p.dm": "-0", - "language": "python", - "runtime-id": "3c2060b13ff1469387b2c823d7d43f18" - }, - "metrics": { - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "process_id": 78507 - }, - "duration": 14945000, - "start": 1698860084269055000 - }, - { - "name": "redis.command", - "service": "redis", - "resource": "GET put_key_in_resource", - "trace_id": 0, - "span_id": 2, - "parent_id": 1, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "component": "yaaredis", - "db.system": "redis", - "out.host": "localhost", - "redis.raw_command": "GET put_key_in_resource", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "redis.args_length": 2 - }, - "duration": 12481000, - "start": 1698860084269179000 - }, - { - "name": "redis.command", - "service": "redis", - "resource": "SET pipeline-cmd1 1\nSET pipeline-cmd2 2", - "trace_id": 0, - "span_id": 3, - "parent_id": 1, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "component": "yaaredis", - "db.system": "redis", - "out.host": "localhost", - "redis.raw_command": "SET pipeline-cmd1 1\nSET pipeline-cmd2 2", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "redis.pipeline_length": 2 - }, - "duration": 2254000, - "start": 1698860084281737000 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_long_command.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_long_command.json deleted file mode 100644 index c21f8fc51f5..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_long_command.json +++ /dev/null @@ -1,37 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "MGET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "MGET 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 36...", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 1001 - }, - "duration": 4141542, - "start": 1692650065704941632 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_opentracing.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_opentracing.json deleted file mode 100644 index ccee94088be..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_opentracing.json +++ /dev/null @@ -1,56 +0,0 @@ -[[ - { - "name": "redis_get", - "service": "redis_svc", - "resource": "redis_get", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "language": "python", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596" - }, - "metrics": { - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "process_id": 5119 - }, - "duration": 402211, - "start": 1692650065865079296 - }, - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 2, - "parent_id": 1, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "redis.args_length": 2 - }, - "duration": 271333, - "start": 1692650065865181924 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_immediate.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_immediate.json deleted file mode 100644 index d8843f3ac0c..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_immediate.json +++ /dev/null @@ -1,72 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "SET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "SET a 1", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 3 - }, - "duration": 246375, - "start": 1692650065826740341 - }], -[ - { - "name": "redis.command", - "service": "redis", - "resource": "SET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "SET a 1", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.pipeline_length": 1 - }, - "duration": 181666, - "start": 1692650065827078091 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_traced.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_traced.json deleted file mode 100644 index 945d5fc508d..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_pipeline_traced.json +++ /dev/null @@ -1,36 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "SET\nRPUSH\nHGETALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "SET blah 32\nRPUSH foo \u00e9\u00e9\nHGETALL xxx", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.pipeline_length": 3 - }, - "duration": 278208, - "start": 1692650065814935466 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema0].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema0].json deleted file mode 100644 index d541510cbcd..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema0].json +++ /dev/null @@ -1,73 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "FLUSHALL", - "runtime-id": "392b844b532747b7a2327f4996a26d26", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5140, - "redis.args_length": 1 - }, - "duration": 17837625, - "start": 1692650066908323174 - }], -[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "392b844b532747b7a2327f4996a26d26", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5140, - "redis.args_length": 2 - }, - "duration": 308291, - "start": 1692650066953965133 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema1].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema1].json deleted file mode 100644 index f0075f33b51..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema1].json +++ /dev/null @@ -1,73 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "FLUSHALL", - "runtime-id": "f9e411adfc7a4c2cabc38392dd511dbf", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5145, - "redis.args_length": 1 - }, - "duration": 13703875, - "start": 1692650068046894758 - }], -[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "ddtrace_subprocess_dir", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "f9e411adfc7a4c2cabc38392dd511dbf", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5145, - "redis.args_length": 2 - }, - "duration": 337542, - "start": 1692650068093153508 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema2].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema2].json deleted file mode 100644 index f1067bca0f9..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema2].json +++ /dev/null @@ -1,75 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "ddtrace_subprocess_dir", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "_dd.peer.service.source": "out.host", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "peer.service": "localhost", - "redis.command": "FLUSHALL", - "runtime-id": "6512d363c3094051ae21a945fbfcf82b", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 14223, - "redis.args_length": 1 - }, - "duration": 4711000, - "start": 1685545835531199000 - }], -[ - { - "name": "redis.command", - "service": "ddtrace_subprocess_dir", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "_dd.peer.service.source": "out.host", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "peer.service": "localhost", - "redis.command": "GET cheese", - "runtime-id": "6512d363c3094051ae21a945fbfcf82b", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 14223, - "redis.args_length": 2 - }, - "duration": 420000, - "start": 1685545835574013000 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema3].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema3].json deleted file mode 100644 index 94285cbff91..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema3].json +++ /dev/null @@ -1,73 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "mysvc", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "FLUSHALL", - "runtime-id": "b99b2a231e9242f5b3e0fb971df07ef8", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5155, - "redis.args_length": 1 - }, - "duration": 9568708, - "start": 1692650070456463426 - }], -[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "mysvc", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "b99b2a231e9242f5b3e0fb971df07ef8", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5155, - "redis.args_length": 2 - }, - "duration": 304459, - "start": 1692650070493672884 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema4].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema4].json deleted file mode 100644 index e0a5fe04089..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema4].json +++ /dev/null @@ -1,73 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "mysvc", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "FLUSHALL", - "runtime-id": "d8705f1ada1545908f6144a2d6b15900", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5160, - "redis.args_length": 1 - }, - "duration": 10873959, - "start": 1692650071606791218 - }], -[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "mysvc", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET cheese", - "runtime-id": "d8705f1ada1545908f6144a2d6b15900", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5160, - "redis.args_length": 2 - }, - "duration": 328375, - "start": 1692650071643705427 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema5].json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema5].json deleted file mode 100644 index 6be40ba43d8..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_schematization[service_schema5].json +++ /dev/null @@ -1,75 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "mysvc", - "resource": "FLUSHALL", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "_dd.peer.service.source": "out.host", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "peer.service": "localhost", - "redis.command": "FLUSHALL", - "runtime-id": "1993bf3ea65c4b658bffdda053387eca", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 14262, - "redis.args_length": 1 - }, - "duration": 7419000, - "start": 1685545839462874000 - }], -[ - { - "name": "redis.command", - "service": "mysvc", - "resource": "GET", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "_dd.peer.service.source": "out.host", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "peer.service": "localhost", - "redis.command": "GET cheese", - "runtime-id": "1993bf3ea65c4b658bffdda053387eca", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 14262, - "redis.args_length": 2 - }, - "duration": 326000, - "start": 1685545839502613000 - }]] diff --git a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_unicode.json b/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_unicode.json deleted file mode 100644 index cb687c7f283..00000000000 --- a/tests/snapshots/tests.contrib.yaaredis.test_yaaredis.test_unicode.json +++ /dev/null @@ -1,37 +0,0 @@ -[[ - { - "name": "redis.command", - "service": "redis", - "resource": "GET", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "redis", - "error": 0, - "meta": { - "_dd.base_service": "tests.contrib.yaaredis", - "_dd.p.dm": "-0", - "_dd.p.tid": "654a694400000000", - "component": "yaaredis", - "db.system": "redis", - "language": "python", - "out.host": "localhost", - "redis.raw_command": "GET \ud83d\ude10", - "runtime-id": "4d0d479c17be4095b8e0dee5a0839596", - "server.address": "localhost", - "span.kind": "client" - }, - "metrics": { - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "db.row_count": 0, - "network.destination.port": 6379, - "out.redis_db": 0, - "process_id": 5119, - "redis.args_length": 2 - }, - "duration": 211291, - "start": 1692650065781665049 - }]] diff --git a/tests/tracer/test_logger.py b/tests/tracer/test_logger.py index 00edeeb18b2..55a712d1ac0 100644 --- a/tests/tracer/test_logger.py +++ b/tests/tracer/test_logger.py @@ -3,7 +3,8 @@ import mock import pytest -from ddtrace.internal.logger import DDLogger +import ddtrace.internal.logger +from ddtrace.internal.logger import LoggingBucket from ddtrace.internal.logger import get_logger from tests.utils import BaseTestCase @@ -11,22 +12,28 @@ ALL_LEVEL_NAMES = ("debug", "info", "warning", "error", "exception", "critical", "fatal") -class DDLoggerTestCase(BaseTestCase): +class LoggerTestCase(BaseTestCase): def setUp(self): - super(DDLoggerTestCase, self).setUp() + super(LoggerTestCase, self).setUp() - self.root = logging.root - self.manager = self.root.manager + self.manager = logging.root.manager + + # Reset to default values + ddtrace.internal.logger._buckets.clear() + ddtrace.internal.logger._rate_limit = 60 def tearDown(self): # Weeee, forget all existing loggers logging.Logger.manager.loggerDict.clear() self.assertEqual(logging.Logger.manager.loggerDict, dict()) - self.root = None self.manager = None - super(DDLoggerTestCase, self).tearDown() + # Reset to default values + ddtrace.internal.logger._buckets.clear() + ddtrace.internal.logger._rate_limit = 60 + + super(LoggerTestCase, self).tearDown() def _make_record( self, @@ -42,42 +49,31 @@ def _make_record( ): return logger.makeRecord(logger.name, level, fn, lno, msg, args, exc_info, func, extra) - @mock.patch("ddtrace.internal.logger.DDLogger.handle") - def assert_log_records(self, log, expected_levels, handle): - for name in ALL_LEVEL_NAMES: - method = getattr(log, name) - method("test") - - records = [args[0][0] for args in handle.call_args_list] - for record in records: - self.assertIsInstance(record, logging.LogRecord) - self.assertTrue("test.logger" in record.name or "ddtrace" in record.name) - - levels = [r.levelname for r in records] - self.assertEqual(levels, expected_levels) - def test_get_logger(self): """ When using `get_logger` to get a logger When the logger does not exist - We create a new DDLogger + We create a new logging.Logger When the logger exists We return the expected logger When a different logger is requested - We return a new DDLogger + We return a new logging.Logger When a Placeholder exists - We return DDLogger + We return logging.Logger """ + assert self.manager is not None + # Assert the logger doesn't already exist self.assertNotIn("test.logger", self.manager.loggerDict) # Fetch a new logger log = get_logger("test.logger") + assert ddtrace.internal.logger.log_filter in log.filters self.assertEqual(log.name, "test.logger") self.assertEqual(log.level, logging.NOTSET) - # Ensure it is a DDLogger - self.assertIsInstance(log, DDLogger) + # Ensure it is a logging.Logger + self.assertIsInstance(log, logging.Logger) # Make sure it is stored in all the places we expect self.assertEqual(self.manager.getLogger("test.logger"), log) self.assertEqual(self.manager.loggerDict["test.logger"], log) @@ -93,163 +89,49 @@ def test_get_logger(self): self.assertNotEqual(log, new_log) # If a PlaceHolder is in place of the logger - # We should return the DDLogger + # We should return the logging.Logger self.assertIsInstance(self.manager.loggerDict["new.test"], logging.PlaceHolder) log = get_logger("new.test") self.assertEqual(log.name, "new.test") - self.assertIsInstance(log, DDLogger) - - def test_get_logger_children(self): - """ - When using `get_logger` to get a logger - We appropriately assign children loggers - - DEV: This test case is to ensure we are calling `manager._fixupChildren(logger)` - """ - root = get_logger("test") - root.setLevel(logging.WARNING) - - child_logger = get_logger("test.newplaceholder.long.component") - self.assertEqual(child_logger.parent, root) - - parent_logger = get_logger("test.newplaceholder") - self.assertEqual(child_logger.parent, parent_logger) - - parent_logger.setLevel(logging.INFO) - # Because the child logger's level remains unset, it should inherit - # the level of its closest parent, which is INFO. - # If we did not properly maintain the logger tree, this would fail - # because child_logger would be set to the default when it was created - # which was logging.WARNING. - self.assertEqual(child_logger.getEffectiveLevel(), logging.INFO) - - # Clean up for future tests. - root.setLevel(logging.NOTSET) - - def test_get_logger_parents(self): - """ - When using `get_logger` to get a logger - We appropriately assign parent loggers - - DEV: This test case is to ensure we are calling `manager._fixupParents(logger)` - """ - # Fetch a new logger - test_log = get_logger("test") - self.assertEqual(test_log.parent, self.root) - - # Fetch a new child log - # Auto-associate with parent `test` logger - child_log = get_logger("test.child") - self.assertEqual(child_log.parent, test_log) - - # Deep child - deep_log = get_logger("test.child.logger.from.test.case") - self.assertEqual(deep_log.parent, child_log) - - def test_logger_init(self): - """ - When creating a new DDLogger - Has the same interface as logging.Logger - Configures a defaultdict for buckets - Properly configures the rate limit - """ - # Create a logger - log = DDLogger("test.logger") - - # Ensure we set the name and use default log level - self.assertEqual(log.name, "test.logger") - self.assertEqual(log.level, logging.NOTSET) - - # Assert DDLogger default properties - self.assertIsInstance(log.buckets, dict) - self.assertEqual(log.rate_limit, 60) - - # Assert manager and parent - # DEV: Parent is `None` because `manager._findParents()` doesn't get called - # unless we use `get_logger` (this is the same behavior as `logging.getLogger` and `Logger('name')`) - self.assertEqual(log.manager, self.manager) - self.assertIsNone(log.parent) - - # Override rate limit from environment variable - with self.override_env(dict(DD_TRACE_LOGGING_RATE="10")): - log = DDLogger("test.logger") - self.assertEqual(log.rate_limit, 10) - - # Set specific log level - log = DDLogger("test.logger", level=logging.DEBUG) - self.assertEqual(log.level, logging.DEBUG) - - def test_logger_log(self): - """ - When calling `DDLogger` log methods - We call `DDLogger.handle` with the expected log record - """ - log = get_logger("test.logger") - - # -- NOTSET - # By default no level is set so we only get warn, error, and critical messages - self.assertEqual(log.level, logging.NOTSET) - # `log.warning`, `log.error`, `log.exception`, `log.critical`, `log.fatal` - self.assert_log_records(log, ["WARNING", "ERROR", "ERROR", "CRITICAL", "CRITICAL"]) - - # -- CRITICAL - log.setLevel(logging.CRITICAL) - # `log.critical`, `log.fatal` - self.assert_log_records(log, ["CRITICAL", "CRITICAL"]) - - # -- ERROR - log.setLevel(logging.ERROR) - # `log.error`, `log.exception`, `log.critical`, `log.fatal` - self.assert_log_records(log, ["ERROR", "ERROR", "CRITICAL", "CRITICAL"]) - - # -- WARN - log.setLevel(logging.WARN) - # `log.warning`, `log.error`, `log.exception`, `log.critical`, `log.fatal` - self.assert_log_records(log, ["WARNING", "ERROR", "ERROR", "CRITICAL", "CRITICAL"]) + self.assertIsInstance(log, logging.Logger) - # -- INFO - log.setLevel(logging.INFO) - # `log.info`, `log.warning`, `log.error`, `log.exception`, `log.critical`, `log.fatal` - self.assert_log_records(log, ["INFO", "WARNING", "ERROR", "ERROR", "CRITICAL", "CRITICAL"]) - - # -- DEBUG - log.setLevel(logging.DEBUG) - # `log.debug`, `log.info`, `log.warning`, `log.error`, `log.exception`, `log.critical`, `log.fatal` - self.assert_log_records(log, ["DEBUG", "INFO", "WARNING", "ERROR", "ERROR", "CRITICAL", "CRITICAL"]) - - @mock.patch("logging.Logger.handle") - def test_logger_handle_no_limit(self, base_handle): + @mock.patch("logging.Logger.callHandlers") + def test_logger_handle_no_limit(self, call_handlers): """ - Calling `DDLogger.handle` + Calling `logging.Logger.handle` When no rate limit is set Always calls the base `Logger.handle` """ # Configure an INFO logger with no rate limit log = get_logger("test.logger") log.setLevel(logging.INFO) - log.rate_limit = 0 + ddtrace.internal.logger._rate_limit = 0 # Log a bunch of times very quickly (this is fast) for _ in range(1000): log.info("test") # Assert that we did not perform any rate limiting - self.assertEqual(base_handle.call_count, 1000) + self.assertEqual(call_handlers.call_count, 1000) # Our buckets are empty - self.assertEqual(log.buckets, dict()) + self.assertEqual(ddtrace.internal.logger._buckets, dict()) - @mock.patch("logging.Logger.handle") - def test_logger_handle_debug(self, base_handle): + @mock.patch("logging.Logger.callHandlers") + def test_logger_handle_debug(self, call_handlers): """ - Calling `DDLogger.handle` + Calling `logging.Logger.handle` When effective level is DEBUG Always calls the base `Logger.handle` """ + # Our buckets are empty + self.assertEqual(ddtrace.internal.logger._buckets, dict()) + # Configure an INFO logger with no rate limit log = get_logger("test.logger") log.setLevel(logging.DEBUG) - assert log.rate_limit > 0 + assert log.getEffectiveLevel() == logging.DEBUG + assert ddtrace.internal.logger._rate_limit > 0 # Log a bunch of times very quickly (this is fast) for level in ALL_LEVEL_NAMES: @@ -259,15 +141,15 @@ def test_logger_handle_debug(self, base_handle): # Assert that we did not perform any rate limiting total = 1000 * len(ALL_LEVEL_NAMES) - self.assertTrue(total <= base_handle.call_count <= total + 1) + self.assertTrue(total <= call_handlers.call_count <= total + 1) # Our buckets are empty - self.assertEqual(log.buckets, dict()) + self.assertEqual(ddtrace.internal.logger._buckets, dict()) - @mock.patch("logging.Logger.handle") - def test_logger_handle_bucket(self, base_handle): + @mock.patch("logging.Logger.callHandlers") + def test_logger_handle_bucket(self, call_handlers): """ - When calling `DDLogger.handle` + When calling `logging.Logger.handle` With a record We pass it to the base `Logger.handle` We create a bucket for tracking @@ -279,22 +161,22 @@ def test_logger_handle_bucket(self, base_handle): log.handle(record) # We passed to base Logger.handle - base_handle.assert_called_once_with(record) + call_handlers.assert_called_once_with(record) # We added an bucket entry for this record key = (record.name, record.levelno, record.pathname, record.lineno) - logging_bucket = log.buckets.get(key) - self.assertIsInstance(logging_bucket, DDLogger.LoggingBucket) + logging_bucket = ddtrace.internal.logger._buckets.get(key) + self.assertIsInstance(logging_bucket, LoggingBucket) # The bucket entry is correct - expected_bucket = int(record.created / log.rate_limit) + expected_bucket = int(record.created / ddtrace.internal.logger._rate_limit) self.assertEqual(logging_bucket.bucket, expected_bucket) self.assertEqual(logging_bucket.skipped, 0) - @mock.patch("logging.Logger.handle") - def test_logger_handle_bucket_limited(self, base_handle): + @mock.patch("logging.Logger.callHandlers") + def test_logger_handle_bucket_limited(self, call_handlers): """ - When calling `DDLogger.handle` + When calling `logging.Logger.handle` With multiple records in a single time frame We pass only the first to the base `Logger.handle` We keep track of the number skipped @@ -302,7 +184,8 @@ def test_logger_handle_bucket_limited(self, base_handle): log = get_logger("test.logger") # Create log record and handle it - first_record = self._make_record(log, msg="first") + record = self._make_record(log, msg="first") + first_record = record log.handle(first_record) for _ in range(100): @@ -312,21 +195,22 @@ def test_logger_handle_bucket_limited(self, base_handle): log.handle(record) # We passed to base Logger.handle - base_handle.assert_called_once_with(first_record) + call_handlers.assert_called_once_with(first_record) # We added an bucket entry for these records key = (record.name, record.levelno, record.pathname, record.lineno) - logging_bucket = log.buckets.get(key) + logging_bucket = ddtrace.internal.logger._buckets.get(key) + assert logging_bucket is not None # The bucket entry is correct - expected_bucket = int(first_record.created / log.rate_limit) + expected_bucket = int(first_record.created / ddtrace.internal.logger._rate_limit) self.assertEqual(logging_bucket.bucket, expected_bucket) self.assertEqual(logging_bucket.skipped, 100) - @mock.patch("logging.Logger.handle") - def test_logger_handle_bucket_skipped_msg(self, base_handle): + @mock.patch("logging.Logger.callHandlers") + def test_logger_handle_bucket_skipped_msg(self, call_handlers): """ - When calling `DDLogger.handle` + When calling `logging.Logger.handle` When a bucket exists for a previous time frame We pass only the record to the base `Logger.handle` We update the record message to include the number of skipped messages @@ -340,15 +224,15 @@ def test_logger_handle_bucket_skipped_msg(self, base_handle): # Create a bucket entry for this record key = (record.name, record.levelno, record.pathname, record.lineno) - bucket = int(record.created / log.rate_limit) + bucket = int(record.created / ddtrace.internal.logger._rate_limit) # We want the time bucket to be for an older bucket - log.buckets[key] = DDLogger.LoggingBucket(bucket=bucket - 1, skipped=20) + ddtrace.internal.logger._buckets[key] = LoggingBucket(bucket=bucket - 1, skipped=20) # Handle our record log.handle(record) # We passed to base Logger.handle - base_handle.assert_called_once_with(record) + call_handlers.assert_called_once_with(record) self.assertEqual(record.msg, original_msg + ", %s additional messages skipped") self.assertEqual(record.args, original_args + (20,)) @@ -356,7 +240,7 @@ def test_logger_handle_bucket_skipped_msg(self, base_handle): def test_logger_handle_bucket_key(self): """ - When calling `DDLogger.handle` + When calling `logging.Logger.handle` With different log messages We use different buckets to limit them """ @@ -388,7 +272,7 @@ def get_key(record): all_records = (record1, record2, record3, record4, record5, record6) [log.handle(record) for record in all_records] - buckets = log.buckets + buckets = ddtrace.internal.logger._buckets # We have 6 records but only end up with 5 buckets self.assertEqual(len(buckets), 5) diff --git a/tests/tracer/test_propagation.py b/tests/tracer/test_propagation.py index f79116826d3..e34167ce04f 100644 --- a/tests/tracer/test_propagation.py +++ b/tests/tracer/test_propagation.py @@ -47,7 +47,6 @@ from tests.contrib.fastapi.conftest import test_spans as fastapi_test_spans # noqa:F401 from tests.contrib.fastapi.conftest import tracer # noqa:F401 -from ..utils import flaky from ..utils import override_env from ..utils import override_global_config @@ -837,7 +836,6 @@ def test_extract_128bit_trace_ids_tracecontext(): assert child_span.trace_id == trace_id -@flaky(1735812000, reason="FIXME: Failing due to the global tracer being used in all tests") def test_last_dd_span_id(): non_dd_remote_context = HTTPPropagator.extract( { diff --git a/tests/tracer/test_tracer.py b/tests/tracer/test_tracer.py index 85d8be52a36..3445dfbefb2 100644 --- a/tests/tracer/test_tracer.py +++ b/tests/tracer/test_tracer.py @@ -676,6 +676,9 @@ def test_tracer_shutdown_timeout(): mock_stop.assert_called_once_with(2) +@pytest.mark.subprocess( + err=b"Spans started after the tracer has been shut down will not be sent to the Datadog Agent.\n", +) def test_tracer_shutdown(): import mock @@ -691,26 +694,6 @@ def test_tracer_shutdown(): mock_write.assert_not_called() -def test_tracer_shutdown_warning(): - import logging - - import mock - - from ddtrace.trace import tracer as t - - t.shutdown() - - with mock.patch.object(logging.Logger, "warning") as mock_logger: - with t.trace("something"): - pass - - mock_logger.assert_has_calls( - [ - mock.call("Spans started after the tracer has been shut down will not be sent to the Datadog Agent."), - ] - ) - - @pytest.mark.skip(reason="Fails to Pickle RateLimiter in the Tracer") @pytest.mark.subprocess def test_tracer_fork(): diff --git a/tests/utils.py b/tests/utils.py index 5d94598ec4b..da37ea88387 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -171,7 +171,7 @@ def override_global_config(values): ddtrace.config._subscriptions = [] # Grab the current values of all keys originals = dict((key, getattr(ddtrace.config, key)) for key in global_config_keys) - asm_originals = dict((key, getattr(ddtrace.settings.asm.config, key)) for key in asm_config_keys) + asm_originals = dict((key, getattr(asm_config, key)) for key in asm_config_keys) # Override from the passed in keys for key, value in values.items(): @@ -180,9 +180,9 @@ def override_global_config(values): # rebuild asm config from env vars and global config for key, value in values.items(): if key in asm_config_keys: - setattr(ddtrace.settings.asm.config, key, value) + setattr(asm_config, key, value) # If ddtrace.settings.asm.config has changed, check _asm_can_be_enabled again - ddtrace.settings.asm.config._eval_asm_can_be_enabled() + asm_config._eval_asm_can_be_enabled() try: core.dispatch("test.config.override") yield @@ -191,9 +191,9 @@ def override_global_config(values): for key, value in originals.items(): setattr(ddtrace.config, key, value) - ddtrace.settings.asm.config.reset() + asm_config.reset() for key, value in asm_originals.items(): - setattr(ddtrace.settings.asm.config, key, value) + setattr(asm_config, key, value) ddtrace.config._reset() ddtrace.config._subscriptions = subscriptions