Merge branch 'main' into baptiste.foy/FA/library-config-libdatadog--2

BaptisteFoy · web-flow · commit 6671f3c7bd6a · 2025-02-10T14:15:33.000+01:00
diff --git a/ddtrace/debugging/_expressions.py b/ddtrace/debugging/_expressions.py
@@ -46,6 +46,7 @@
 from ddtrace.debugging._safety import safe_getitem
 from ddtrace.internal.compat import PYTHON_VERSION_INFO as PY
 from ddtrace.internal.logger import get_logger
+from ddtrace.internal.safety import _isinstance
 
 
 DDASTType = Union[Dict[str, Any], Dict[str, List[Any]], Any]
@@ -126,7 +127,7 @@ def _make_function(self, ast: DDASTType, args: Tuple[str, ...], name: str) -> Fu
         return FunctionType(abstract_code.to_code(), {}, name, (), None)
 
     def _make_lambda(self, ast: DDASTType) -> Callable[[Any, Any], Any]:
-        return self._make_function(ast, ("_dd_it", "_locals"), "<lambda>")
+        return self._make_function(ast, ("_dd_it", "_dd_key", "_dd_value", "_locals"), "<lambda>")
 
     def _compile_direct_predicate(self, ast: DDASTType) -> Optional[List[Instr]]:
         # direct_predicate       =>  {"<direct_predicate_type>": <predicate>}
@@ -200,12 +201,12 @@ def _compile_arg_predicate(self, ast: DDASTType) -> Optional[List[Instr]]:
             if ca is None:
                 raise ValueError("Invalid argument: %r" % a)
 
-            return self._call_function(
-                lambda i, c, _locals: f(c(_, _locals) for _ in i),
-                ca,
-                [Instr("LOAD_CONST", fb)],
-                [Instr("LOAD_FAST", "_locals")],
-            )
+            def coll_iter(it, cond, _locals):
+                if _isinstance(it, dict):
+                    return f(cond(k, k, v, _locals) for k, v in it.items())
+                return f(cond(e, None, None, _locals) for e in it)
+
+            return self._call_function(coll_iter, ca, [Instr("LOAD_CONST", fb)], [Instr("LOAD_FAST", "_locals")])
 
         if _type in {"startsWith", "endsWith"}:
             a, b = args
@@ -245,8 +246,8 @@ def _compile_direct_operation(self, ast: DDASTType) -> Optional[List[Instr]]:
             if not isinstance(arg, str):
                 return None
 
-            if arg == "@it":
-                return [Instr("LOAD_FAST", "_dd_it")]
+            if arg in {"@it", "@key", "@value"}:
+                return [Instr("LOAD_FAST", f"_dd_{arg[1:]}")]
 
             return self._call_function(
                 get_local, [Instr("LOAD_FAST", "_locals")], [Instr("LOAD_CONST", self.__ref__(arg))]
@@ -297,12 +298,12 @@ def _compile_arg_operation(self, ast: DDASTType) -> Optional[List[Instr]]:
             if ca is None:
                 raise ValueError("Invalid argument: %r" % a)
 
-            return self._call_function(
-                lambda i, c, _locals: type(i)(_ for _ in i if c(_, _locals)),
-                ca,
-                [Instr("LOAD_CONST", fb)],
-                [Instr("LOAD_FAST", "_locals")],
-            )
+            def coll_filter(it, cond, _locals):
+                if _isinstance(it, dict):
+                    return type(it)({k: v for k, v in it.items() if cond(k, k, v, _locals)})
+                return type(it)(e for e in it if cond(e, None, None, _locals))
+
+            return self._call_function(coll_filter, ca, [Instr("LOAD_CONST", fb)], [Instr("LOAD_FAST", "_locals")])
 
         if _type == "getmember":
             v, attr = args
diff --git a/ddtrace/internal/logger.py b/ddtrace/internal/logger.py
@@ -1,180 +1,80 @@
 import collections
 import logging
 import os
-import typing
-from typing import Optional  # noqa:F401
-from typing import cast  # noqa:F401
+from typing import DefaultDict
+from typing import Tuple
 
 
-if typing.TYPE_CHECKING:
-    from typing import Any  # noqa:F401
-    from typing import DefaultDict  # noqa:F401
-    from typing import Tuple  # noqa:F401
-
-
-def get_logger(name):
-    # type: (str) -> DDLogger
+def get_logger(name: str) -> logging.Logger:
     """
-    Retrieve or create a ``DDLogger`` instance.
-
-    This function mirrors the behavior of `logging.getLogger`.
+    Retrieve or create a ``Logger`` instance with consistent behavior for internal use.
 
-    If no logger with the provided name has been fetched before then
-    a new one is created.
+    Configure all loggers with a rate limiter filter to prevent excessive logging.
 
-    If a previous logger has been created then it is returned.
-
-    DEV: We do not want to mess with `logging.setLoggerClass()`
-         That will totally mess with the user's loggers, we want
-         just our own, selective loggers to be DDLoggers
-
-    :param name: The name of the logger to fetch or create
-    :type name: str
-    :return: The logger instance
-    :rtype: ``DDLogger``
     """
-    # DEV: `logging.Logger.manager` refers to the single root `logging.Manager` instance
-    #   https://github.com/python/cpython/blob/48769a28ad6ef4183508951fa6a378531ace26a4/Lib/logging/__init__.py#L1824-L1826  # noqa:E501
-    manager = logging.Logger.manager
+    logger = logging.getLogger(name)
+    # addFilter will only add the filter if it is not already present
+    logger.addFilter(log_filter)
+    return logger
 
-    # If the logger does not exist yet, create it
-    # DEV: `Manager.loggerDict` is a dict mapping logger name to logger
-    # DEV: This is a simplified version of `logging.Manager.getLogger`
-    #   https://github.com/python/cpython/blob/48769a28ad6ef4183508951fa6a378531ace26a4/Lib/logging/__init__.py#L1221-L1253  # noqa:E501
-    # DEV: _fixupParents could be adding a placeholder, we want to replace it if that's the case
-    if name in manager.loggerDict:
-        logger = manager.loggerDict[name]
-        if isinstance(manager.loggerDict[name], logging.PlaceHolder):
-            placeholder = logger
-            logger = DDLogger(name=name)
-            manager.loggerDict[name] = logger
-            # DEV: `_fixupChildren` and `_fixupParents` have been around for awhile,
-            # DEV: but add the `hasattr` guard... just in case.
-            if hasattr(manager, "_fixupChildren"):
-                manager._fixupChildren(placeholder, logger)
-            if hasattr(manager, "_fixupParents"):
-                manager._fixupParents(logger)
-    else:
-        logger = DDLogger(name=name)
-        manager.loggerDict[name] = logger
-        if hasattr(manager, "_fixupParents"):
-            manager._fixupParents(logger)
 
-    # Return our logger
-    return cast(DDLogger, logger)
+# Named tuple used for keeping track of a log lines current time bucket and the number of log lines skipped
+LoggingBucket = collections.namedtuple("LoggingBucket", ("bucket", "skipped"))
+# Dict to keep track of the current time bucket per name/level/pathname/lineno
+_buckets: DefaultDict[Tuple[str, int, str, int], LoggingBucket] = collections.defaultdict(lambda: LoggingBucket(0, 0))
 
+# Allow 1 log record per name/level/pathname/lineno every 60 seconds by default
+# Allow configuring via `DD_TRACE_LOGGING_RATE`
+# DEV: `DD_TRACE_LOGGING_RATE=0` means to disable all rate limiting
+_rate_limit = int(os.getenv("DD_TRACE_LOGGING_RATE", default=60))
 
-def hasHandlers(self):
-    # type: (DDLogger) -> bool
-    """
-    See if this logger has any handlers configured.
-    Loop through all handlers for this logger and its parents in the
-    logger hierarchy. Return True if a handler was found, else False.
-    Stop searching up the hierarchy whenever a logger with the "propagate"
-    attribute set to zero is found - that will be the last logger which
-    is checked for the existence of handlers.
 
-    https://github.com/python/cpython/blob/8f192d12af82c4dc40730bf59814f6a68f68f950/Lib/logging/__init__.py#L1629
+def log_filter(record: logging.LogRecord) -> bool:
     """
-    c = self
-    rv = False
-    while c:
-        if c.handlers:
-            rv = True
-            break
-        if not c.propagate:
-            break
-        else:
-            c = c.parent  # type: ignore
-    return rv
+    Function used to determine if a log record should be outputted or not (True = output, False = skip).
 
-
-class DDLogger(logging.Logger):
-    """
-    Custom rate limited logger used by ``ddtrace``
-
-    This logger class is used to rate limit the output of
-    log messages from within the ``ddtrace`` package.
+    This function will:
+      - Log all records with a level of ERROR or higher with telemetry
+      - Rate limit log records based on the logger name, record level, filename, and line number
     """
+    if record.levelno >= logging.ERROR:
+        # avoid circular import
+        from ddtrace.internal import telemetry
 
-    # Named tuple used for keeping track of a log lines current time bucket and the number of log lines skipped
-    LoggingBucket = collections.namedtuple("LoggingBucket", ("bucket", "skipped"))
-
-    def __init__(self, *args, **kwargs):
-        # type: (*Any, **Any) -> None
-        """Constructor for ``DDLogger``"""
-        super(DDLogger, self).__init__(*args, **kwargs)
-
-        # Dict to keep track of the current time bucket per name/level/pathname/lineno
-        self.buckets = collections.defaultdict(
-            lambda: DDLogger.LoggingBucket(0, 0)
-        )  # type: DefaultDict[Tuple[str, int, str, int], DDLogger.LoggingBucket]
+        # currently we only have one error code
+        full_file_name = os.path.join(record.pathname, record.filename)
+        telemetry.telemetry_writer.add_error(1, record.msg % record.args, full_file_name, record.lineno)
 
-        # Allow 1 log record per name/level/pathname/lineno every 60 seconds by default
-        # Allow configuring via `DD_TRACE_LOGGING_RATE`
-        # DEV: `DD_TRACE_LOGGING_RATE=0` means to disable all rate limiting
-        rate_limit = os.getenv("DD_TRACE_LOGGING_RATE", default=None)
-
-        if rate_limit is not None:
-            self.rate_limit = int(rate_limit)
-        else:
-            self.rate_limit = 60
-
-    def handle(self, record):
-        # type: (logging.LogRecord) -> None
-        """
-        Function used to call the handlers for a log line.
-
-        This implementation will first determine if this log line should
-        be logged or rate limited, and then call the base ``logging.Logger.handle``
-        function if it should be logged
-
-        DEV: This method has all of it's code inlined to reduce on functions calls
-
-        :param record: The log record being logged
-        :type record: ``logging.LogRecord``
-        """
-        if record.levelno >= logging.ERROR:
-            # avoid circular import
-            from ddtrace.internal import telemetry
-
-            # currently we only have one error code
-            full_file_name = os.path.join(record.pathname, record.filename)
-            telemetry.telemetry_writer.add_error(1, record.msg % record.args, full_file_name, record.lineno)
-
-        # If rate limiting has been disabled (`DD_TRACE_LOGGING_RATE=0`) then apply no rate limit
-        # If the logging is in debug, then do not apply any limits to any log
-        if not self.rate_limit or self.getEffectiveLevel() == logging.DEBUG:
-            super(DDLogger, self).handle(record)
-            return
+    logger = logging.getLogger(record.name)
 
+    # If rate limiting has been disabled (`DD_TRACE_LOGGING_RATE=0`) then apply no rate limit
+    # If the logger is set to debug, then do not apply any limits to any log
+    if not _rate_limit or logger.getEffectiveLevel() == logging.DEBUG:
+        return True
         # Allow 1 log record by name/level/pathname/lineno every X seconds
-        # DEV: current unix time / rate (e.g. 300 seconds) = time bucket
-        #      int(1546615098.8404942 / 300) = 515538
-        # DEV: LogRecord `created` is a unix timestamp/float
-        # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10`
-        current_bucket = int(record.created / self.rate_limit)
-
-        # Limit based on logger name, record level, filename, and line number
-        #   ('ddtrace.writer', 'DEBUG', '../site-packages/ddtrace/writer.py', 137)
-        # This way each unique log message can get logged at least once per time period
-        # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10`
-        key = (record.name, record.levelno, record.pathname, record.lineno)
-
-        # Only log this message if the time bucket has changed from the previous time we ran
-        logging_bucket = self.buckets[key]
-        if logging_bucket.bucket != current_bucket:
-            # Append count of skipped messages if we have skipped some since our last logging
-            if logging_bucket.skipped:
-                record.msg = "{}, %s additional messages skipped".format(record.msg)
-                record.args = record.args + (logging_bucket.skipped,)  # type: ignore
-
+    # DEV: current unix time / rate (e.g. 300 seconds) = time bucket
+    #      int(1546615098.8404942 / 300) = 515538
+    # DEV: LogRecord `created` is a unix timestamp/float
+    # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10`
+    current_bucket = int(record.created / _rate_limit)
+    # Limit based on logger name, record level, filename, and line number
+    #   ('ddtrace.writer', 'DEBUG', '../site-packages/ddtrace/writer.py', 137)
+    # This way each unique log message can get logged at least once per time period
+    # DEV: LogRecord has `levelname` and `levelno`, we want `levelno` e.g. `logging.DEBUG = 10`
+    key = (record.name, record.levelno, record.pathname, record.lineno)
+    # Only log this message if the time bucket has changed from the previous time we ran
+    logging_bucket = _buckets[key]
+    if logging_bucket.bucket != current_bucket:
+        # Append count of skipped messages if we have skipped some since our last logging
+        if logging_bucket.skipped:
+            record.msg = "{}, %s additional messages skipped".format(record.msg)
+            record.args = record.args + (logging_bucket.skipped,)  # type: ignore
             # Reset our bucket
-            self.buckets[key] = DDLogger.LoggingBucket(current_bucket, 0)
-
-            # Call the base handle to actually log this record
-            super(DDLogger, self).handle(record)
-        else:
-            # Increment the count of records we have skipped
-            # DEV: `self.buckets[key]` is a tuple which is immutable so recreate instead
-            self.buckets[key] = DDLogger.LoggingBucket(logging_bucket.bucket, logging_bucket.skipped + 1)
+        _buckets[key] = LoggingBucket(current_bucket, 0)
+        # Actually log this record
+        return True
+    # Increment the count of records we have skipped
+    # DEV: `buckets[key]` is a tuple which is immutable so recreate instead
+    _buckets[key] = LoggingBucket(logging_bucket.bucket, logging_bucket.skipped + 1)
+    # Skip this log message
+    return False
diff --git a/ddtrace/settings/symbol_db.py b/ddtrace/settings/symbol_db.py
@@ -11,7 +11,7 @@ class SymbolDatabaseConfig(En):
     enabled = En.v(
         bool,
         "upload_enabled",
-        default=False,
+        default=True,
         help_type="Boolean",
         help="Whether to upload source code symbols to the Datadog backend",
     )
diff --git a/releasenotes/notes/feat-symdb-enabled-by-default-25eebb43fc8c5a0d.yaml b/releasenotes/notes/feat-symdb-enabled-by-default-25eebb43fc8c5a0d.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Symbol Database is now enabled by default.
diff --git a/releasenotes/notes/fix-internal-logging-init-6058c02b527cdf77.yaml b/releasenotes/notes/fix-internal-logging-init-6058c02b527cdf77.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    internal: Fix ``ddtrace`` internal logger initialization mutating an unlocked shared resource.
diff --git a/tests/debugging/test_expressions.py b/tests/debugging/test_expressions.py
@@ -74,6 +74,10 @@ def __getitem__(self, name):
         ({"eq": [{"ref": "hits"}, None]}, {"hits": None}, True),
         ({"substring": [{"ref": "payload"}, 4, 7]}, {"payload": "hello world"}, "hello world"[4:7]),
         ({"any": [{"ref": "collection"}, {"isEmpty": {"ref": "@it"}}]}, {"collection": ["foo", "bar", ""]}, True),
+        ({"any": [{"ref": "coll"}, {"isEmpty": {"ref": "@value"}}]}, {"coll": {0: "foo", 1: "bar", 2: ""}}, True),
+        ({"any": [{"ref": "coll"}, {"isEmpty": {"ref": "@value"}}]}, {"coll": {0: "foo", 1: "bar", 2: "baz"}}, False),
+        ({"any": [{"ref": "coll"}, {"isEmpty": {"ref": "@key"}}]}, {"coll": {"foo": 0, "bar": 1, "": 2}}, True),
+        ({"any": [{"ref": "coll"}, {"isEmpty": {"ref": "@key"}}]}, {"coll": {"foo": 0, "bar": 1, "baz": 2}}, False),
         ({"startsWith": [{"ref": "local_string"}, "hello"]}, {"local_string": "hello world!"}, True),
         ({"startsWith": [{"ref": "local_string"}, "world"]}, {"local_string": "hello world!"}, False),
         (
@@ -91,6 +95,16 @@ def __getitem__(self, name):
             {"collection": {"foo", "bar", ""}},
             {"foo", "bar"},
         ),
+        (
+            {"filter": [{"ref": "collection"}, {"not": {"isEmpty": {"ref": "@value"}}}]},
+            {"collection": {1: "foo", 2: "bar", 3: ""}},
+            {1: "foo", 2: "bar"},
+        ),
+        (
+            {"filter": [{"ref": "collection"}, {"not": {"isEmpty": {"ref": "@key"}}}]},
+            {"collection": {"foo": 1, "bar": 2, "": 3}},
+            {"foo": 1, "bar": 2},
+        ),
         ({"contains": [{"ref": "payload"}, "hello"]}, {"payload": CustomObject("contains")}, SideEffect),
         (
             {"contains": [{"ref": "payload"}, "hello"]},
diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py
@@ -420,7 +420,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
             "value": str(file),
         },
         {"name": "DD_SYMBOL_DATABASE_INCLUDES", "origin": "default", "value": "set()"},
-        {"name": "DD_SYMBOL_DATABASE_UPLOAD_ENABLED", "origin": "default", "value": False},
+        {"name": "DD_SYMBOL_DATABASE_UPLOAD_ENABLED", "origin": "default", "value": True},
         {"name": "DD_TAGS", "origin": "env_var", "value": "team:apm,component:web"},
         {"name": "DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", "origin": "default", "value": True},
         {"name": "DD_TELEMETRY_HEARTBEAT_INTERVAL", "origin": "default", "value": 60},
diff --git a/tests/tracer/test_logger.py b/tests/tracer/test_logger.py

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ class SymbolDatabaseConfig(En):`
`11`	`11`	`enabled = En.v(`
`12`	`12`	`bool,`
`13`	`13`	`"upload_enabled",`
`14`		`- default=False,`
	`14`	`+ default=True,`
`15`	`15`	`help_type="Boolean",`
`16`	`16`	`help="Whether to upload source code symbols to the Datadog backend",`
`17`	`17`	`)`
-Original file line number
+Diff line change
 +---
 +fixes:
 +  - |
 +    internal: Fix ``ddtrace`` internal logger initialization mutating an unlocked shared resource.