Merge remote-tracking branch 'origin/3.x-staging' into christophe-pap…

…azian/remove_appsec_dependencies_if_serverless

ddtrace/appsec/_iast/_evidence_redaction/_sensitive_handler.py

@@ -6,10 +6,12 @@
 
 from .._utils import _get_source_index
 from ..constants import VULN_CMDI
+from ..constants import VULN_CODE_INJECTION
 from ..constants import VULN_HEADER_INJECTION
 from ..constants import VULN_SQL_INJECTION
 from ..constants import VULN_SSRF
 from .command_injection_sensitive_analyzer import command_injection_sensitive_analyzer
+from .default_sensitive_analyzer import default_sensitive_analyzer
 from .header_injection_sensitive_analyzer import header_injection_sensitive_analyzer
 from .sql_sensitive_analyzer import sql_sensitive_analyzer
 from .url_sensitive_analyzer import url_sensitive_analyzer
@@ -19,6 +21,7 @@
 
 REDACTED_SOURCE_BUFFER = string.ascii_letters + string.digits
 LEN_SOURCE_BUFFER = len(REDACTED_SOURCE_BUFFER)
+VALUE_MAX_LENGHT = 45
 
 
 def get_redacted_source(length):
@@ -42,6 +45,7 @@ def __init__(self):
             VULN_SQL_INJECTION: sql_sensitive_analyzer,
             VULN_SSRF: url_sensitive_analyzer,
             VULN_HEADER_INJECTION: header_injection_sensitive_analyzer,
+            VULN_CODE_INJECTION: default_sensitive_analyzer,
         }
 
     @staticmethod
@@ -288,7 +292,7 @@ def to_redacted_json(self, evidence_value, sensitive, tainted_ranges, sources):
         return {"redacted_value_parts": value_parts, "redacted_sources": redacted_sources}
 
     def redact_source(self, sources, redacted_sources, redacted_sources_context, source_index, start, end):
-        if source_index is not None:
+        if source_index is not None and source_index < len(sources):
             if not sources[source_index].redacted:
                 redacted_sources.append(source_index)
                 sources[source_index].pattern = get_redacted_source(len(sources[source_index].value))
@@ -303,8 +307,10 @@ def write_value_part(self, value_parts, value, source_index=None):
         if value:
             if source_index is not None:
                 value_parts.append({"value": value, "source": source_index})
-            else:
+            elif len(value) < VALUE_MAX_LENGHT:
                 value_parts.append({"value": value})
+            else:
+                value_parts.append({"redacted": True})
 
     def write_redacted_value_part(
         self,

ddtrace/appsec/_iast/_evidence_redaction/default_sensitive_analyzer.py

@@ -0,0 +1,11 @@
+from ddtrace.internal.logger import get_logger
+
+
+log = get_logger(__name__)
+
+
+def default_sensitive_analyzer(evidence, name_pattern, value_pattern):
+    if name_pattern.search(evidence.value) or value_pattern.search(evidence.value):
+        return [{"start": 0, "end": len(evidence.value)}]
+
+    return []

ddtrace/appsec/_iast/_handlers.py

@@ -153,6 +153,7 @@ def _on_django_patch():
                     functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
                 )
             )
+
             # we instrument those sources on _on_django_func_wrapped
             _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
             _set_metric_iast_instrumented_source(OriginType.HEADER)

ddtrace/contrib/internal/langchain/patch.py

@@ -643,114 +643,6 @@ def traced_embedding(langchain, pin, func, instance, args, kwargs):
     return embeddings
 
 
-@with_traced_module
-def traced_chain_call(langchain, pin, func, instance, args, kwargs):
-    integration = langchain._datadog_integration
-    span = integration.trace(
-        pin,
-        "{}.{}".format(instance.__module__, instance.__class__.__name__),
-        submit_to_llmobs=True,
-        interface_type="chain",
-    )
-    inputs = None
-    final_outputs = {}
-    try:
-        inputs = get_argument_value(args, kwargs, 0, "input")
-        if not isinstance(inputs, dict):
-            inputs = {instance.input_keys[0]: inputs}
-        if integration.is_pc_sampled_span(span):
-            for k, v in inputs.items():
-                span.set_tag_str("langchain.request.inputs.%s" % k, integration.trunc(str(v)))
-            template = deep_getattr(instance, "prompt.template", default="")
-            if template:
-                span.set_tag_str("langchain.request.prompt", integration.trunc(str(template)))
-        final_outputs = func(*args, **kwargs)
-        if integration.is_pc_sampled_span(span):
-            for k, v in final_outputs.items():
-                span.set_tag_str("langchain.response.outputs.%s" % k, integration.trunc(str(v)))
-        if asm_config._iast_enabled:
-            taint_outputs(instance, inputs, final_outputs)
-    except Exception:
-        span.set_exc_info(*sys.exc_info())
-        integration.metric(span, "incr", "request.error", 1)
-        raise
-    finally:
-        integration.llmobs_set_tags(span, args=[], kwargs=inputs, response=final_outputs, operation="chain")
-        span.finish()
-        integration.metric(span, "dist", "request.duration", span.duration_ns)
-        if integration.is_pc_sampled_log(span):
-            log_inputs = {}
-            log_outputs = {}
-            for k, v in inputs.items():
-                log_inputs[k] = str(v)
-            for k, v in final_outputs.items():
-                log_outputs[k] = str(v)
-            integration.log(
-                span,
-                "info" if span.error == 0 else "error",
-                "sampled %s.%s" % (instance.__module__, instance.__class__.__name__),
-                attrs={
-                    "inputs": log_inputs,
-                    "prompt": str(deep_getattr(instance, "prompt.template", default="")),
-                    "outputs": log_outputs,
-                },
-            )
-    return final_outputs
-
-
-@with_traced_module
-async def traced_chain_acall(langchain, pin, func, instance, args, kwargs):
-    integration = langchain._datadog_integration
-    span = integration.trace(
-        pin,
-        "{}.{}".format(instance.__module__, instance.__class__.__name__),
-        submit_to_llmobs=True,
-        interface_type="chain",
-    )
-    inputs = None
-    final_outputs = {}
-    try:
-        inputs = get_argument_value(args, kwargs, 0, "input")
-        if not isinstance(inputs, dict):
-            inputs = {instance.input_keys[0]: inputs}
-        if integration.is_pc_sampled_span(span):
-            for k, v in inputs.items():
-                span.set_tag_str("langchain.request.inputs.%s" % k, integration.trunc(str(v)))
-            template = deep_getattr(instance, "prompt.template", default="")
-            if template:
-                span.set_tag_str("langchain.request.prompt", integration.trunc(str(template)))
-        final_outputs = await func(*args, **kwargs)
-        if integration.is_pc_sampled_span(span):
-            for k, v in final_outputs.items():
-                span.set_tag_str("langchain.response.outputs.%s" % k, integration.trunc(str(v)))
-    except Exception:
-        span.set_exc_info(*sys.exc_info())
-        integration.metric(span, "incr", "request.error", 1)
-        raise
-    finally:
-        integration.llmobs_set_tags(span, args=[], kwargs=inputs, response=final_outputs, operation="chain")
-        span.finish()
-        integration.metric(span, "dist", "request.duration", span.duration_ns)
-        if integration.is_pc_sampled_log(span):
-            log_inputs = {}
-            log_outputs = {}
-            for k, v in inputs.items():
-                log_inputs[k] = str(v)
-            for k, v in final_outputs.items():
-                log_outputs[k] = str(v)
-            integration.log(
-                span,
-                "info" if span.error == 0 else "error",
-                "sampled %s.%s" % (instance.__module__, instance.__class__.__name__),
-                attrs={
-                    "inputs": log_inputs,
-                    "prompt": str(deep_getattr(instance, "prompt.template", default="")),
-                    "outputs": log_outputs,
-                },
-            )
-    return final_outputs
-
-
 @with_traced_module
 def traced_lcel_runnable_sequence(langchain, pin, func, instance, args, kwargs):
     """
@@ -1287,8 +1179,6 @@ def patch():
         "language_models.chat_models.BaseChatModel.agenerate",
         traced_chat_model_agenerate(langchain),
     )
-    wrap("langchain", "chains.base.Chain.invoke", traced_chain_call(langchain))
-    wrap("langchain", "chains.base.Chain.ainvoke", traced_chain_acall(langchain))
     wrap("langchain_core", "runnables.base.RunnableSequence.invoke", traced_lcel_runnable_sequence(langchain))
     wrap("langchain_core", "runnables.base.RunnableSequence.ainvoke", traced_lcel_runnable_sequence_async(langchain))
     wrap("langchain_core", "runnables.base.RunnableSequence.batch", traced_lcel_runnable_sequence(langchain))
@@ -1342,8 +1232,6 @@ def unpatch():
     unwrap(langchain_core.language_models.llms.BaseLLM, "agenerate")
     unwrap(langchain_core.language_models.chat_models.BaseChatModel, "generate")
     unwrap(langchain_core.language_models.chat_models.BaseChatModel, "agenerate")
-    unwrap(langchain.chains.base.Chain, "invoke")
-    unwrap(langchain.chains.base.Chain, "ainvoke")
     unwrap(langchain_core.runnables.base.RunnableSequence, "invoke")
     unwrap(langchain_core.runnables.base.RunnableSequence, "ainvoke")
     unwrap(langchain_core.runnables.base.RunnableSequence, "batch")

ddtrace/internal/remoteconfig/product.py

@@ -11,7 +11,7 @@ def start():
         from ddtrace.internal.remoteconfig.worker import remoteconfig_poller
 
         remoteconfig_poller.enable()
-        config.enable_remote_configuration()
+        config._enable_remote_configuration()
 
 
 def restart(join=False):

ddtrace/settings/_config.py

@@ -324,32 +324,6 @@ class Config(object):
     available and can be updated by users.
     """
 
-    # Maps deprecated configuration attributes to their corresponding environment variable and
-    # internalized attribute name
-    _DEPRECATED_ATTRS = {
-        "http_tag_query_string": ("_http_tag_query_string", "DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING"),
-        "trace_http_header_tags": ("_trace_http_header_tags", "DD_TRACE_HEADER_TAGS"),
-        "report_hostname": ("_report_hostname", "DD_TRACE_REPORT_HOSTNAME"),
-        "health_metrics_enabled": ("_health_metrics_enabled", "DD_TRACE_HEALTH_METRICS_ENABLED"),
-        "analytics_enabled": ("_analytics_enabled", "DD_TRACE_ANALYTICS_ENABLED"),
-        "client_ip_header": ("_client_ip_header", "DD_TRACE_CLIENT_IP_HEADER"),
-        "retrieve_client_ip": ("_retrieve_client_ip", "DD_TRACE_CLIENT_IP_ENABLED"),
-        "propagation_http_baggage_enabled": (
-            "_propagation_http_baggage_enabled",
-            "DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED",
-        ),
-        "global_query_string_obfuscation_disabled": (
-            "_global_query_string_obfuscation_disabled",
-            'DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP=""',
-        ),
-        "trace_methods": ("_trace_methods", "DD_TRACE_METHODS"),
-        "ci_visibility_log_level": ("_ci_visibility_log_level", "DD_CIVISIBILITY_LOG_LEVEL"),
-        "test_session_name": ("_test_session_name", "DD_TEST_SESSION_NAME"),
-        "logs_injection": ("_logs_injection", "DD_LOGS_INJECTION"),
-        "http_server": ("_http_server", "DD_TRACE_HTTP_SERVER_ERROR_STATUSES"),
-        "http": ("_http", "DD_TRACE_HEADER_TAGS"),
-    }
-
     class _HTTPServerConfig(object):
         _error_statuses = _get_config("DD_TRACE_HTTP_SERVER_ERROR_STATUSES", "500-599")  # type: str
         _error_ranges = get_error_ranges(_error_statuses)  # type: List[Tuple[int, int]]
@@ -642,22 +616,6 @@ def __init__(self):
         self._inject_was_attempted = _get_config("_DD_INJECT_WAS_ATTEMPTED", False, asbool)
 
     def __getattr__(self, name) -> Any:
-        if name in self._DEPRECATED_ATTRS:
-            new_name, env_var = self._DEPRECATED_ATTRS[name]
-            deprecate(
-                f"ddtrace.config.{name} is deprecated",
-                message=f"Use the environment variable {env_var} instead. "
-                "This variable must be set before importing ddtrace.",
-                removal_version="3.0.0",
-                category=DDTraceDeprecationWarning,
-            )
-            if name == new_name:
-                raise RuntimeError(
-                    f"Circular mapping detected: deprecated attribute {name} "
-                    f"in {self._DEPRECATED_ATTRS} maps to a deprecated attribute {new_name}"
-                )
-            return getattr(self, new_name)
-
         if name in self._config:
             return self._config[name].value()
 
@@ -681,13 +639,6 @@ def _get_extra_services(self):
             self._extra_services.pop()
         return self._extra_services
 
-    def get_from(self, obj):
-        deprecate(
-            "ddtrace.config.get_from is deprecated",
-            message="Use the `config` attribute directly.",
-        )
-        self._get_from(obj)
-
     def _get_from(self, obj):
         """Retrieves the configuration for the given object.
         Any object that has an attached `Pin` must have a configuration
@@ -732,36 +683,6 @@ def _add(self, integration, settings, merge=True):
         else:
             self._integration_configs[integration] = IntegrationConfig(self, integration, settings)
 
-    def trace_headers(self, whitelist):
-        """
-        Registers a set of headers to be traced at global level or integration level.
-        :param whitelist: the case-insensitive list of traced headers
-        :type whitelist: list of str or str
-        :return: self
-        :rtype: HttpConfig
-        """
-        deprecate(
-            "ddtrace.config.trace_headers is deprecated",
-            message="Use the environment variable DD_TRACE_HEADER_TAGS instead.",
-            removal_version="3.0.0",
-        )
-        self._http.trace_headers(whitelist)
-        return self
-
-    def header_is_traced(self, header_name):
-        # type: (str) -> bool
-        """
-        Returns whether or not the current header should be traced.
-        :param header_name: the header name
-        :type header_name: str
-        :rtype: bool
-        """
-        deprecate(
-            "ddtrace.config.header_is_traced is deprecated",
-            removal_version="3.0.0",
-        )
-        return self._http.header_is_traced(header_name)
-
     @cachedmethod()
     def _header_tag_name(self, header_name):
         # type: (str) -> Optional[str]
@@ -812,17 +733,6 @@ def __setattr__(self, key, value):
             self._set_config_items([(key, value, "code")])
             return None
         else:
-            if key in self._DEPRECATED_ATTRS:
-                # replace deprecated attribute name with the new name
-                new_key, env_var = self._DEPRECATED_ATTRS[key]
-                deprecate(
-                    f"ddtrace.config.{key} is deprecated",
-                    message=f"Use the environment variable {env_var} instead. "
-                    "This variable must be set before importing ddtrace.",
-                    removal_version="3.0.0",
-                    category=DDTraceDeprecationWarning,
-                )
-                key = new_key
             return super(self.__class__, self).__setattr__(key, value)
 
     def _set_config_items(self, items):
@@ -933,14 +843,6 @@ def _format_tags(self, tags: List[Union[str, Dict]]) -> Dict[str, str]:
             pairs = [t.split(":") for t in tags]  # type: ignore[union-attr,misc]
         return {k: v for k, v in pairs}
 
-    def enable_remote_configuration(self):
-        deprecate(
-            "ddtrace.config.enable_remote_configuration(...) is deprecated",
-            message="Use DD_TRACE_REMOTE_CONFIG_ENABLED instead.",
-            version="3.0.0",
-        )
-        self._enable_remote_configuration()
-
     def _enable_remote_configuration(self):
         # type: () -> None
         """Enable fetching configuration from Datadog."""
@@ -978,14 +880,6 @@ def _tags_to_dict(self, tags: List[Dict]):
             return {tag["key"]: tag["value_glob"] for tag in tags}
         return tags
 
-    def convert_rc_trace_sampling_rules(self, rc_rules: List[Dict[str, Any]]) -> Optional[str]:
-        deprecate(
-            "ddtrace.config.convert_rc_trace_sampling_rules(...) is deprecated",
-            message="Use DD_REMOTE_CONFIGURATION_ENABLED instead.",
-            version="3.0.0",
-        )
-        return self._convert_rc_trace_sampling_rules(rc_rules)
-
     def _convert_rc_trace_sampling_rules(self, rc_rules: List[Dict[str, Any]]) -> Optional[str]:
         """Example of an incoming rule:
         [

releasenotes/notes/langchain-drop-langchain-base-chains-support-48d8290af6eb86e3.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    langchain: removed tracing support for ``langchain.chains.base.Chains``. To maintain support, migrate to using Langchain's LCEL instead.

releasenotes/notes/remove-core-settings-c2c7518876a1ca20.yaml

@@ -0,0 +1,4 @@
+---
+deprecations:
+  - |
+    configurations: Removes deprecated attributes from ``ddtrace.config``.

tests/appsec/appsec/test_telemetry.py

@@ -1,7 +1,7 @@
 import os
 from time import sleep
+from unittest import mock
 
-import mock
 import pytest
 
 import ddtrace.appsec._asm_request_context as asm_request_context