feat(config): support libdatadog's library_config for config through file

Author: BaptisteFoy

ddtrace/__init__.py

@@ -5,6 +5,11 @@
 
 LOADED_MODULES = frozenset(sys.modules.keys())
 
+# Configuration for the whole tracer from file. Do it before anything else happens.
+from ddtrace.internal.native import _apply_configuration_from_disk
+
+_apply_configuration_from_disk()
+
 from ddtrace.internal.module import ModuleWatchdog
 
 

ddtrace/internal/native/__init__.py

@@ -1 +1,38 @@
+import os
+from typing import Dict
+
 from ._native import DDSketch  # noqa: F401
+from ._native import PyConfigurator
+
+
+def get_configuration_from_disk(debug_logs: bool = False, file_override="") -> Dict[str, str]:
+    """
+    Retrieves the tracer configuration from disk. Calls the PyConfigurator object
+    to read the configuration from the disk using the libdatadog shared library
+    and returns the corresponding configuration
+    See https://github.com/DataDog/libdatadog/blob/06d2b6a19d7ec9f41b3bfd4ddf521585c55298f6/library-config/src/lib.rs
+    for more information on how the configuration is read from disk
+    """
+    configurator = PyConfigurator(debug_logs)
+
+    # Set the file override if provided. Only used for testing purposes.
+    if file_override:
+        configurator.set_local_file_override(file_override)
+
+    return configurator.get_configuration()
+
+
+def _apply_configuration_from_disk():
+    """
+    Sets the configuration from disk as environment variables.
+    This is not ideal and we should consider a better mechanism to
+    apply this configuration to the tracer.
+    Currently here is the order of precedence (higher takes precedence):
+    1. Dynamic remote configuration
+    2. Runtime configuration (ie fields set manually by customers / from the ddtrace code)
+    3. Configuration from disk
+    4. Environment variables
+    5. Default values
+    """
+    for key, value in get_configuration_from_disk().items():
+        os.environ[key] = str(value).lower()

ddtrace/internal/native/_native.pyi

@@ -1,6 +1,34 @@
+from typing import Dict
+
 class DDSketch:
     def __init__(self): ...
     def add(self, value: float) -> None: ...
     def to_proto(self) -> bytes: ...
     @property
     def count(self) -> float: ...
+
+class PyConfigurator:
+    """
+    PyConfigurator is a class responsible for configuring the Python environment
+    for the application. It allows setting environment variables, command-line
+    arguments, and file overrides, and retrieving the current configuration.
+    """
+
+    def __init__(self, debug_logs: bool):
+        """
+        Initialize the PyConfigurator.
+        :param debug_logs: A boolean indicating whether debug logs should be enabled.
+        """
+        ...
+    def set_local_file_override(self, file: str) -> None:
+        """
+        Overrides the file path for the configuration. Should not be used outside of tests.
+        :param file: The path to the file to override.
+        """
+        ...
+    def get_configuration(self) -> Dict[str, str]:
+        """
+        Retrieve the on-disk configuration.
+        :return: A dictionary containing the current configuration of the form {key: value}.
+        """
+        ...

src/native/Cargo.lock

@@ -10,7 +10,8 @@ opt-level = 3
 
 [dependencies]
 pyo3 = { version = "0.22.3", features = ["extension-module"] }
-datadog-ddsketch = { git = "https://github.com/DataDog/libdatadog", rev = "v15.0.0" }
+datadog-ddsketch = { git = "https://github.com/DataDog/libdatadog", rev = "v16.0.1" }
+datadog-library-config = { git = "https://github.com/DataDog/libdatadog", rev = "v16.0.1" }
 
 [build-dependencies]
 pyo3-build-config = "0.21.2"

src/native/Cargo.toml

@@ -1,9 +1,11 @@
 mod ddsketch;
+mod library_config;
 
 use pyo3::prelude::*;
 
 #[pymodule]
 fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<ddsketch::DDSketchPy>()?;
+    m.add_class::<library_config::PyConfigurator>()?;
     Ok(())
 }

src/native/lib.rs

@@ -0,0 +1,52 @@
+use datadog_library_config::{Configurator, ProcessInfo};
+use pyo3::exceptions::PyException;
+use pyo3::prelude::*;
+use pyo3::types::PyDict;
+
+#[pyclass(name = "PyConfigurator", module = "ddtrace.internal._native")]
+pub struct PyConfigurator {
+    configurator: Box<Configurator>,
+    local_file: String,
+    fleet_file: String,
+}
+
+#[pymethods]
+impl PyConfigurator {
+    #[new]
+    pub fn new(debug_logs: bool) -> Self {
+        PyConfigurator {
+            configurator: Box::new(Configurator::new(debug_logs)),
+            fleet_file: Configurator::FLEET_STABLE_CONFIGURATION_PATH.to_string(),
+            local_file: Configurator::LOCAL_STABLE_CONFIGURATION_PATH.to_string(),
+        }
+    }
+
+    pub fn set_local_file_override(&mut self, file: String) -> PyResult<()> {
+        self.local_file = file;
+        Ok(())
+    }
+
+    pub fn get_configuration(&self, py: Python<'_>) -> PyResult<PyObject> {
+        let res_config = self
+            .configurator
+            .get_config_from_file(
+                self.local_file.as_ref(), 
+                self.fleet_file.as_ref(), 
+                ProcessInfo::detect_global("python".to_string()),
+            );
+        match res_config {
+            Ok(config) => {
+                let dict = PyDict::new_bound(py);
+                for c in config.iter() {
+                    let key = c.name.to_str().to_owned();
+                    let _ = dict.set_item(key, c.value.clone());
+                }
+                Ok(dict.into())
+            }
+            Err(e) => {
+                let err_msg = format!("Failed to get configuration: {:?}", e);
+                Err(PyException::new_err(err_msg))
+            }
+        }
+    }
+}

src/native/library_config.rs

@@ -0,0 +1,59 @@
+from ddtrace.internal.native import get_configuration_from_disk
+
+
+def test_get_configuration_from_disk(tmp_path):
+    # First test -- config matches & should be returned
+    config_1 = tmp_path / "config_1.yaml"
+    config_1.write_text(
+        """
+rules:
+  - selectors:
+    - origin: language
+      matches:
+        - python
+      operator: equals
+    configuration:
+      DD_SERVICE: my-service
+""",
+        encoding="utf-8",
+    )
+
+    config = get_configuration_from_disk(file_override=str(config_1))
+    assert config == {"DD_SERVICE": "my-service"}
+
+    # Second test -- config does not match & should not be returned
+    config_2 = tmp_path / "config_2.yaml"
+    config_2.write_text(
+        """
+rules:
+  - selectors:
+    - origin: language
+      matches:
+        - nodejs
+      operator: equals
+    configuration:
+      DD_SERVICE: my-service
+""",
+        encoding="utf-8",
+    )
+
+    config = get_configuration_from_disk(file_override=str(config_2))
+    assert config == {}
+
+
+def is_imported_before(module_a, module_b):
+    import sys
+
+    # Check if both modules are in sys.modules
+    if module_a in sys.modules and module_b in sys.modules:
+        # Get the position of the modules in sys.modules (which is an OrderedDict in Python 3.7+)
+        modules = list(sys.modules.keys())
+        return modules.index(module_a) < modules.index(module_b)
+    return False  # If one or both modules are not imported, return False
+
+
+def test_native_before_settings():
+    # Ensure that the native module is imported before the settings module
+    import ddtrace  # noqa: F401
+
+    assert is_imported_before("ddtrace.internal.native", "ddtrace.settings")