diff --git a/ddtrace/__init__.py b/ddtrace/__init__.py index 008e931a482..17d41a18fa6 100644 --- a/ddtrace/__init__.py +++ b/ddtrace/__init__.py @@ -5,6 +5,11 @@ LOADED_MODULES = frozenset(sys.modules.keys()) +# Configuration for the whole tracer from file. Do it before anything else happens. +from ddtrace.internal.native import _apply_configuration_from_disk + +_apply_configuration_from_disk() + from ddtrace.internal.module import ModuleWatchdog diff --git a/ddtrace/internal/native/__init__.py b/ddtrace/internal/native/__init__.py index 0c85a824b44..feffef6e18f 100644 --- a/ddtrace/internal/native/__init__.py +++ b/ddtrace/internal/native/__init__.py @@ -1 +1,38 @@ +import os +from typing import Dict + from ._native import DDSketch # noqa: F401 +from ._native import PyConfigurator + + +def get_configuration_from_disk(debug_logs: bool = False, file_override="") -> Dict[str, str]: + """ + Retrieves the tracer configuration from disk. Calls the PyConfigurator object + to read the configuration from the disk using the libdatadog shared library + and returns the corresponding configuration + See https://github.com/DataDog/libdatadog/blob/06d2b6a19d7ec9f41b3bfd4ddf521585c55298f6/library-config/src/lib.rs + for more information on how the configuration is read from disk + """ + configurator = PyConfigurator(debug_logs) + + # Set the file override if provided. Only used for testing purposes. + if file_override: + configurator.set_local_file_override(file_override) + + return configurator.get_configuration() + + +def _apply_configuration_from_disk(): + """ + Sets the configuration from disk as environment variables. + This is not ideal and we should consider a better mechanism to + apply this configuration to the tracer. + Currently here is the order of precedence (higher takes precedence): + 1. Dynamic remote configuration + 2. Runtime configuration (ie fields set manually by customers / from the ddtrace code) + 3. Configuration from disk + 4. Environment variables + 5. Default values + """ + for key, value in get_configuration_from_disk().items(): + os.environ[key] = str(value).lower() diff --git a/ddtrace/internal/native/_native.pyi b/ddtrace/internal/native/_native.pyi index 96f54180c42..82eed529dc8 100644 --- a/ddtrace/internal/native/_native.pyi +++ b/ddtrace/internal/native/_native.pyi @@ -1,6 +1,34 @@ +from typing import Dict + class DDSketch: def __init__(self): ... def add(self, value: float) -> None: ... def to_proto(self) -> bytes: ... @property def count(self) -> float: ... + +class PyConfigurator: + """ + PyConfigurator is a class responsible for configuring the Python environment + for the application. It allows setting environment variables, command-line + arguments, and file overrides, and retrieving the current configuration. + """ + + def __init__(self, debug_logs: bool): + """ + Initialize the PyConfigurator. + :param debug_logs: A boolean indicating whether debug logs should be enabled. + """ + ... + def set_local_file_override(self, file: str) -> None: + """ + Overrides the file path for the configuration. Should not be used outside of tests. + :param file: The path to the file to override. + """ + ... + def get_configuration(self) -> Dict[str, str]: + """ + Retrieve the on-disk configuration. + :return: A dictionary containing the current configuration of the form {key: value}. + """ + ... diff --git a/src/native/Cargo.lock b/src/native/Cargo.lock index bbf189de0c2..8420ae5e7ea 100644 --- a/src/native/Cargo.lock +++ b/src/native/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "anyhow" @@ -28,17 +28,28 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "datadog-ddsketch" -version = "15.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=v15.0.0#0ef49864317b0728648b2b7f26fe2f1deeeeebc4" +version = "16.0.2" +source = "git+https://github.com/DataDog/libdatadog?rev=v16.0.2#52bd068269da43fb216e278a8576184bf6307177" dependencies = [ "prost", ] [[package]] -name = "ddtrace-core" +name = "datadog-library-config" +version = "0.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=v16.0.2#52bd068269da43fb216e278a8576184bf6307177" +dependencies = [ + "anyhow", + "serde", + "serde_yaml", +] + +[[package]] +name = "ddtrace-native" version = "0.1.0" dependencies = [ "datadog-ddsketch", + "datadog-library-config", "pyo3", "pyo3-build-config 0.21.2", ] @@ -49,12 +60,34 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.5" @@ -70,6 +103,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + [[package]] name = "libc" version = "0.2.154" @@ -211,6 +250,45 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ryu" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.61", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "syn" version = "1.0.109" @@ -250,3 +328,9 @@ name = "unindent" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" diff --git a/src/native/Cargo.toml b/src/native/Cargo.toml index a73bab58a05..d09eae4f809 100644 --- a/src/native/Cargo.toml +++ b/src/native/Cargo.toml @@ -10,7 +10,8 @@ opt-level = 3 [dependencies] pyo3 = { version = "0.22.3", features = ["extension-module"] } -datadog-ddsketch = { git = "https://github.com/DataDog/libdatadog", rev = "v15.0.0" } +datadog-ddsketch = { git = "https://github.com/DataDog/libdatadog", rev = "v16.0.2" } +datadog-library-config = { git = "https://github.com/DataDog/libdatadog", rev = "v16.0.2" } [build-dependencies] pyo3-build-config = "0.21.2" diff --git a/src/native/lib.rs b/src/native/lib.rs index a3e2825e21a..d026c7a7997 100644 --- a/src/native/lib.rs +++ b/src/native/lib.rs @@ -1,9 +1,11 @@ mod ddsketch; +mod library_config; use pyo3::prelude::*; #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/native/library_config.rs b/src/native/library_config.rs new file mode 100644 index 00000000000..b68a417dabd --- /dev/null +++ b/src/native/library_config.rs @@ -0,0 +1,50 @@ +use datadog_library_config::{Configurator, ProcessInfo}; +use pyo3::exceptions::PyException; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyclass(name = "PyConfigurator", module = "ddtrace.internal._native")] +pub struct PyConfigurator { + configurator: Box, + local_file: String, + fleet_file: String, +} + +#[pymethods] +impl PyConfigurator { + #[new] + pub fn new(debug_logs: bool) -> Self { + PyConfigurator { + configurator: Box::new(Configurator::new(debug_logs)), + fleet_file: Configurator::FLEET_STABLE_CONFIGURATION_PATH.to_string(), + local_file: Configurator::LOCAL_STABLE_CONFIGURATION_PATH.to_string(), + } + } + + pub fn set_local_file_override(&mut self, file: String) -> PyResult<()> { + self.local_file = file; + Ok(()) + } + + pub fn get_configuration(&self, py: Python<'_>) -> PyResult { + let res_config = self.configurator.get_config_from_file( + self.local_file.as_ref(), + self.fleet_file.as_ref(), + ProcessInfo::detect_global("python".to_string()), + ); + match res_config { + Ok(config) => { + let dict = PyDict::new_bound(py); + for c in config.iter() { + let key = c.name.to_str().to_owned(); + let _ = dict.set_item(key, c.value.clone()); + } + Ok(dict.into()) + } + Err(e) => { + let err_msg = format!("Failed to get configuration: {:?}", e); + Err(PyException::new_err(err_msg)) + } + } + } +} diff --git a/tests/internal/test_native.py b/tests/internal/test_native.py new file mode 100644 index 00000000000..bdd31d3b3dc --- /dev/null +++ b/tests/internal/test_native.py @@ -0,0 +1,59 @@ +from ddtrace.internal.native import get_configuration_from_disk + + +def test_get_configuration_from_disk(tmp_path): + # First test -- config matches & should be returned + config_1 = tmp_path / "config_1.yaml" + config_1.write_text( + """ +rules: + - selectors: + - origin: language + matches: + - python + operator: equals + configuration: + DD_SERVICE: my-service +""", + encoding="utf-8", + ) + + config = get_configuration_from_disk(file_override=str(config_1)) + assert config == {"DD_SERVICE": "my-service"} + + # Second test -- config does not match & should not be returned + config_2 = tmp_path / "config_2.yaml" + config_2.write_text( + """ +rules: + - selectors: + - origin: language + matches: + - nodejs + operator: equals + configuration: + DD_SERVICE: my-service +""", + encoding="utf-8", + ) + + config = get_configuration_from_disk(file_override=str(config_2)) + assert config == {} + + +def is_imported_before(module_a, module_b): + import sys + + # Check if both modules are in sys.modules + if module_a in sys.modules and module_b in sys.modules: + # Get the position of the modules in sys.modules (which is an OrderedDict in Python 3.7+) + modules = list(sys.modules.keys()) + return modules.index(module_a) < modules.index(module_b) + return False # If one or both modules are not imported, return False + + +def test_native_before_settings(): + # Ensure that the native module is imported before the settings module + import ddtrace # noqa: F401 + + assert is_imported_before("ddtrace.internal.native", "ddtrace.settings")