[ddtelemetry] Support metric distributions with ddsketch (#268)

* Add a minimal ddsketch implementation with PB serialization

* Add a distribution telemetry implementation

* Adjust reaktive accuracy to match datadog's backend

* fmt

* Add docs and rename bucket to bin

Rename bucket to bin in store function to be consistent with terminology
in the protobuf definition. The name store_index is introduced to
distinguish bins index from the index used in the store which have a
different offset.

* Reserve only required amount

The reserve function takes as a parameter the additional space to
reserve not the total size of the queue.

* Format protobuf generated module

* Deprecate telemetry distribution

* Remove telemetry distributions

Distributions should be replaced by sketches.

* Update copyright header

* Add owners for ddsketch

* Apply suggestions

* Rename sketches to distribution in public interfaces

* Upgrade base64

* Disable rustfmt on automatically generated files

Files with `@generated`tag in the header are ignored by rustfmt to avoid
formatting conflicts.

* Update DDSketch version

* Update 3rd party LICENSE

* Rename field in payload for serialization

* Use Errors for add_point

* Fix tests

* Enable trace_api_bytes

* Remove todo comments

* Update doc

* Disable formationg with skip instead of @generated

* Add tests for distribution metrics

---------

Co-authored-by: Vianney Ruhlmann <[email protected]>

Author: paullegranddc

.github/CODEOWNERS

@@ -21,3 +21,4 @@ serverless              @Datadog/serverless
 sidecar                 @Datadog/libdatadog-php @Datadog/libdatadog-apm
 sidecar-ffi             @Datadog/libdatadog-php @Datadog/libdatadog-apm
 data-pipeline*/         @Datadog/libdatadog-apm
+ddsketch		@Datadog/libdatadog-apm @Datadog/libdatadog-telemetry

Cargo.lock

@@ -28,7 +28,8 @@ members = [
   "serverless",
   "bin_tests",
   "data-pipeline",
-  "data-pipeline-ffi"
+  "data-pipeline-ffi",
+  "ddsketch",
 ]
 # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2
 resolver = "2"

Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "datadog-ddsketch"
+description = "Minimal implementation of Datadog's DDSketch"
+edition.workspace = true
+version.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prost = "0.11.6"
+
+[build-dependencies]
+prost-build = { version = "0.11.9", optional = true  }
+protoc-bin-vendored = { version = "3.0.0", optional = true }
+
+[features]
+generate-protobuf = ["dep:prost-build", "dep:protoc-bin-vendored"]

LICENSE-3rdparty.yml

@@ -0,0 +1,52 @@
+// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use std::error::Error;
+
+#[cfg(feature = "generate-protobuf")]
+use std::{
+    env,
+    fs::File,
+    io::{Read, Write},
+    path::Path,
+};
+
+#[cfg(feature = "generate-protobuf")]
+const HEADER: &str = "// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+";
+
+fn main() -> Result<(), Box<dyn Error>> {
+    #[cfg(feature = "generate-protobuf")]
+    {
+        // protoc is required to compile proto files. This uses protobuf_src to compile protoc
+        // from the source, setting the env var to tell prost_build where to find it.
+        std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap());
+
+        let mut config = prost_build::Config::new();
+
+        let cur_working_dir = env::var("CARGO_MANIFEST_DIR")?;
+        let output_path = Path::new(&cur_working_dir).join("src");
+
+        config.out_dir(output_path.clone());
+
+        println!("cargo:rerun-if-changed=src/pb/DDSketch.proto");
+        config.compile_protos(&["src/pb/DDSketch.proto"], &["src/"])?;
+
+        prepend_to_file(HEADER.as_bytes(), &output_path.join("pb.rs"))?;
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "generate-protobuf")]
+fn prepend_to_file(data: &[u8], file_path: &Path) -> Result<(), Box<dyn Error>> {
+    let mut f = File::open(file_path)?;
+    let mut content = data.to_owned();
+    f.read_to_end(&mut content)?;
+
+    let mut f = File::create(file_path)?;
+    f.write_all(content.as_slice())?;
+    Ok(())
+}