diff --git a/appsec/cmake/helper.cmake b/appsec/cmake/helper.cmake index f243f86b03..836285b3ae 100644 --- a/appsec/cmake/helper.cmake +++ b/appsec/cmake/helper.cmake @@ -18,7 +18,7 @@ set_target_properties(helper_objects PROPERTIES POSITION_INDEPENDENT_CODE 1) target_include_directories(helper_objects PUBLIC ${HELPER_INCLUDE_DIR}) target_compile_definitions(helper_objects PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE) -target_link_libraries(helper_objects PUBLIC libddwaf_objects pthread spdlog cpp-base64 msgpack_c RapidJSON::rapidjson Boost::system) +target_link_libraries(helper_objects PUBLIC libddwaf_objects pthread spdlog cpp-base64 msgpack_c RapidJSON::rapidjson Boost::system zlibstatic) add_executable(ddappsec-helper src/helper/main.cpp $ diff --git a/appsec/src/helper/client.cpp b/appsec/src/helper/client.cpp index 47877d8644..c7c6f77ebb 100644 --- a/appsec/src/helper/client.cpp +++ b/appsec/src/helper/client.cpp @@ -434,6 +434,19 @@ bool client::handle_command(network::request_shutdown::request &command) auto response = std::make_shared(); try { + auto sampler = service_->get_schema_sampler(); + std::optional scope; + if (sampler) { + scope = sampler->get(); + if (scope.has_value()) { + parameter context_processor = parameter::map(); + context_processor.add( + "extract-schema", parameter::as_boolean(true)); + command.data.add( + "waf.context.processor", std::move(context_processor)); + } + } + auto res = context_->publish(std::move(command.data)); if (res) { switch (res->type) { diff --git a/appsec/src/helper/compression.cpp b/appsec/src/helper/compression.cpp new file mode 100644 index 0000000000..29f86bc082 --- /dev/null +++ b/appsec/src/helper/compression.cpp @@ -0,0 +1,105 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +#include "compression.hpp" +#include +#include +#include + +namespace dds { + +namespace { +constexpr int64_t encoding = -0xf; +constexpr int max_round_decompression = 100; +// Taken from PHP approach +// https://heap.space/xref/PHP-7.3/ext/zlib/php_zlib.h?r=8d3f8ca1#36 +size_t estimate_compressed_size(size_t in_len) +{ + // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) + return (((size_t)((double)in_len * (double)1.015)) + 10 + 8 + 4 + 1); + // NOLINTEND(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) +} +} // namespace + +// The implementation of this function is based on how PHP does +// https://heap.space/xref/PHP-7.3/ext/zlib/zlib.c?r=9afce019#336 +std::optional compress(const std::string &text) +{ + std::string ret_string; + z_stream strm = {}; + + if (text.length() == 0) { + return std::nullopt; + } + + if (Z_OK == deflateInit2(&strm, -1, Z_DEFLATED, encoding, MAX_MEM_LEVEL, + Z_DEFAULT_STRATEGY)) { + auto size = estimate_compressed_size(text.length()); + ret_string.resize(size); + + // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast) + strm.next_in = reinterpret_cast(text.data()); + strm.next_out = reinterpret_cast(ret_string.data()); + // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast) + strm.avail_in = text.length(); + strm.avail_out = size; + + if (Z_STREAM_END == deflate(&strm, Z_FINISH)) { + deflateEnd(&strm); + /* size buffer down to actual length */ + ret_string.resize(strm.total_out); + ret_string.shrink_to_fit(); + return ret_string; + } + deflateEnd(&strm); + } + return std::nullopt; +} + +// Taken from PHP approach +// https://heap.space/xref/PHP-7.3/ext/zlib/zlib.c?r=9afce019#422 +std::optional uncompress(const std::string &compressed) +{ + int round = 0; + size_t used = 0; + size_t free; + size_t capacity; + z_stream strm = {}; + + if (compressed.length() < 1 || Z_OK != inflateInit2(&strm, encoding)) { + return std::nullopt; + } + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + strm.next_in = reinterpret_cast(compressed.data()); + strm.avail_in = compressed.length(); + std::string output; + int status = Z_OK; + + capacity = strm.avail_in; + output.resize(capacity); + while ((Z_BUF_ERROR == status || (Z_OK == status && strm.avail_in > 0)) && + ++round < max_round_decompression) { + strm.avail_out = free = capacity - used; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + strm.next_out = reinterpret_cast(output.data()) + used; + status = inflate(&strm, Z_NO_FLUSH); + used += free - strm.avail_out; + capacity += (output.size() >> 3) + 1; + output.resize(capacity); + } + if (status == Z_STREAM_END) { + inflateEnd(&strm); + output.resize(used); + output.shrink_to_fit(); + return output; + } + inflateEnd(&strm); + + return std::nullopt; +} + +} // namespace dds diff --git a/appsec/src/helper/compression.hpp b/appsec/src/helper/compression.hpp new file mode 100644 index 0000000000..36a8508350 --- /dev/null +++ b/appsec/src/helper/compression.hpp @@ -0,0 +1,16 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. +#pragma once + +#include +#include + +namespace dds { + +std::optional compress(const std::string &text); +std::optional uncompress(const std::string &compressed); + +} // namespace dds diff --git a/appsec/src/helper/engine_settings.hpp b/appsec/src/helper/engine_settings.hpp index 4b0fd488a1..e9bdf91012 100644 --- a/appsec/src/helper/engine_settings.hpp +++ b/appsec/src/helper/engine_settings.hpp @@ -16,8 +16,9 @@ namespace dds { struct schema_extraction_settings { static constexpr double default_sample_rate = 0.1; // 10% of requests + static constexpr bool default_enabled = false; - bool enabled = false; + bool enabled = default_enabled; double sample_rate = default_sample_rate; MSGPACK_DEFINE_MAP(enabled, sample_rate); diff --git a/appsec/src/helper/json_helper.cpp b/appsec/src/helper/json_helper.cpp index 7036763524..45d0c54f8c 100644 --- a/appsec/src/helper/json_helper.cpp +++ b/appsec/src/helper/json_helper.cpp @@ -37,6 +37,9 @@ void parameter_to_json_helper(const parameter_view &pv, T &output, auto sv = std::string_view(pv); output.SetString(sv.data(), sv.size(), alloc); } break; + case parameter_type::boolean: + output.SetBool(bool(pv)); + break; case parameter_type::map: output.SetObject(); for (const auto &v : pv) { diff --git a/appsec/src/helper/network/msgpack_helpers.cpp b/appsec/src/helper/network/msgpack_helpers.cpp index c1588f4824..7778c25c22 100644 --- a/appsec/src/helper/network/msgpack_helpers.cpp +++ b/appsec/src/helper/network/msgpack_helpers.cpp @@ -43,7 +43,7 @@ dds::parameter msgpack_to_param(const msgpack::object &o, unsigned depth = 0) case msgpack::type::STR: return dds::parameter::string(o.as()); case msgpack::type::BOOLEAN: - return dds::parameter::boolean(o.as()); + return dds::parameter::as_boolean(o.as()); case msgpack::type::FLOAT64: return dds::parameter::float64(o.as()); case msgpack::type::POSITIVE_INTEGER: diff --git a/appsec/src/helper/parameter.cpp b/appsec/src/helper/parameter.cpp index abc2ff9008..1cffdfde58 100644 --- a/appsec/src/helper/parameter.cpp +++ b/appsec/src/helper/parameter.cpp @@ -87,7 +87,7 @@ parameter parameter::string(std::string_view str) noexcept return parameter{obj}; } -parameter parameter::boolean(bool value) noexcept +parameter parameter::as_boolean(bool value) noexcept { ddwaf_object obj; ddwaf_object_bool(&obj, value); diff --git a/appsec/src/helper/parameter.hpp b/appsec/src/helper/parameter.hpp index 6f91bbd87c..c3dd43eb22 100644 --- a/appsec/src/helper/parameter.hpp +++ b/appsec/src/helper/parameter.hpp @@ -45,7 +45,7 @@ class parameter : public parameter_base { static parameter string(std::string_view str) noexcept; static parameter string(uint64_t value) noexcept; static parameter string(int64_t value) noexcept; - static parameter boolean(bool value) noexcept; + static parameter as_boolean(bool value) noexcept; static parameter float64(float value) noexcept; static parameter null() noexcept; diff --git a/appsec/src/helper/parameter_base.hpp b/appsec/src/helper/parameter_base.hpp index 5057c0da64..06fbe6521e 100644 --- a/appsec/src/helper/parameter_base.hpp +++ b/appsec/src/helper/parameter_base.hpp @@ -18,7 +18,8 @@ enum parameter_type : unsigned { uint64 = DDWAF_OBJ_UNSIGNED, string = DDWAF_OBJ_STRING, map = DDWAF_OBJ_MAP, - array = DDWAF_OBJ_ARRAY + array = DDWAF_OBJ_ARRAY, + boolean = DDWAF_OBJ_BOOL }; class parameter_base : public ddwaf_object { @@ -80,6 +81,10 @@ class parameter_base : public ddwaf_object { { return ddwaf_object::type == DDWAF_OBJ_SIGNED; } + [[nodiscard]] bool is_boolean() const noexcept + { + return ddwaf_object::type == DDWAF_OBJ_BOOL; + } [[nodiscard]] bool is_valid() const noexcept { return ddwaf_object::type != DDWAF_OBJ_INVALID; @@ -116,6 +121,14 @@ class parameter_base : public ddwaf_object { return intValue; } + explicit operator bool() const + { + if (!is_boolean()) { + throw bad_cast("parameter not a bool"); + } + return boolean; + } + [[nodiscard]] std::string debug_str() const noexcept; using length_type = decltype(nbEntries); diff --git a/appsec/src/helper/sampler.hpp b/appsec/src/helper/sampler.hpp new file mode 100644 index 0000000000..620e7aaefb --- /dev/null +++ b/appsec/src/helper/sampler.hpp @@ -0,0 +1,91 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace dds { +static const double min_rate = 0.0001; +class sampler { +public: + sampler(double sample_rate) : sample_rate_(sample_rate) + { + // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) + if (sample_rate_ <= 0) { + sample_rate_ = 0; + } else if (sample_rate_ > 1) { + sample_rate_ = 1; + } else if (sample_rate_ < min_rate) { + sample_rate_ = min_rate; + } + // NOLINTEND(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) + } + class scope { + public: + explicit scope(std::atomic &concurrent) : concurrent_(&concurrent) + { + *concurrent_ = true; + } + + scope(const scope &) = delete; + scope &operator=(const scope &) = delete; + scope(scope &&oth) noexcept + { + concurrent_ = oth.concurrent_; + oth.concurrent_ = nullptr; + } + scope &operator=(scope &&oth) + { + concurrent_ = oth.concurrent_; + oth.concurrent_ = nullptr; + + return *this; + } + + ~scope() + { + if (concurrent_ != nullptr) { + *concurrent_ = false; + } + } + + protected: + std::atomic *concurrent_; + }; + + std::optional get() + { + const std::lock_guard lock_guard(mtx_); + + std::optional result = std::nullopt; + + if (!concurrent_ && floor(request_ * sample_rate_) != + floor((request_ + 1) * sample_rate_)) { + result = {scope{concurrent_}}; + } + + if (request_ < UINT_MAX) { + request_++; + } else { + request_ = 1; + } + + return result; + } + +protected: + unsigned request_{1}; + double sample_rate_; + std::atomic concurrent_{false}; + std::mutex mtx_; +}; +} // namespace dds diff --git a/appsec/src/helper/service.cpp b/appsec/src/helper/service.cpp index 2084b3ab74..687f6700f2 100644 --- a/appsec/src/helper/service.cpp +++ b/appsec/src/helper/service.cpp @@ -10,7 +10,8 @@ namespace dds { service::service(std::shared_ptr engine, std::shared_ptr service_config, - dds::remote_config::client_handler::ptr &&client_handler) + dds::remote_config::client_handler::ptr &&client_handler, + const schema_extraction_settings &schema_extraction_settings) : engine_(std::move(engine)), service_config_(std::move(service_config)), client_handler_(std::move(client_handler)) { @@ -22,6 +23,14 @@ service::service(std::shared_ptr engine, if (client_handler_) { client_handler_->start(); } + + double sample_rate = schema_extraction_settings.sample_rate; + + if (!schema_extraction_settings.enabled) { + sample_rate = 0; + } + + schema_sampler_ = std::make_shared(sample_rate); } service::ptr service::from_settings(service_identifier &&id, @@ -38,7 +47,7 @@ service::ptr service::from_settings(service_identifier &&id, std::move(id), eng_settings, service_config, rc_settings, engine_ptr, dynamic_enablement); - return std::make_shared( - engine_ptr, std::move(service_config), std::move(client_handler)); + return std::make_shared(engine_ptr, std::move(service_config), + std::move(client_handler), eng_settings.schema_extraction); } } // namespace dds diff --git a/appsec/src/helper/service.hpp b/appsec/src/helper/service.hpp index 8998d3796a..416f5a2393 100644 --- a/appsec/src/helper/service.hpp +++ b/appsec/src/helper/service.hpp @@ -8,6 +8,7 @@ #include "engine.hpp" #include "exception.hpp" #include "remote_config/client_handler.hpp" +#include "sampler.hpp" #include "service_config.hpp" #include "service_identifier.hpp" #include "std_logging.hpp" @@ -26,7 +27,8 @@ class service { service(std::shared_ptr engine, std::shared_ptr service_config, - dds::remote_config::client_handler::ptr &&client_handler); + dds::remote_config::client_handler::ptr &&client_handler, + const schema_extraction_settings &schema_extraction_settings = {}); service(const service &) = delete; service &operator=(const service &) = delete; @@ -68,10 +70,13 @@ class service { return service_config_; } + std::shared_ptr get_schema_sampler() { return schema_sampler_; } + protected: std::shared_ptr engine_{}; std::shared_ptr service_config_{}; dds::remote_config::client_handler::ptr client_handler_{}; + std::shared_ptr schema_sampler_; }; } // namespace dds diff --git a/appsec/tests/fuzzer/CMakeLists.txt b/appsec/tests/fuzzer/CMakeLists.txt index 8d4dccb779..d3f095ceff 100644 --- a/appsec/tests/fuzzer/CMakeLists.txt +++ b/appsec/tests/fuzzer/CMakeLists.txt @@ -11,10 +11,10 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSIO ) execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE) - + target_compile_definitions(ddappsec_helper_fuzzer PUBLIC ZLIB_CONST=1) target_link_directories(ddappsec_helper_fuzzer PRIVATE ${LLVM_RUNTIME_DIR}) target_link_libraries(ddappsec_helper_fuzzer - PRIVATE libddwaf_objects pthread spdlog cpp-base64 msgpack_c lib_rapidjson Boost::system libclang_rt.fuzzer_no_main-${ARCHITECTURE}.a) + PRIVATE libddwaf_objects pthread spdlog cpp-base64 msgpack_c lib_rapidjson Boost::system libclang_rt.fuzzer_no_main-${ARCHITECTURE}.a zlibstatic) add_executable(corpus_generator corpus_generator.cpp) target_link_libraries(corpus_generator PRIVATE helper_objects libddwaf_objects pthread spdlog) diff --git a/appsec/tests/helper/compression_test.cpp b/appsec/tests/helper/compression_test.cpp new file mode 100644 index 0000000000..fcf82bb6f5 --- /dev/null +++ b/appsec/tests/helper/compression_test.cpp @@ -0,0 +1,78 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. +#include "common.hpp" +#include +#include + +namespace dds { + +TEST(CompressionTest, Compress) +{ + { + std::string text = "Some string to compress"; + std::optional compressed = compress(text); + + auto uncompressed = uncompress(compressed.value()); + + EXPECT_STREQ(text.c_str(), uncompressed->c_str()); + } + { + std::string text = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer " + "mattis libero quis velit tempus ultrices. Sed eget tortor lectus. " + "Fusce posuere luctus luctus. Vestibulum ante ipsum primis in " + "faucibus orci luctus et ultrices posuere cubilia curae; Quisque " + "dui tortor, dictum consequat posuere at, tempus non nunc. Sed " + "libero dolor, bibendum sed lacus quis, viverra fringilla felis. " + "Integer rutrum hendrerit venenatis. Ut aliquam nisi vitae libero " + "blandit venenatis. Suspendisse viverra id mauris id placerat. " + "Mauris sodales venenatis rhoncus. Maecenas finibus venenatis leo, " + "ut convallis urna faucibus et. Mauris a euismod libero, nec " + "pellentesque neque. Nunc pretium a mi et malesuada. Donec eget " + "leo semper, mollis velit non, elementum felis. Vivamus eu varius " + "ipsum.\n" + "\n" + "In consectetur nisi sit amet turpis lobortis euismod. Vestibulum " + "iaculis dui id ipsum consectetur, quis facilisis sem tristique. " + "Quisque non velit porttitor, luctus ex at, lobortis diam. " + "Curabitur eu metus maximus, vehicula turpis at, suscipit sapien. " + "Duis nibh purus, gravida vitae cursus at, placerat vel erat. " + "Etiam malesuada purus non pellentesque aliquam. Maecenas egestas " + "suscipit eros sit amet fermentum. Vivamus ullamcorper molestie " + "tempus. Maecenas semper accumsan vulputate. Praesent cursus ante " + "risus, sed tincidunt nibh ullamcorper ultricies. Suspendisse " + "potenti."; + std::optional compressed = compress(text); + + auto uncompressed = uncompress(compressed.value()); + + EXPECT_STREQ(text.c_str(), uncompressed->c_str()); + } + { + std::string text = ""; + std::optional compressed = compress(text); + + EXPECT_FALSE(compressed.has_value()); + } +} + +TEST(CompressionTest, Uncompressed) +{ + { + std::string text = ""; + std::optional compressed = uncompress(text); + + EXPECT_FALSE(compressed.has_value()); + } + { + std::string text = "something not compressed"; + std::optional compressed = uncompress(text); + + EXPECT_FALSE(compressed.has_value()); + } +} + +} // namespace dds diff --git a/appsec/tests/helper/json_helper_test.cpp b/appsec/tests/helper/json_helper_test.cpp index 72ea07bcf4..9e0cff9d0f 100644 --- a/appsec/tests/helper/json_helper_test.cpp +++ b/appsec/tests/helper/json_helper_test.cpp @@ -48,4 +48,25 @@ TEST(JsonHelperTest, InvalidTypeToJson) EXPECT_EQ("", result); } +TEST(JsonHelperTest, BoolType) +{ + { + ddwaf_object obj; + ddwaf_object_bool(&obj, false); + + parameter_view pv(obj); + std::string result = parameter_to_json(pv); + EXPECT_EQ("false", result); + } + + { + ddwaf_object obj; + ddwaf_object_bool(&obj, true); + + parameter_view pv(obj); + std::string result = parameter_to_json(pv); + EXPECT_EQ("true", result); + } +} + } // namespace dds diff --git a/appsec/tests/helper/parameter_test.cpp b/appsec/tests/helper/parameter_test.cpp index 9fbdb348b1..0874a9193a 100644 --- a/appsec/tests/helper/parameter_test.cpp +++ b/appsec/tests/helper/parameter_test.cpp @@ -332,4 +332,19 @@ TEST(ParameterTest, StaticCastFromArrayObject) EXPECT_THROW(p[size], std::out_of_range); } +TEST(ParameterTest, Bool) +{ + bool value = false; + parameter p = parameter::as_boolean(value); + EXPECT_EQ(p.type(), parameter_type::boolean); + EXPECT_THROW(p[0], invalid_type); + + EXPECT_THROW(auto s = std::string(p), bad_cast); + EXPECT_THROW(auto sv = std::string_view(p), bad_cast); + EXPECT_THROW(auto u64 = uint64_t(p), bad_cast); + EXPECT_NO_THROW(auto boolean = bool(p)); + + EXPECT_EQ(value, p.boolean); +} + } // namespace dds diff --git a/appsec/tests/helper/sampler_test.cpp b/appsec/tests/helper/sampler_test.cpp new file mode 100644 index 0000000000..eb2c90d8a8 --- /dev/null +++ b/appsec/tests/helper/sampler_test.cpp @@ -0,0 +1,226 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. +#include "common.hpp" +#include +#include + +namespace dds { + +namespace mock { + +class sampler : public dds::sampler { +public: + sampler(double sample_rate) : dds::sampler(sample_rate) {} + void set_request(unsigned int i) { request_ = i; } + auto get_request() { return request_; } +}; + +} // namespace mock + +std::atomic picked = 0; + +void count_picked(dds::sampler &sampler, int iterations) +{ + for (int i = 0; i < iterations; i++) { + auto is_pick = sampler.get(); + if (is_pick != std::nullopt) { + picked++; + } + } +} + +TEST(SamplerTest, ItPicksAllWhenRateIs1) +{ + sampler s(1); + picked = 0; + count_picked(s, 100); + + EXPECT_EQ(100, picked); +} + +TEST(SamplerTest, ItPicksNoneWhenRateIs0) +{ + sampler s(0); + picked = 0; + count_picked(s, 100); + + EXPECT_EQ(0, picked); +} + +TEST(SamplerTest, ItPicksHalfWhenPortionGiven) +{ + sampler s(0.5); + picked = 0; + count_picked(s, 100); + + EXPECT_EQ(50, picked); +} + +TEST(SamplerTest, ItResetTokensAfter100Calls) +{ + sampler s(1); + + picked = 0; + count_picked(s, 100); + count_picked(s, 100); + + EXPECT_EQ(200, picked); +} + +TEST(SamplerTest, ItWorksWithDifferentMagnitudes) +{ + { + sampler s(0.1); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(1, picked); + } + { + sampler s(0.5); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(5, picked); + } + { + sampler s(0.01); + picked = 0; + count_picked(s, 100); + + EXPECT_EQ(1, picked); + } + { + sampler s(0.02); + picked = 0; + count_picked(s, 100); + + EXPECT_EQ(2, picked); + } + { + sampler s(0.001); + picked = 0; + count_picked(s, 1000); + + EXPECT_EQ(1, picked); + } + { + sampler s(0.003); + picked = 0; + count_picked(s, 1000); + + EXPECT_EQ(3, picked); + } + { + sampler s(0.0001); + picked = 0; + count_picked(s, 10000); + + EXPECT_EQ(1, picked); + } + { + sampler s(0.0007); + picked = 0; + count_picked(s, 10000); + + EXPECT_EQ(7, picked); + } + { + sampler s(0.123); + picked = 0; + count_picked(s, 1000); + + EXPECT_EQ(123, picked); + } + { + sampler s(0.6); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(6, picked); + } +} + +TEST(SamplerTest, TestInvalidSampleRatesDefaultToTenPercent) +{ + { + sampler s(2); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(10, picked); + } + { + sampler s(-1); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(0, picked); + } + { // Below limit goes to default 10 percent + sampler s(0.000001); + picked = 0; + count_picked(s, 1000000); + + EXPECT_EQ(100, picked); + } +} + +TEST(SamplerTest, TestLimits) +{ + { + sampler s(0); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(0, picked); + } + { + sampler s(1); + picked = 0; + count_picked(s, 10); + + EXPECT_EQ(10, picked); + } + { + sampler s(0.0001); + picked = 0; + count_picked(s, 10000); + + EXPECT_EQ(1, picked); + } +} + +TEST(SamplerTest, TestOverflow) +{ + mock::sampler s(0); + s.set_request(UINT_MAX); + s.get(); + EXPECT_EQ(1, s.get_request()); +} + +TEST(ScopeTest, TestConcurrent) +{ + std::atomic concurrent = false; + { + auto s = sampler::scope(std::ref(concurrent)); + EXPECT_TRUE(concurrent); + } + EXPECT_FALSE(concurrent); +} + +TEST(ScopeTest, TestItDoesNotPickTokenUntilScopeReleased) +{ + sampler sampler(1); + auto is_pick = sampler.get(); + EXPECT_TRUE(is_pick != std::nullopt); + is_pick = sampler.get(); + EXPECT_FALSE(is_pick != std::nullopt); + is_pick.reset(); + is_pick = sampler.get(); + EXPECT_TRUE(is_pick != std::nullopt); +} +} // namespace dds diff --git a/appsec/tests/helper/service_test.cpp b/appsec/tests/helper/service_test.cpp index b801ec6e44..a7ef5241e9 100644 --- a/appsec/tests/helper/service_test.cpp +++ b/appsec/tests/helper/service_test.cpp @@ -39,4 +39,70 @@ TEST(ServiceTest, NullServiceHandler) EXPECT_EQ(engine.get(), svc.get_engine().get()); } +TEST(ServiceTest, ServicePickSchemaExtractionSamples) +{ + std::shared_ptr engine{engine::create()}; + + service_identifier sid{"service", {"extra01", "extra02"}, "env", + "tracer_version", "app_version", "runtime_id"}; + auto client = std::make_unique(sid); + auto service_config = std::make_shared(); + engine_settings engine_settings = {}; + engine_settings.rules_file = create_sample_rules_ok(); + std::map meta; + std::map metrics; + + { // Constructor. It picks based on rate + double all_requests_are_picked = 1.0; + auto s = service( + engine, service_config, nullptr, {true, all_requests_are_picked}); + + EXPECT_TRUE(s.get_schema_sampler()->get().has_value()); + } + + { // Constructor. It does not pick based on rate + double no_request_is_picked = 0.0; + auto s = service( + engine, service_config, nullptr, {true, no_request_is_picked}); + + EXPECT_FALSE(s.get_schema_sampler()->get().has_value()); + } + + { // Constructor. It does not pick if disabled + double all_requests_are_picked = 1.0; + bool schema_extraction_disabled = false; + auto s = service(engine, service_config, nullptr, + {schema_extraction_disabled, all_requests_are_picked}); + + EXPECT_FALSE(s.get_schema_sampler()->get().has_value()); + } + + { // Static constructor. It picks based on rate + engine_settings.schema_extraction.enabled = true; + engine_settings.schema_extraction.sample_rate = 1.0; + auto service = service::from_settings( + service_identifier(sid), engine_settings, {}, meta, metrics, false); + + EXPECT_TRUE(service->get_schema_sampler()->get().has_value()); + } + + { // Static constructor. It does not pick based on rate + engine_settings.schema_extraction.enabled = true; + engine_settings.schema_extraction.sample_rate = 0.0; + auto service = service::from_settings( + service_identifier(sid), engine_settings, {}, meta, metrics, false); + + EXPECT_FALSE(service->get_schema_sampler()->get().has_value()); + } + + { // Static constructor. It does not pick if disabled + engine_settings.schema_extraction.enabled = false; + engine_settings.schema_extraction.sample_rate = 1.0; + auto service = service::from_settings( + service_identifier(sid), engine_settings, {}, meta, metrics, false); + + EXPECT_FALSE(service->get_schema_sampler()->get().has_value()); + } +} + } // namespace dds diff --git a/appsec/third_party/CMakeLists.txt b/appsec/third_party/CMakeLists.txt index 5afce81af9..42857843b5 100644 --- a/appsec/third_party/CMakeLists.txt +++ b/appsec/third_party/CMakeLists.txt @@ -45,6 +45,20 @@ FetchContent_Declare( GIT_TAG eb3220622e73a4889eee355ffa37972b3cac3df5) FetchContent_MakeAvailable(spdlog) +set(ZLIB_VERSION v1.3) +FetchContent_Declare( + zlib + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG ${ZLIB_VERSION} +) +FetchContent_MakeAvailable(zlib) +if(NOT(MSVC)) + set_property(TARGET zlibstatic PROPERTY POSITION_INDEPENDENT_CODE ON) +endif() + +target_compile_definitions(zlibstatic PUBLIC ZLIB_CONST=1) +target_include_directories(zlibstatic INTERFACE ${zlib_SOURCE_DIR} ${zlib_BINARY_DIR}) + include(ExternalProject) ExternalProject_Add(event_rules GIT_REPOSITORY https://github.com/DataDog/appsec-event-rules.git