diff --git a/appsec/src/extension/commands/client_init.c b/appsec/src/extension/commands/client_init.c index 35d856a6bab..8945ba0cfca 100644 --- a/appsec/src/extension/commands/client_init.c +++ b/appsec/src/extension/commands/client_init.c @@ -164,10 +164,21 @@ static dd_result _pack_command( mpack_start_map(w, 2); dd_mpack_write_lstr(w, "enabled"); - mpack_write_bool(w, get_global_DD_EXPERIMENTAL_API_SECURITY_ENABLED()); - dd_mpack_write_lstr(w, "sample_rate"); - mpack_write(w, get_global_DD_API_SECURITY_REQUEST_SAMPLE_RATE()); +#define MIN_SE_SAMPLE_RATE 0.0001 + + double se_sample_rate = get_global_DD_API_SECURITY_REQUEST_SAMPLE_RATE(); + if (se_sample_rate >= MIN_SE_SAMPLE_RATE) { + mpack_write_bool(w, true); + + dd_mpack_write_lstr(w, "sample_rate"); + mpack_write(w, se_sample_rate); + } else { + mpack_write_bool(w, false); + + dd_mpack_write_lstr(w, "sample_rate"); + mpack_write(w, 0.0); + } mpack_finish_map(w); diff --git a/appsec/src/extension/configuration.h b/appsec/src/extension/configuration.h index 11a165f783f..d349fdbb92f 100644 --- a/appsec/src/extension/configuration.h +++ b/appsec/src/extension/configuration.h @@ -62,7 +62,6 @@ extern bool runtime_config_first_init; CONFIG(CUSTOM(STRING), DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING, "safe", .parser = dd_parse_automated_user_events_tracking) \ CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML, "") \ CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, "") \ - CONFIG(BOOL, DD_EXPERIMENTAL_API_SECURITY_ENABLED, "false") \ CONFIG(DOUBLE, DD_API_SECURITY_REQUEST_SAMPLE_RATE, "0.1") // clang-format on diff --git a/appsec/src/helper/client.cpp b/appsec/src/helper/client.cpp index aa15d8584ab..d7f859e790b 100644 --- a/appsec/src/helper/client.cpp +++ b/appsec/src/helper/client.cpp @@ -471,16 +471,6 @@ bool client::handle_command(network::request_shutdown::request &command) response->triggers = std::move(res->events); response->force_keep = res->force_keep; - for (const auto &[key, value] : res->schemas) { - std::string schema = value; - if (value.length() > max_plain_schema_allowed) { - auto encoded = compress(schema); - if (encoded) { - schema = base64_encode(encoded.value(), false); - } - } - response->meta.emplace(key, std::move(schema)); - } DD_STDLOG(DD_STDLOG_ATTACK_DETECTED); } else { diff --git a/appsec/src/helper/client.hpp b/appsec/src/helper/client.hpp index 9b72283fb03..2d5a8d040fa 100644 --- a/appsec/src/helper/client.hpp +++ b/appsec/src/helper/client.hpp @@ -21,7 +21,6 @@ namespace dds { class client { public: // Below this limit the encoding+compression might result on a longer string - static constexpr int max_plain_schema_allowed = 260; client(std::shared_ptr service_manager, network::base_broker::ptr &&broker) : service_manager_(std::move(service_manager)), diff --git a/appsec/src/helper/engine.cpp b/appsec/src/helper/engine.cpp index 8501a54f848..ff6cc16e279 100644 --- a/appsec/src/helper/engine.cpp +++ b/appsec/src/helper/engine.cpp @@ -75,7 +75,6 @@ std::optional engine::context::publish(parameter &¶m) std::vector event_data; std::unordered_set event_actions; - std::map schemas; for (auto &sub : common_->subscribers) { auto it = listeners_.find(sub); @@ -89,7 +88,6 @@ std::optional engine::context::publish(parameter &¶m) std::make_move_iterator(event->data.begin()), std::make_move_iterator(event->data.end())); event_actions.merge(event->actions); - schemas.merge(event->schemas); } } catch (std::exception &e) { SPDLOG_ERROR("subscriber failed: {}", e.what()); @@ -100,8 +98,7 @@ std::optional engine::context::publish(parameter &¶m) return std::nullopt; } - dds::engine::result res{ - action_type::record, {}, std::move(event_data), std::move(schemas)}; + dds::engine::result res{action_type::record, {}, std::move(event_data)}; // Currently the only action the extension can perform is block if (!event_actions.empty()) { // The extension can only handle one action, so we pick the first one diff --git a/appsec/src/helper/engine.hpp b/appsec/src/helper/engine.hpp index 29964c02473..fc50b428811 100644 --- a/appsec/src/helper/engine.hpp +++ b/appsec/src/helper/engine.hpp @@ -50,7 +50,6 @@ class engine { action_type type; std::unordered_map parameters; std::vector events; - std::map schemas; bool force_keep; }; diff --git a/appsec/src/helper/network/proto.hpp b/appsec/src/helper/network/proto.hpp index dc38be16667..50d89424e6a 100644 --- a/appsec/src/helper/network/proto.hpp +++ b/appsec/src/helper/network/proto.hpp @@ -280,10 +280,9 @@ struct request_shutdown { std::map meta; std::map metrics; - std::map schemas; MSGPACK_DEFINE( - verdict, parameters, triggers, force_keep, meta, metrics, schemas); + verdict, parameters, triggers, force_keep, meta, metrics); }; }; diff --git a/appsec/src/helper/subscriber/base.hpp b/appsec/src/helper/subscriber/base.hpp index 07e5ea76fa3..2c4d93a31f0 100644 --- a/appsec/src/helper/subscriber/base.hpp +++ b/appsec/src/helper/subscriber/base.hpp @@ -21,7 +21,6 @@ class subscriber { struct event { std::vector data; std::unordered_set actions; - std::map schemas; }; class listener { diff --git a/appsec/src/helper/subscriber/waf.cpp b/appsec/src/helper/subscriber/waf.cpp index e3157d3f11b..0f79ef92804 100644 --- a/appsec/src/helper/subscriber/waf.cpp +++ b/appsec/src/helper/subscriber/waf.cpp @@ -3,8 +3,6 @@ // // This product includes software developed at Datadog // (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. -#include "../std_logging.hpp" -#include "ddwaf.h" #include #include #include @@ -18,7 +16,11 @@ #include #include "../json_helper.hpp" +#include "../std_logging.hpp" #include "../tags.hpp" +#include "base64.h" +#include "compression.hpp" +#include "ddwaf.h" #include "waf.hpp" namespace dds::waf { @@ -39,11 +41,6 @@ dds::subscriber::event format_waf_result(ddwaf_result &res) output.data.emplace_back(std::move(parameter_to_json(event))); } - const parameter_view schemas{res.derivatives}; - for (const auto &schema : schemas) { - output.schemas.emplace( - schema.key(), std::move(parameter_to_json(schema))); - } } catch (const std::exception &e) { SPDLOG_ERROR("failed to parse WAF output: {}", e.what()); } @@ -209,6 +206,11 @@ std::optional instance::listener::call( // NOLINTNEXTLINE total_runtime_ += res.total_runtime / 1000.0; + const parameter_view schemas{res.derivatives}; + for (const auto &schema : schemas) { + schemas_.emplace(schema.key(), std::move(parameter_to_json(schema))); + } + switch (code) { case DDWAF_MATCH: return format_waf_result(res); @@ -236,6 +238,20 @@ void instance::listener::get_meta_and_metrics( { meta[std::string(tag::event_rules_version)] = ruleset_version_; metrics[tag::waf_duration] = total_runtime_; + + for (const auto &[key, value] : schemas_) { + std::string schema = value; + if (value.length() > max_plain_schema_allowed) { + auto encoded = compress(schema); + if (encoded) { + schema = base64_encode(encoded.value(), false); + } + } + + if (schema.length() <= max_schema_size) { + meta.emplace(key, std::move(schema)); + } + } } instance::instance(parameter &rule, std::map &meta, diff --git a/appsec/src/helper/subscriber/waf.hpp b/appsec/src/helper/subscriber/waf.hpp index b7d26f7ad5b..2f8b34305ba 100644 --- a/appsec/src/helper/subscriber/waf.hpp +++ b/appsec/src/helper/subscriber/waf.hpp @@ -23,7 +23,8 @@ void initialise_logging(spdlog::level::level_enum level); class instance : public dds::subscriber { public: static constexpr int default_waf_timeout_us = 10000; - + static constexpr int max_plain_schema_allowed = 260; + static constexpr int max_schema_size = 25000; using ptr = std::shared_ptr; class listener : public dds::subscriber::listener { public: @@ -46,6 +47,7 @@ class instance : public dds::subscriber { std::chrono::microseconds waf_timeout_; double total_runtime_{0.0}; std::string_view ruleset_version_; + std::map schemas_; }; // NOLINTNEXTLINE(google-runtime-references) diff --git a/appsec/tests/extension/api_security_env_variables.phpt b/appsec/tests/extension/api_security_env_variables.phpt index 8cfffe0b702..f387cb6be58 100644 --- a/appsec/tests/extension/api_security_env_variables.phpt +++ b/appsec/tests/extension/api_security_env_variables.phpt @@ -1,13 +1,10 @@ --TEST-- Set and test API security ini settings --ENV-- -DD_EXPERIMENTAL_API_SECURITY_ENABLED=1 DD_API_SECURITY_REQUEST_SAMPLE_RATE=0.8 --FILE-- --EXPECTF-- -string(1) "1" string(3) "0.8" diff --git a/appsec/tests/extension/rinit_rshutdown_basic.phpt b/appsec/tests/extension/rinit_rshutdown_basic.phpt index c7956b98e2e..0ed364bbece 100644 Binary files a/appsec/tests/extension/rinit_rshutdown_basic.phpt and b/appsec/tests/extension/rinit_rshutdown_basic.phpt differ diff --git a/appsec/tests/helper/broker_test.cpp b/appsec/tests/helper/broker_test.cpp index 13255a34183..b4e7bef2a63 100644 --- a/appsec/tests/helper/broker_test.cpp +++ b/appsec/tests/helper/broker_test.cpp @@ -166,7 +166,7 @@ TEST(BrokerTest, SendRequestShutdown) packer.pack_array(1); // Array of messages packer.pack_array(2); // First message pack_str(packer, "request_shutdown"); // Type - packer.pack_array(7); + packer.pack_array(6); pack_str(packer, "block"); packer.pack_map(2); pack_str(packer, "type"); @@ -179,7 +179,6 @@ TEST(BrokerTest, SendRequestShutdown) packer.pack_true(); // Force keep packer.pack_map(0); packer.pack_map(0); - packer.pack_map(0); const auto &expected_data = ss.str(); network::header_t h; diff --git a/appsec/tests/helper/client_test.cpp b/appsec/tests/helper/client_test.cpp index 4ed652f9ba6..c672724247a 100644 --- a/appsec/tests/helper/client_test.cpp +++ b/appsec/tests/helper/client_test.cpp @@ -2365,7 +2365,7 @@ TEST(ClientTest, SchemasOverTheLimitAreCompressed) int body_length = 0; int i = 0; // Lets generate a body which goes over max_plain_schema_allowed limit - while (body_length < client::max_plain_schema_allowed) { + while (body_length < waf::instance::max_plain_schema_allowed) { body.add(std::to_string(i), parameter::int64(i)); auto schema = parameter::array(); schema.add(parameter::int64(4)); diff --git a/appsec/tests/helper/waf_test.cpp b/appsec/tests/helper/waf_test.cpp index 63ece3cb2ca..88debfccf84 100644 --- a/appsec/tests/helper/waf_test.cpp +++ b/appsec/tests/helper/waf_test.cpp @@ -323,7 +323,8 @@ TEST(WafTest, SchemasAreAdded) EXPECT_FALSE(doc.HasParseError()); EXPECT_TRUE(doc.IsObject()); - EXPECT_FALSE(res->schemas.empty()); - EXPECT_STREQ(res->schemas["_dd.appsec.s.arg2"].c_str(), "[8]"); + ctx->get_meta_and_metrics(meta, metrics); + EXPECT_FALSE(meta.empty()); + EXPECT_STREQ(meta["_dd.appsec.s.arg2"].c_str(), "[8]"); } } // namespace dds