Merge back changes 04122023 (#2404)

Author: estringana

appsec/src/helper/client.cpp

@@ -3,13 +3,16 @@
 //
 // This product includes software developed at Datadog
 // (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
+
 #include <chrono>
 #include <spdlog/spdlog.h>
 #include <stdexcept>
 #include <string>
 #include <thread>
 
+#include "base64.h"
 #include "client.hpp"
+#include "compression.hpp"
 #include "exception.hpp"
 #include "network/broker.hpp"
 #include "network/proto.hpp"
@@ -150,7 +153,7 @@ bool client::handle_command(const network::client_init::request &command)
     auto &&eng_settings = command.engine_settings;
     DD_STDLOG(DD_STDLOG_STARTUP);
 
-    std::map<std::string_view, std::string> meta;
+    std::map<std::string, std::string> meta;
     std::map<std::string_view, double> metrics;
 
     std::vector<std::string> errors;
@@ -433,6 +436,7 @@ bool client::handle_command(network::request_shutdown::request &command)
     auto free_ctx = defer([this]() { this->context_.reset(); });
 
     auto response = std::make_shared<network::request_shutdown::response>();
+
     try {
         auto sampler = service_->get_schema_sampler();
         std::optional<sampler::scope> scope;
@@ -467,6 +471,16 @@ bool client::handle_command(network::request_shutdown::request &command)
 
             response->triggers = std::move(res->events);
             response->force_keep = res->force_keep;
+            for (const auto &[key, value] : res->schemas) {
+                std::string schema = value;
+                if (value.length() > max_plain_schema_allowed) {
+                    auto encoded = compress(schema);
+                    if (encoded) {
+                        schema = base64_encode(encoded.value(), false);
+                    }
+                }
+                response->meta.emplace(key, std::move(schema));
+            }
 
             DD_STDLOG(DD_STDLOG_ATTACK_DETECTED);
         } else {

appsec/src/helper/client.hpp

@@ -20,6 +20,8 @@ namespace dds {
 
 class client {
 public:
+    // Below this limit the encoding+compression might result on a longer string
+    static constexpr int max_plain_schema_allowed = 260;
     client(std::shared_ptr<service_manager> service_manager,
         network::base_broker::ptr &&broker)
         : service_manager_(std::move(service_manager)),

appsec/src/helper/engine.cpp

@@ -26,7 +26,7 @@ void engine::subscribe(const subscriber::ptr &sub)
 }
 
 void engine::update(engine_ruleset &ruleset,
-    std::map<std::string_view, std::string> &meta,
+    std::map<std::string, std::string> &meta,
     std::map<std::string_view, double> &metrics)
 {
     auto new_actions =
@@ -75,6 +75,7 @@ std::optional<engine::result> engine::context::publish(parameter &&param)
 
     std::vector<std::string> event_data;
     std::unordered_set<std::string> event_actions;
+    std::map<std::string, std::string> schemas;
 
     for (auto &sub : common_->subscribers) {
         auto it = listeners_.find(sub);
@@ -88,6 +89,7 @@ std::optional<engine::result> engine::context::publish(parameter &&param)
                     std::make_move_iterator(event->data.begin()),
                     std::make_move_iterator(event->data.end()));
                 event_actions.merge(event->actions);
+                schemas.merge(event->schemas);
             }
         } catch (std::exception &e) {
             SPDLOG_ERROR("subscriber failed: {}", e.what());
@@ -98,7 +100,8 @@ std::optional<engine::result> engine::context::publish(parameter &&param)
         return std::nullopt;
     }
 
-    dds::engine::result res{action_type::record, {}, std::move(event_data)};
+    dds::engine::result res{
+        action_type::record, {}, std::move(event_data), std::move(schemas)};
     // Currently the only action the extension can perform is block
     if (!event_actions.empty()) {
         // The extension can only handle one action, so we pick the first one
@@ -119,7 +122,7 @@ std::optional<engine::result> engine::context::publish(parameter &&param)
 }
 
 void engine::context::get_meta_and_metrics(
-    std::map<std::string_view, std::string> &meta,
+    std::map<std::string, std::string> &meta,
     std::map<std::string_view, double> &metrics)
 {
     for (const auto &[subscriber, listener] : listeners_) {
@@ -231,7 +234,7 @@ engine::action_map engine::parse_actions(
 }
 
 engine::ptr engine::from_settings(const dds::engine_settings &eng_settings,
-    std::map<std::string_view, std::string> &meta,
+    std::map<std::string, std::string> &meta,
     std::map<std::string_view, double> &metrics)
 
 {

appsec/src/helper/engine.hpp

@@ -50,6 +50,7 @@ class engine {
         action_type type;
         std::unordered_map<std::string, std::string> parameters;
         std::vector<std::string> events;
+        std::map<std::string, std::string> schemas;
         bool force_keep;
     };
 
@@ -77,7 +78,7 @@ class engine {
 
         std::optional<result> publish(parameter &&param);
         // NOLINTNEXTLINE(google-runtime-references)
-        void get_meta_and_metrics(std::map<std::string_view, std::string> &meta,
+        void get_meta_and_metrics(std::map<std::string, std::string> &meta,
             std::map<std::string_view, double> &metrics);
 
     protected:
@@ -94,7 +95,7 @@ class engine {
     virtual ~engine() = default;
 
     static engine::ptr from_settings(const dds::engine_settings &eng_settings,
-        std::map<std::string_view, std::string> &meta,
+        std::map<std::string, std::string> &meta,
         std::map<std::string_view, double> &metrics);
 
     static auto create(
@@ -111,7 +112,7 @@ class engine {
     // Update is not thread-safe, although only one remote config client should
     // be able to update it so in practice it should not be a problem.
     virtual void update(engine_ruleset &ruleset,
-        std::map<std::string_view, std::string> &meta,
+        std::map<std::string, std::string> &meta,
         std::map<std::string_view, double> &metrics);
 
     // Only exposed for testing purposes

appsec/src/helper/network/proto.hpp

@@ -126,7 +126,7 @@ struct client_init {
         std::string version{dds::php_ddappsec_version};
         std::vector<std::string> errors;
 
-        std::map<std::string_view, std::string> meta;
+        std::map<std::string, std::string> meta;
         std::map<std::string_view, double> metrics;
 
         MSGPACK_DEFINE(status, version, errors, meta, metrics);
@@ -278,7 +278,7 @@ struct request_shutdown {
 
         bool force_keep;
 
-        std::map<std::string_view, std::string> meta;
+        std::map<std::string, std::string> meta;
         std::map<std::string_view, double> metrics;
         std::map<std::string_view, std::string> schemas;
 

appsec/src/helper/remote_config/listeners/engine_listener.cpp

@@ -80,7 +80,7 @@ void engine_listener::commit()
     }
 
     // TODO find a way to provide this information to the service
-    std::map<std::string_view, std::string> meta;
+    std::map<std::string, std::string> meta;
     std::map<std::string_view, double> metrics;
 
     engine_ruleset ruleset = dds::engine_ruleset(std::move(ruleset_));

appsec/src/helper/sampler.hpp

@@ -33,7 +33,7 @@ class sampler {
     public:
         explicit scope(std::atomic<bool> &concurrent) : concurrent_(&concurrent)
         {
-            *concurrent_ = true;
+            concurrent_->store(true, std::memory_order_relaxed);
         }
 
         scope(const scope &) = delete;
@@ -54,7 +54,7 @@ class sampler {
         ~scope()
         {
             if (concurrent_ != nullptr) {
-                *concurrent_ = false;
+                concurrent_->store(false, std::memory_order_relaxed);
             }
         }
 
@@ -73,7 +73,7 @@ class sampler {
             result = {scope{concurrent_}};
         }
 
-        if (request_ < UINT_MAX) {
+        if (request_ < std::numeric_limits<unsigned>::max()) {
             request_++;
         } else {
             request_ = 1;

appsec/src/helper/service.cpp

@@ -36,7 +36,7 @@ service::service(std::shared_ptr<engine> engine,
 service::ptr service::from_settings(service_identifier &&id,
     const dds::engine_settings &eng_settings,
     const remote_config::settings &rc_settings,
-    std::map<std::string_view, std::string> &meta,
+    std::map<std::string, std::string> &meta,
     std::map<std::string_view, double> &metrics, bool dynamic_enablement)
 {
     auto engine_ptr = engine::from_settings(eng_settings, meta, metrics);

appsec/src/helper/service.hpp

@@ -41,7 +41,7 @@ class service {
     static service::ptr from_settings(service_identifier &&id,
         const dds::engine_settings &eng_settings,
         const remote_config::settings &rc_settings,
-        std::map<std::string_view, std::string> &meta,
+        std::map<std::string, std::string> &meta,
         std::map<std::string_view, double> &metrics, bool dynamic_enablement);
 
     virtual void register_runtime_id(const std::string &id)
@@ -70,7 +70,10 @@ class service {
         return service_config_;
     }
 
-    std::shared_ptr<sampler> get_schema_sampler() { return schema_sampler_; }
+    [[nodiscard]] std::shared_ptr<sampler> get_schema_sampler()
+    {
+        return schema_sampler_;
+    }
 
 protected:
     std::shared_ptr<engine> engine_{};

appsec/src/helper/service_manager.cpp

@@ -10,7 +10,7 @@ namespace dds {
 std::shared_ptr<service> service_manager::create_service(
     service_identifier &&id, const engine_settings &settings,
     const remote_config::settings &rc_settings,
-    std::map<std::string_view, std::string> &meta,
+    std::map<std::string, std::string> &meta,
     std::map<std::string_view, double> &metrics, bool dynamic_enablement)
 {
     const std::lock_guard guard{mutex_};