Make it possible to retrieve before provider is initialized

Create an instance of provider_options in server_state, and
provider_options so that it does not depend on provider.
Call into config_service when provider is created, i.e. after
library is loaded, before it is initialized, so that we can
inject additional options at startup.

Author: sciascid

Diff for: include/wsrep/provider.hpp

@@ -51,6 +51,7 @@ namespace wsrep
     class event_service;
     class client_service;
     class connection_monitor_service;
+    class provider_options;
     class stid
     {
     public:
@@ -521,6 +522,7 @@ namespace wsrep
             wsrep::server_state&,
             const std::string& provider_spec,
             const std::function<std::string()>& provider_options_cb,
+            provider_options& options,
             const wsrep::provider::services& services
             = wsrep::provider::services());
     protected:

Diff for: include/wsrep/provider_options.hpp

@@ -214,7 +214,7 @@ namespace wsrep
             int flags_;
         };
 
-        provider_options(wsrep::provider&);
+        provider_options();
         provider_options(const provider_options&) = delete;
         provider_options& operator=(const provider_options&) = delete;
 
@@ -224,6 +224,7 @@ namespace wsrep
          * Individual options should be accessed through set()/get().
          *
          * @return Provider status code.
+         * @deprecated Provider options are now loaded in provider ctor
          */
         enum wsrep::provider::status initial_options();
 
@@ -241,8 +242,9 @@ namespace wsrep
          * @return wsrep::provider::error_size_exceeded if memory could
          *         not be allocated for the new value.
          */
-        enum wsrep::provider::status set(const std::string& name,
-                                         std::unique_ptr<option_value> value);
+        enum wsrep::provider::status set(wsrep::provider& provider,
+                                          const std::string& name,
+                                          std::unique_ptr<option_value> value);
 
         /**
          * Create a new option with default value.
@@ -255,7 +257,6 @@ namespace wsrep
         void for_each(const std::function<void(option*)>& fn);
 
     private:
-        provider& provider_;
         using options_map = std::map<std::string, std::unique_ptr<option>>;
         options_map options_;
     };

Diff for: include/wsrep/server_state.hpp

@@ -91,6 +91,7 @@
 #include "provider.hpp"
 #include "compiler.hpp"
 #include "xid.hpp"
+#include "provider_options.hpp"
 
 #include <memory>
 #include <deque>
@@ -356,6 +357,11 @@ namespace wsrep
             return *provider_;
         }
 
+        wsrep::provider_options* provider_options()
+        {
+            return &provider_options_;
+        }
+
         /**
          * Initialize connection to cluster.
          *
@@ -664,6 +670,7 @@ namespace wsrep
             , streaming_appliers_recovered_()
             , provider_()
             , provider_factory_(wsrep::provider::make_provider)
+            , provider_options_()
             , name_(name)
             , id_(wsrep::id::undefined())
             , incoming_address_(incoming_address)
@@ -746,6 +753,7 @@ namespace wsrep
         bool streaming_appliers_recovered_;
         std::unique_ptr<wsrep::provider> provider_;
         provider_factory_func provider_factory_;
+        wsrep::provider_options provider_options_;
         std::string name_;
         wsrep::id id_;
         std::string incoming_address_;

Diff for: src/config_service_v1.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "config_service_v1.hpp"
+#include "wsrep_api.h"
 #include "service_helpers.hpp"
 #include "v26/wsrep_config_service.h"
 #include "wsrep/logger.hpp"
@@ -147,10 +148,9 @@ static void config_service_v1_deinit(void* dlh)
         dlh, WSREP_CONFIG_SERVICE_DEINIT_FUNC_V1, "config service v1");
 }
 
-int wsrep::config_service_v1_fetch(wsrep::provider& provider,
+int wsrep::config_service_v1_fetch(struct wsrep_st* wsrep,
                                    wsrep::provider_options* options)
 {
-    struct wsrep_st* wsrep = (struct wsrep_st*)provider.native();
     if (wsrep == nullptr)
     {
         // Not a provider which was loaded via wsrep-API

Diff for: src/config_service_v1.hpp

@@ -20,11 +20,13 @@
 #ifndef WSREP_CONFIG_SERVICE_V1_HPP
 #define WSREP_CONFIG_SERVICE_V1_HPP
 
+struct wsrep_st;
+
 namespace wsrep
 {
     class provider;
     class provider_options;
-    int config_service_v1_fetch(provider& provider, provider_options* opts);
+    int config_service_v1_fetch(struct wsrep_st* wsrep, provider_options* opts);
 } // namespace wsrep
 
 #endif // WSREP_CONFIG_SERVICE_V1_HPP

Diff for: src/provider.cpp

@@ -30,12 +30,14 @@ std::unique_ptr<wsrep::provider> wsrep::provider::make_provider(
     wsrep::server_state& server_state,
     const std::string& provider_spec,
     const std::function<std::string()>& provider_options_cb,
+    provider_options& provider_options,
     const wsrep::provider::services& services)
 {
     try
     {
         return std::unique_ptr<wsrep::provider>(new wsrep::wsrep_provider_v26(
-            server_state, provider_spec, provider_options_cb, services));
+            server_state, provider_spec, provider_options_cb, provider_options,
+            services));
     }
     catch (const wsrep::runtime_error& e)
     {

Diff for: src/provider_options.cpp

@@ -92,23 +92,14 @@ void wsrep::provider_options::option::update_value(
 
 wsrep::provider_options::option::~option() {}
 
-wsrep::provider_options::provider_options(wsrep::provider& provider)
-    : provider_(provider)
-    , options_()
+wsrep::provider_options::provider_options()
+    : options_()
 {
 }
 
 enum wsrep::provider::status wsrep::provider_options::initial_options()
 {
-    options_.clear();
-    if (config_service_v1_fetch(provider_, this))
-    {
-        return wsrep::provider::error_not_implemented;
-    }
-    else
-    {
-        return wsrep::provider::success;
-    }
+  return wsrep::provider::error_not_implemented;
 }
 
 const wsrep::provider_options::option*
@@ -123,7 +114,7 @@ wsrep::provider_options::get_option(const std::string& name) const
 }
 
 enum wsrep::provider::status wsrep::provider_options::set(
-    const std::string& name,
+    wsrep::provider& provider, const std::string& name,
     std::unique_ptr<wsrep::provider_options::option_value> value)
 {
     auto option(options_.find(name));
@@ -132,7 +123,7 @@ enum wsrep::provider::status wsrep::provider_options::set(
         return not_found_error;
     }
     provider_options_sep sep;
-    auto ret(provider_.options(std::string(option->second->real_name())
+    auto ret(provider.options(std::string(option->second->real_name())
                                + sep.key_value + value->as_string()
                                + sep.param));
     if (ret == provider::success)

Diff for: src/server_state.cpp

@@ -505,8 +505,8 @@ int wsrep::server_state::load_provider(
 {
     wsrep::log_info() << "Loading provider " << provider_spec
                       << " initial position: " << initial_position_;
-    provider_
-        = provider_factory_(*this, provider_spec, provider_options_cb, services);
+    provider_ = provider_factory_(*this, provider_spec, provider_options_cb,
+                                  provider_options_, services);
     return (provider_ ? 0 : 1);
 }
 

Diff for: src/wsrep_provider_v26.cpp

@@ -35,6 +35,7 @@
 #include "tls_service_v1.hpp"
 #include "allowlist_service_v1.hpp"
 #include "event_service_v1.hpp"
+#include "config_service_v1.hpp"
 #include "v26/wsrep_api.h"
 #include "v26/wsrep_node_isolation.h"
 #include "connection_monitor_service_v1.hpp"
@@ -773,6 +774,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
     wsrep::server_state& server_state,
     const std::string& provider_spec,
     const std::function<std::string()>& provider_options_cb,
+    provider_options& options,
     const wsrep::provider::services& services)
     : provider(server_state)
     , wsrep_()
@@ -785,6 +787,15 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
                 server_state.initial_position().id().data(),
                 sizeof(state_id.uuid.data));
     state_id.seqno = server_state.initial_position().seqno().get();
+
+    if (wsrep_load(provider_spec.c_str(), &wsrep_, logger_cb))
+    {
+        throw wsrep::runtime_error("Failed to load wsrep library");
+    }
+
+    init_services(services);
+    config_service_v1_fetch(wsrep_, &options);
+
     struct wsrep_init_args init_args;
     memset(&init_args, 0, sizeof(init_args));
     init_args.app_ctx = &server_state;
@@ -807,13 +818,6 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
     init_args.sst_donate_cb = &sst_donate_cb;
     init_args.synced_cb = &synced_cb;
 
-    if (wsrep_load(provider_spec.c_str(), &wsrep_, logger_cb))
-    {
-        throw wsrep::runtime_error("Failed to load wsrep library");
-    }
-
-    init_services(services);
-
     if (wsrep_->init(wsrep_, &init_args) != WSREP_OK)
     {
         throw wsrep::runtime_error("Failed to initialize wsrep provider");

Diff for: src/wsrep_provider_v26.hpp

@@ -26,14 +26,16 @@ struct wsrep_st;
 
 namespace wsrep
 {
-    class thread_service;
+    class provider_options;
+
     class wsrep_provider_v26 : public wsrep::provider
     {
     public:
         void init_services(const wsrep::provider::services& services);
         void deinit_services();
         wsrep_provider_v26(wsrep::server_state&, const std::string&,
                            const std::function<std::string()>&,
+                           provider_options& provider_options,
                            const wsrep::provider::services& services);
         ~wsrep_provider_v26() WSREP_OVERRIDE;
         enum wsrep::provider::status