This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new d16e65cd8 feat(Ranger): Use Apache Ranger for replica access
controller (#1433)
d16e65cd8 is described below
commit d16e65cd88b4fb6113d5e6a99234b5be527f28da
Author: WHBANG <[email protected]>
AuthorDate: Sun Apr 16 22:48:27 2023 +0800
feat(Ranger): Use Apache Ranger for replica access controller (#1433)
https://github.com/apache/incubator-pegasus/issues/1054
This patch implements replica access controller using Ranger for ACL.
1. The Ranger policy info of the table is written to the app_envs of the
table.
2. Support using the policy in the app_envs for ACL when the replica server
reads and writes.
3. Modify some unit tests, and be compatible with old and new ACL.
---
src/replica/replica.cpp | 8 +-
src/replica/replica.h | 7 ++
src/replica/replica_2pc.cpp | 3 +-
src/replica/replica_config.cpp | 10 +++
src/replica/replica_stub.cpp | 2 +
src/replica/replica_stub.h | 6 ++
src/runtime/security/access_controller.cpp | 14 +---
src/runtime/security/access_controller.h | 23 +++---
src/runtime/security/meta_access_controller.cpp | 55 ++++++--------
src/runtime/security/meta_access_controller.h | 4 +-
src/runtime/security/replica_access_controller.cpp | 85 ++++++++++++++++++----
src/runtime/security/replica_access_controller.h | 37 ++++++++--
.../test/replica_access_controller_test.cpp | 8 +-
13 files changed, 176 insertions(+), 86 deletions(-)
diff --git a/src/replica/replica.cpp b/src/replica/replica.cpp
index be2cbd309..73182cb5c 100644
--- a/src/replica/replica.cpp
+++ b/src/replica/replica.cpp
@@ -244,7 +244,7 @@ replica::~replica(void)
void replica::on_client_read(dsn::message_ex *request, bool ignore_throttling)
{
- if (!_access_controller->allowed(request)) {
+ if (!_access_controller->allowed(request, ranger::access_type::kRead)) {
response_client_read(request, ERR_ACL_DENY);
return;
}
@@ -599,5 +599,11 @@ error_code replica::store_app_info(app_info &info, const
std::string &path)
return err;
}
+bool replica::access_controller_allowed(message_ex *msg, ranger::access_type
req_type) const
+{
+ return !_access_controller->is_enable_ranger_acl() ||
+ _access_controller->allowed(msg, req_type);
+}
+
} // namespace replication
} // namespace dsn
diff --git a/src/replica/replica.h b/src/replica/replica.h
index fdcdfbb19..a79550f8d 100644
--- a/src/replica/replica.h
+++ b/src/replica/replica.h
@@ -64,6 +64,7 @@
#include "replica/replica_base.h"
#include "replica_context.h"
#include "runtime/api_layer1.h"
+#include "runtime/ranger/access_type.h"
#include "runtime/rpc/rpc_message.h"
#include "runtime/serverlet.h"
#include "runtime/task/task.h"
@@ -502,6 +503,9 @@ private:
// update allowed users for access controller
void update_ac_allowed_users(const std::map<std::string, std::string>
&envs);
+ // update replica access controller Ranger policies
+ void update_ac_ranger_policies(const std::map<std::string, std::string>
&envs);
+
// update bool app envs
void update_bool_envs(const std::map<std::string, std::string> &envs,
const std::string &name,
@@ -524,6 +528,9 @@ private:
bool is_data_corrupted() const { return _data_corrupted; }
+ // use Apache Ranger for replica access control
+ bool access_controller_allowed(message_ex *msg, ranger::access_type
req_type) const;
+
private:
friend class ::dsn::replication::test::test_checker;
friend class ::dsn::replication::mutation_queue;
diff --git a/src/replica/replica_2pc.cpp b/src/replica/replica_2pc.cpp
index 2e85c4bc4..0fd3b9851 100644
--- a/src/replica/replica_2pc.cpp
+++ b/src/replica/replica_2pc.cpp
@@ -55,6 +55,7 @@
#include "replica/replication_app_base.h"
#include "replica_stub.h"
#include "runtime/api_layer1.h"
+#include "runtime/ranger/access_type.h"
#include "runtime/rpc/network.h"
#include "runtime/rpc/rpc_address.h"
#include "runtime/rpc/rpc_message.h"
@@ -118,7 +119,7 @@ void replica::on_client_write(dsn::message_ex *request,
bool ignore_throttling)
{
_checker.only_one_thread_access();
- if (!_access_controller->allowed(request)) {
+ if (!_access_controller->allowed(request, ranger::access_type::kWrite)) {
response_client_write(request, ERR_ACL_DENY);
return;
}
diff --git a/src/replica/replica_config.cpp b/src/replica/replica_config.cpp
index b99f1c365..db4d7f103 100644
--- a/src/replica/replica_config.cpp
+++ b/src/replica/replica_config.cpp
@@ -533,6 +533,8 @@ void replica::update_app_envs_internal(const
std::map<std::string, std::string>
update_ac_allowed_users(envs);
+ update_ac_ranger_policies(envs);
+
update_allow_ingest_behind(envs);
update_deny_client(envs);
@@ -564,6 +566,14 @@ void replica::update_ac_allowed_users(const
std::map<std::string, std::string> &
_access_controller->update_allowed_users(allowed_users);
}
+void replica::update_ac_ranger_policies(const std::map<std::string,
std::string> &envs)
+{
+ auto iter =
envs.find(replica_envs::REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES);
+ if (iter != envs.end()) {
+ _access_controller->update_ranger_policies(iter->second);
+ }
+}
+
void replica::update_allow_ingest_behind(const std::map<std::string,
std::string> &envs)
{
bool new_value = false;
diff --git a/src/replica/replica_stub.cpp b/src/replica/replica_stub.cpp
index deb912342..3b3c35e4c 100644
--- a/src/replica/replica_stub.cpp
+++ b/src/replica/replica_stub.cpp
@@ -74,6 +74,7 @@
#include "runtime/api_layer1.h"
#include "runtime/rpc/rpc_message.h"
#include "runtime/rpc/serialization.h"
+#include "runtime/security/access_controller.h"
#include "runtime/task/async_calls.h"
#include "split/replica_split_manager.h"
#include "utils/command_manager.h"
@@ -564,6 +565,7 @@ void replica_stub::initialize(bool clear /* = false*/)
replication_options opts;
opts.initialize();
initialize(opts, clear);
+ _access_controller = std::make_unique<dsn::security::access_controller>();
}
void replica_stub::initialize(const replication_options &opts, bool clear /* =
false*/)
diff --git a/src/replica/replica_stub.h b/src/replica/replica_stub.h
index 81be17212..edcbbca71 100644
--- a/src/replica/replica_stub.h
+++ b/src/replica/replica_stub.h
@@ -76,6 +76,10 @@ class command_deregister;
class message_ex;
class nfs_node;
+namespace security {
+class access_controller;
+} // namespace security
+
namespace replication {
class configuration_query_by_node_response;
class configuration_update_request;
@@ -456,6 +460,8 @@ private:
bool _is_running;
+ std::unique_ptr<dsn::security::access_controller> _access_controller;
+
#ifdef DSN_ENABLE_GPERF
std::atomic_bool _is_releasing_memory{false};
#endif
diff --git a/src/runtime/security/access_controller.cpp
b/src/runtime/security/access_controller.cpp
index 56e9acd7e..2a2f230f1 100644
--- a/src/runtime/security/access_controller.cpp
+++ b/src/runtime/security/access_controller.cpp
@@ -46,15 +46,7 @@ access_controller::access_controller()
access_controller::~access_controller() {}
-bool access_controller::pre_check(const std::string &user_name)
-{
- if (!FLAGS_enable_acl || _super_users.find(user_name) !=
_super_users.end()) {
- return true;
- }
- return false;
-}
-
-bool access_controller::is_enable_ranger_acl() { return
FLAGS_enable_ranger_acl; }
+bool access_controller::is_enable_ranger_acl() const { return
FLAGS_enable_ranger_acl; }
bool access_controller::is_super_user(const std::string &user_name) const
{
@@ -67,9 +59,9 @@ std::shared_ptr<access_controller>
create_meta_access_controller(
return std::make_shared<meta_access_controller>(policy_manager);
}
-std::unique_ptr<access_controller> create_replica_access_controller(const
std::string &name)
+std::unique_ptr<access_controller> create_replica_access_controller(const
std::string &replica_name)
{
- return std::make_unique<replica_access_controller>(name);
+ return std::make_unique<replica_access_controller>(replica_name);
}
} // namespace security
} // namespace dsn
diff --git a/src/runtime/security/access_controller.h
b/src/runtime/security/access_controller.h
index 5ae622494..863779deb 100644
--- a/src/runtime/security/access_controller.h
+++ b/src/runtime/security/access_controller.h
@@ -36,38 +36,35 @@ class access_controller
{
public:
access_controller();
- virtual ~access_controller() = 0;
+ virtual ~access_controller();
// Update the access controller.
// users - the new allowed users to update
virtual void update_allowed_users(const std::string &users) {}
// Check whether the Ranger ACL is enabled or not.
- bool is_enable_ranger_acl();
+ bool is_enable_ranger_acl() const;
+
+ // Update the access controller policy
+ // policies - the new Ranger policies to update
+ virtual void update_ranger_policies(const std::string &policies) {}
// Check if the message received is allowd to access the system.
// msg - the message received
- virtual bool allowed(message_ex *msg, dsn::ranger::access_type req_type) {
return false; }
+ virtual bool allowed(message_ex *msg, dsn::ranger::access_type req_type)
const { return false; }
// Check if the message received is allowd to access the table.
// msg - the message received
// app_name - tables involved in ACL
- virtual bool allowed(message_ex *msg, const std::string &app_name) {
return false; }
-
- // TODO(wanghao): this method will be deleted in the next patch.
- // check if the message received is allowd to do something.
- // msg - the message received
- virtual bool allowed(message_ex *msg) = 0;
+ virtual bool allowed(message_ex *msg, const std::string &app_name = "")
const { return false; }
protected:
- // TODO(wanghao): this method will be deleted in the next patch.
- bool pre_check(const std::string &user_name);
-
// Check if 'user_name' is the super user.
bool is_super_user(const std::string &user_name) const;
- friend class meta_access_controller_test;
std::unordered_set<std::string> _super_users;
+
+ friend class meta_access_controller_test;
};
std::shared_ptr<access_controller> create_meta_access_controller(
diff --git a/src/runtime/security/meta_access_controller.cpp
b/src/runtime/security/meta_access_controller.cpp
index 8adf81f75..74b75f097 100644
--- a/src/runtime/security/meta_access_controller.cpp
+++ b/src/runtime/security/meta_access_controller.cpp
@@ -45,9 +45,9 @@ meta_access_controller::meta_access_controller(
// MetaServer serves the allow-list RPC from all users. RPCs unincluded
are accessible to only
// superusers.
if (utils::is_empty(FLAGS_meta_acl_rpc_allow_list)) {
- register_allowed_rpc_code_list({"RPC_CM_LIST_APPS",
+ register_allowed_rpc_code_list({"RPC_CM_CLUSTER_INFO",
+ "RPC_CM_LIST_APPS",
"RPC_CM_LIST_NODES",
- "RPC_CM_CLUSTER_INFO",
"RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX"});
} else {
std::vector<std::string> rpc_code_white_list;
@@ -57,50 +57,37 @@ meta_access_controller::meta_access_controller(
// use Ranger policy
if (FLAGS_enable_ranger_acl) {
-
register_allowed_rpc_code_list({"RPC_CM_UPDATE_PARTITION_CONFIGURATION",
+ register_allowed_rpc_code_list({"RPC_BULK_LOAD",
+ "RPC_CALL_RAW_MESSAGE",
+ "RPC_CALL_RAW_SESSION_DISCONNECT",
+ "RPC_CLEAR_COLD_BACKUP",
"RPC_CM_CONFIG_SYNC",
"RPC_CM_DUPLICATION_SYNC",
-
"RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX",
- "RPC_CM_REPORT_RESTORE_STATUS",
"RPC_CM_NOTIFY_STOP_SPLIT",
"RPC_CM_QUERY_CHILD_STATE",
- "RPC_NEGOTIATION",
- "RPC_CALL_RAW_MESSAGE",
- "RPC_CALL_RAW_SESSION_DISCONNECT",
- "RPC_NFS_GET_FILE_SIZE",
- "RPC_FD_FAILURE_DETECTOR_PING",
- "RPC_CALL_RAW_MESSAGE",
- "RPC_CALL_RAW_SESSION_DISCONNECT",
+
"RPC_CM_QUERY_PARTITION_CONFIG_BY_INDEX",
+ "RPC_CM_REPORT_RESTORE_STATUS",
+
"RPC_CM_UPDATE_PARTITION_CONFIGURATION",
"RPC_CONFIG_PROPOSAL",
+ "RPC_COLD_BACKUP",
+ "RPC_FD_FAILURE_DETECTOR_PING",
+ "RPC_GROUP_BULK_LOAD",
"RPC_GROUP_CHECK",
- "RPC_QUERY_REPLICA_INFO",
- "RPC_QUERY_LAST_CHECKPOINT_INFO",
+ "RPC_LEARN_ADD_LEARNER",
+ "RPC_LEARN_COMPLETION_NOTIFY",
+ "RPC_NEGOTIATION",
"RPC_PREPARE",
- "RPC_GROUP_CHECK",
"RPC_QUERY_APP_INFO",
- "RPC_LEARN_COMPLETION_NOTIFY",
- "RPC_LEARN_ADD_LEARNER",
+ "RPC_QUERY_LAST_CHECKPOINT_INFO",
+ "RPC_QUERY_REPLICA_INFO",
"RPC_REMOVE_REPLICA",
- "RPC_COLD_BACKUP",
- "RPC_CLEAR_COLD_BACKUP",
"RPC_SPLIT_NOTIFY_CATCH_UP",
-
"RPC_SPLIT_UPDATE_CHILD_PARTITION_COUNT",
- "RPC_BULK_LOAD",
- "RPC_GROUP_BULK_LOAD"});
+
"RPC_SPLIT_UPDATE_CHILD_PARTITION_COUNT"});
_ranger_resource_policy_manager->start();
}
}
-bool meta_access_controller::allowed(message_ex *msg)
-{
- if (pre_check(msg->io_session->get_client_username()) ||
- _allowed_rpc_code_list.find(msg->rpc_code().code()) !=
_allowed_rpc_code_list.end()) {
- return true;
- }
- return false;
-}
-
-bool meta_access_controller::allowed(message_ex *msg, const std::string
&app_name)
+bool meta_access_controller::allowed(message_ex *msg, const std::string
&app_name) const
{
const auto rpc_code = msg->rpc_code().code();
const auto &user_name = msg->io_session->get_client_username();
@@ -121,7 +108,7 @@ bool meta_access_controller::allowed(message_ex *msg, const
std::string &app_nam
if (_allowed_rpc_code_list.find(rpc_code) != _allowed_rpc_code_list.end())
{
return true;
}
- std::string database_name =
ranger::get_database_name_from_app_name(app_name);
+ auto database_name = ranger::get_database_name_from_app_name(app_name);
LOG_DEBUG("Ranger access controller with user_name = {}, rpc = {},
database_name = {}",
user_name,
msg->rpc_code(),
@@ -135,7 +122,7 @@ void meta_access_controller::register_allowed_rpc_code_list(
_allowed_rpc_code_list.clear();
for (const auto &rpc_code : rpc_list) {
auto code = task_code::try_get(rpc_code, TASK_CODE_INVALID);
- CHECK_NE_MSG(code, TASK_CODE_INVALID, "invalid task code.");
+ CHECK_NE_MSG(code, TASK_CODE_INVALID, "invalid task code(code = {}).",
rpc_code);
_allowed_rpc_code_list.insert(code);
}
}
diff --git a/src/runtime/security/meta_access_controller.h
b/src/runtime/security/meta_access_controller.h
index 0def5bc49..7cfe9f71f 100644
--- a/src/runtime/security/meta_access_controller.h
+++ b/src/runtime/security/meta_access_controller.h
@@ -39,9 +39,7 @@ public:
meta_access_controller(
const std::shared_ptr<ranger::ranger_resource_policy_manager>
&policy_manager);
- bool allowed(message_ex *msg) override;
-
- bool allowed(message_ex *msg, const std::string &app_name = "") override;
+ bool allowed(message_ex *msg, const std::string &app_name = "") const
override;
private:
void register_allowed_rpc_code_list(const std::vector<std::string>
&rpc_list);
diff --git a/src/runtime/security/replica_access_controller.cpp
b/src/runtime/security/replica_access_controller.cpp
index 87da8ae03..5832b05d3 100644
--- a/src/runtime/security/replica_access_controller.cpp
+++ b/src/runtime/security/replica_access_controller.cpp
@@ -15,36 +15,61 @@
// specific language governing permissions and limitations
// under the License.
-#include "replica_access_controller.h"
+#include <utility>
+
+// Disable class-memaccess warning to facilitate compilation with gcc>7
+// https://github.com/Tencent/rapidjson/issues/1700
+#pragma GCC diagnostic push
+#if defined(__GNUC__) && __GNUC__ >= 8
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
+#include "common/json_helper.h"
+
+#pragma GCC diagnostic pop
+#include "replica_access_controller.h"
#include "runtime/rpc/network.h"
#include "runtime/rpc/rpc_message.h"
#include "utils/autoref_ptr.h"
+#include "utils/blob.h"
+#include "utils/flags.h"
#include "utils/fmt_logging.h"
#include "utils/strings.h"
namespace dsn {
namespace security {
-replica_access_controller::replica_access_controller(const std::string &name)
{ _name = name; }
+DSN_DECLARE_bool(enable_acl);
+DSN_DECLARE_bool(enable_ranger_acl);
+replica_access_controller::replica_access_controller(const std::string
&replica_name)
+{
+ _name = replica_name;
+}
-bool replica_access_controller::allowed(message_ex *msg)
+bool replica_access_controller::allowed(message_ex *msg, ranger::access_type
req_type) const
{
const std::string &user_name = msg->io_session->get_client_username();
- if (pre_check(user_name)) {
- return true;
+ if (!FLAGS_enable_ranger_acl) {
+ if (!FLAGS_enable_acl || is_super_user(user_name)) {
+ return true;
+ }
+ {
+ utils::auto_read_lock l(_lock);
+ // If the user didn't specify any ACL, it means this table is
publicly accessible to
+ // everyone. This is a backdoor to allow old-version clients to
gracefully upgrade.
+ // After they are finally ensured to be fully upgraded, they can
specify some usernames
+ // to ACL and the table will be truly protected.
+ if (!_allowed_users.empty() && _allowed_users.count(user_name) ==
0) {
+ LOG_INFO("{}: user_name({}) doesn't exist in acls map", _name,
user_name);
+ return false;
+ }
+ return true;
+ }
}
+ // use Ranger policy for ACL.
{
utils::auto_read_lock l(_lock);
- // If the user didn't specify any ACL, it means this table is publicly
accessible to
- // everyone. This is a backdoor to allow old-version clients to
gracefully upgrade. After
- // they are finally ensured to be fully upgraded, they can specify
some usernames to ACL and
- // the table will be truly protected.
- if (!_users.empty() && _users.find(user_name) == _users.end()) {
- LOG_INFO("{}: user_name {} doesn't exist in acls map", _name,
user_name);
- return false;
- }
- return true;
+ return _ranger_policies.allowed(req_type, user_name);
}
}
@@ -54,6 +79,7 @@ void replica_access_controller::update_allowed_users(const
std::string &users)
// check to see whether we should update it or not.
utils::auto_read_lock l(_lock);
if (_env_users == users) {
+ check_allowed_users_valid();
return;
}
}
@@ -62,10 +88,37 @@ void replica_access_controller::update_allowed_users(const
std::string &users)
utils::split_args(users.c_str(), users_set, ',');
{
utils::auto_write_lock l(_lock);
- // This swap operation is in constant time
- _users.swap(users_set);
+ _allowed_users.swap(users_set);
_env_users = users;
+ check_allowed_users_valid();
+ }
+}
+
+void replica_access_controller::update_ranger_policies(const std::string
&policies)
+{
+ {
+ utils::auto_read_lock l(_lock);
+ if (_env_policies == policies) {
+ return;
+ }
+ }
+ ranger::acl_policies tmp_policies;
+ auto tmp_policies_str = policies;
+ dsn::json::json_forwarder<ranger::acl_policies>::decode(
+ dsn::blob::create_from_bytes(std::move(tmp_policies_str)),
tmp_policies);
+ {
+ utils::auto_write_lock l(_lock);
+ _env_policies = policies;
+ _ranger_policies = std::move(tmp_policies);
}
}
+
+void replica_access_controller::check_allowed_users_valid() const
+{
+ LOG_WARNING_IF(
+ FLAGS_enable_acl && !FLAGS_enable_ranger_acl && _allowed_users.empty(),
+ "Without specifying any ACL, the table is not really protected, please
check in time.");
+}
+
} // namespace security
} // namespace dsn
diff --git a/src/runtime/security/replica_access_controller.h
b/src/runtime/security/replica_access_controller.h
index c89a1872e..f3790d5d4 100644
--- a/src/runtime/security/replica_access_controller.h
+++ b/src/runtime/security/replica_access_controller.h
@@ -21,6 +21,8 @@
#include <unordered_set>
#include "access_controller.h"
+#include "runtime/ranger/access_type.h"
+#include "runtime/ranger/ranger_resource_policy.h"
#include "utils/synchronize.h"
namespace dsn {
@@ -30,15 +32,40 @@ namespace security {
class replica_access_controller : public access_controller
{
public:
- explicit replica_access_controller(const std::string &name);
- bool allowed(message_ex *msg) override;
+ explicit replica_access_controller(const std::string &replica_name);
+
+ // Check whether replica can be accessed, this method is compatible with
ACL using
+ // '_allowed_users' and ACL using Ranger policy.
+ bool allowed(message_ex *msg, ranger::access_type req_type) const override;
+
+ // Update '_allowed_users' when the
app_env(REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS) of the
+ // table changes
void update_allowed_users(const std::string &users) override;
+ // Update '_ranger_policies' when the
app_env(REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES) of the
+ // table changes
+ void update_ranger_policies(const std::string &policies) override;
+
+private:
+ // Security check to avoid allowed_users is not empty in special scenarios.
+ void check_allowed_users_valid() const;
+
private:
- utils::rw_lock_nr _lock; // [
- std::unordered_set<std::string> _users;
+ mutable utils::rw_lock_nr _lock;
+ // Users will pass the access control in the old ACL.
+ std::unordered_set<std::string> _allowed_users;
+
+ // App_env(REPLICA_ACCESS_CONTROLLER_ALLOWED_USERS) to facilitate whether
to update
+ // '_allowed_users'.
std::string _env_users;
- // ]
+
+ // App_env(REPLICA_ACCESS_CONTROLLER_RANGER_POLICIES) to facilitate
whether to update
+ // '_ranger_policies'.
+ std::string _env_policies;
+
+ // The Ranger policies for ACL.
+ ranger::acl_policies _ranger_policies;
+
std::string _name;
friend class replica_access_controller_test;
diff --git a/src/runtime/test/replica_access_controller_test.cpp
b/src/runtime/test/replica_access_controller_test.cpp
index aefb6cee2..539bc3cf7 100644
--- a/src/runtime/test/replica_access_controller_test.cpp
+++ b/src/runtime/test/replica_access_controller_test.cpp
@@ -24,6 +24,7 @@
#include <utility>
#include "common/replication.codes.h"
+#include "runtime/ranger/access_type.h"
#include "runtime/rpc/network.h"
#include "runtime/rpc/network.sim.h"
#include "runtime/rpc/rpc_address.h"
@@ -44,11 +45,14 @@ public:
_replica_access_controller =
std::make_unique<replica_access_controller>("test");
}
- bool allowed(dsn::message_ex *msg) { return
_replica_access_controller->allowed(msg); }
+ bool allowed(dsn::message_ex *msg)
+ {
+ return _replica_access_controller->allowed(msg,
dsn::ranger::access_type::kRead);
+ }
void set_replica_users(std::unordered_set<std::string> &&replica_users)
{
- _replica_access_controller->_users.swap(replica_users);
+ _replica_access_controller->_allowed_users.swap(replica_users);
}
std::unique_ptr<replica_access_controller> _replica_access_controller;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
