This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a36828f3e3d [feature](meta-service) Support dynamic configuration for
meta service (#45394)
a36828f3e3d is described below
commit a36828f3e3d08543041b590e4a3bb36d79e956c5
Author: Siyang Tang <[email protected]>
AuthorDate: Mon Jan 6 17:24:39 2025 +0800
[feature](meta-service) Support dynamic configuration for meta service
(#45394)
Provide an http interface to change metaservice's mutable configuration.
```
curl
http://ms_ip:ms_port/MetaService/http/v1/update_config?config=${configs}&persist=${boolean}
curl
http://ms_ip:ms_port/MetaService/http/v1/update_config?configs=recycle_interval_seconds=3601,retention_seconds=259201
```
---
cloud/src/common/config.h | 4 +
cloud/src/common/configbase.cpp | 129 +++++++++++++++++++++++++++
cloud/src/common/configbase.h | 10 ++-
cloud/src/main.cpp | 7 ++
cloud/src/meta-service/meta_service_http.cpp | 39 ++++++++
cloud/test/meta_service_http_test.cpp | 122 +++++++++++++++++++++++++
6 files changed, 310 insertions(+), 1 deletion(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index a3274479189..78d3358b581 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -54,6 +54,10 @@ CONF_Int32(log_verbose_level, "5");
// Only works when starting Cloud with --console.
CONF_Bool(enable_file_logger, "true");
+// Custom conf path is default the same as conf path, and configs will be
append to it.
+// Otherwise, will a new custom conf file will be created.
+CONF_String(custom_conf_path, "./conf/doris_cloud.conf");
+
// recycler config
CONF_mInt64(recycle_interval_seconds, "3600");
CONF_mInt64(retention_seconds, "259200"); // 72h, global retention time
diff --git a/cloud/src/common/configbase.cpp b/cloud/src/common/configbase.cpp
index a1ab4853822..cacbf3479d8 100644
--- a/cloud/src/common/configbase.cpp
+++ b/cloud/src/common/configbase.cpp
@@ -15,14 +15,19 @@
// specific language governing permissions and limitations
// under the License.
+#include <fmt/core.h>
+
#include <algorithm>
#include <cerrno>
#include <cstring>
+#include <filesystem>
#include <fstream>
#include <iostream>
#include <list>
#include <map>
+#include <mutex>
#include <sstream>
+#include <utility>
#define __IN_CONFIGBASE_CPP__
#include "common/config.h"
@@ -33,6 +38,8 @@ namespace doris::cloud::config {
std::map<std::string, Register::Field>* Register::_s_field_map = nullptr;
std::map<std::string, std::function<bool()>>*
RegisterConfValidator::_s_field_validator = nullptr;
std::map<std::string, std::string>* full_conf_map = nullptr;
+std::mutex mutable_string_config_lock;
+std::mutex conf_persist_lock;
// trim string
std::string& trim(std::string& s) {
@@ -224,6 +231,37 @@ bool Properties::load(const char* conf_file, bool
must_exist) {
return true;
}
+bool Properties::dump(const std::string& conffile) {
+ std::string conffile_tmp = conffile + ".tmp";
+
+ if (std::filesystem::exists(conffile)) {
+ // copy for modify
+ std::ifstream in(conffile, std::ios::binary);
+ std::ofstream out(conffile_tmp, std::ios::binary);
+ out << in.rdbuf();
+ in.close();
+ out.close();
+ }
+
+ std::ofstream file_writer;
+
+ file_writer.open(conffile_tmp, std::ios::out | std::ios::app);
+
+ file_writer << std::endl;
+
+ for (auto const& iter : file_conf_map) {
+ file_writer << iter.first << " = " << iter.second << std::endl;
+ }
+
+ file_writer.close();
+ if (!file_writer.good()) {
+ return false;
+ }
+
+ std::filesystem::rename(conffile_tmp, conffile);
+ return std::filesystem::exists(conffile);
+}
+
template <typename T>
bool Properties::get_or_default(const char* key, const char* defstr, T& retval,
bool* is_retval_set) const {
@@ -297,6 +335,37 @@ std::ostream& operator<<(std::ostream& out, const
std::vector<T>& v) {
continue;
\
}
+#define UPDATE_FIELD(FIELD, VALUE, TYPE, PERSIST)
\
+ if (strcmp((FIELD).type, #TYPE) == 0) {
\
+ TYPE new_value;
\
+ if (!convert((VALUE), new_value)) {
\
+ std::cerr << "convert " << VALUE << "as" << #TYPE << "failed";
\
+ return false;
\
+ }
\
+ TYPE& ref_conf_value = *reinterpret_cast<TYPE*>((FIELD).storage);
\
+ TYPE old_value = ref_conf_value;
\
+ if (RegisterConfValidator::_s_field_validator != nullptr) {
\
+ auto validator =
RegisterConfValidator::_s_field_validator->find((FIELD).name); \
+ if (validator != RegisterConfValidator::_s_field_validator->end()
&& \
+ !(validator->second)()) {
\
+ ref_conf_value = old_value;
\
+ std::cerr << "validate " << (FIELD).name << "=" << new_value
<< " failed" \
+ << std::endl;
\
+ return false;
\
+ }
\
+ }
\
+ ref_conf_value = new_value;
\
+ if (full_conf_map != nullptr) {
\
+ std::ostringstream oss;
\
+ oss << new_value;
\
+ (*full_conf_map)[(FIELD).name] = oss.str();
\
+ }
\
+ if (PERSIST) {
\
+ props.set_force(std::string((FIELD).name), VALUE);
\
+ }
\
+ return true;
\
+ }
+
// init conf fields
bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool
set_to_default) {
Properties props;
@@ -328,4 +397,64 @@ bool init(const char* conf_file, bool fill_conf_map, bool
must_exist, bool set_t
return true;
}
+bool do_set_config(const Register::Field& feild, const std::string& value,
bool need_persist,
+ Properties& props) {
+ UPDATE_FIELD(feild, value, bool, need_persist);
+ UPDATE_FIELD(feild, value, int16_t, need_persist);
+ UPDATE_FIELD(feild, value, int32_t, need_persist);
+ UPDATE_FIELD(feild, value, int64_t, need_persist);
+ UPDATE_FIELD(feild, value, double, need_persist);
+ {
+ // add lock to ensure thread safe
+ std::lock_guard<std::mutex> lock(mutable_string_config_lock);
+ UPDATE_FIELD(feild, value, std::string, need_persist);
+ }
+ return false;
+}
+
+std::pair<bool, std::string> set_config(const std::string& field, const
std::string& value,
+ bool need_persist, Properties& props) {
+ auto it = Register::_s_field_map->find(field);
+ if (it == Register::_s_field_map->end()) {
+ return {false, fmt::format("config field={} not exists", field)};
+ }
+ if (!it->second.valmutable) {
+ return {false, fmt::format("config field={} is immutable", field)};
+ }
+
+ if (!do_set_config(it->second, value, need_persist, props)) {
+ return {false, fmt::format("not supported to modify field={},
value={}", field, value)};
+ }
+ return {true, {}};
+}
+
+std::pair<bool, std::string> set_config(std::unordered_map<std::string,
std::string> field_map,
+ bool need_persist, const std::string&
custom_conf_path) {
+ Properties props;
+ auto set_conf_closure = [&]() -> std::pair<bool, std::string> {
+ for (const auto& [field, value] : field_map) {
+ if (auto [succ, cause] = set_config(field, value, need_persist,
props); !succ) {
+ return {false, std::move(cause)};
+ }
+ }
+ return {true, {}};
+ };
+
+ if (!need_persist) {
+ return set_conf_closure();
+ }
+
+ // lock to make sure only one thread can modify the conf file
+ std::lock_guard<std::mutex> l(conf_persist_lock);
+ auto [succ, cause] = set_conf_closure();
+ if (!succ) {
+ return {succ, std::move(cause)};
+ }
+ if (props.dump(custom_conf_path)) {
+ return {true, {}};
+ }
+ return {false, fmt::format("dump config modification to
custom_conf_path={} "
+ "failed, plz check config::custom_conf_path and
io status",
+ custom_conf_path)};
+}
} // namespace doris::cloud::config
diff --git a/cloud/src/common/configbase.h b/cloud/src/common/configbase.h
index a3b3b70829a..6eee2639468 100644
--- a/cloud/src/common/configbase.h
+++ b/cloud/src/common/configbase.h
@@ -22,6 +22,7 @@
#include <map>
#include <mutex>
#include <string>
+#include <utility>
#include <vector>
namespace doris::cloud::config {
@@ -152,7 +153,11 @@ public:
void set_force(const std::string& key, const std::string& val);
- // dump props to conf file
+ // Dump props to conf file if conffile exists, will append to it
+ // else will dump to a new conffile.
+ //
+ // This Function will generate a tmp file for modification and rename it
+ // to subtitute the original one if modication success.
bool dump(const std::string& conffile);
private:
@@ -170,4 +175,7 @@ extern std::map<std::string, std::string>* full_conf_map;
bool init(const char* conf_file, bool fill_conf_map = false, bool must_exist =
true,
bool set_to_default = true);
+std::pair<bool, std::string> set_config(std::unordered_map<std::string,
std::string> field_map,
+ bool need_persist, const std::string&
custom_conf_path);
+
} // namespace doris::cloud::config
diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp
index 26033cdaad2..2ea251a2d2a 100644
--- a/cloud/src/main.cpp
+++ b/cloud/src/main.cpp
@@ -33,6 +33,7 @@
#include "common/arg_parser.h"
#include "common/config.h"
+#include "common/configbase.h"
#include "common/encryption_util.h"
#include "common/logging.h"
#include "meta-service/mem_txn_kv.h"
@@ -201,6 +202,12 @@ int main(int argc, char** argv) {
std::cerr << "failed to init config file, conf=" << conf_file <<
std::endl;
return -1;
}
+ if (!std::filesystem::equivalent(conf_file, config::custom_conf_path) &&
+ !config::init(config::custom_conf_path.c_str(), false)) {
+ std::cerr << "failed to init custom config file, conf=" <<
config::custom_conf_path
+ << std::endl;
+ return -1;
+ }
if (auto ret = prepare_extra_conf_file(); !ret.empty()) {
std::cerr << "failed to prepare extra conf file, err=" << ret <<
std::endl;
diff --git a/cloud/src/meta-service/meta_service_http.cpp
b/cloud/src/meta-service/meta_service_http.cpp
index e919c6f365e..78f2dc03a95 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -20,6 +20,7 @@
#include <brpc/controller.h>
#include <brpc/http_status_code.h>
#include <brpc/uri.h>
+#include <fmt/core.h>
#include <fmt/format.h>
#include <gen_cpp/cloud.pb.h>
#include <glog/logging.h>
@@ -44,7 +45,9 @@
#include <vector>
#include "common/config.h"
+#include "common/configbase.h"
#include "common/logging.h"
+#include "common/string_util.h"
#include "meta-service/keys.h"
#include "meta-service/txn_kv.h"
#include "meta-service/txn_kv_error.h"
@@ -448,6 +451,40 @@ static HttpResponse
process_query_rate_limit(MetaServiceImpl* service, brpc::Con
return http_json_reply(MetaServiceCode::OK, "", sb.GetString());
}
+static HttpResponse process_update_config(MetaServiceImpl* service,
brpc::Controller* cntl) {
+ const auto& uri = cntl->http_request().uri();
+ bool persist = (http_query(uri, "persist") == "true");
+ auto configs = std::string {http_query(uri, "configs")};
+ auto reason = std::string {http_query(uri, "reason")};
+ LOG(INFO) << "modify configs for reason=" << reason << ", configs=" <<
configs
+ << ", persist=" << http_query(uri, "persist");
+ if (configs.empty()) [[unlikely]] {
+ LOG(WARNING) << "query param `config` should not be empty";
+ return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
+ "query param `config` should not be empty");
+ }
+ std::unordered_map<std::string, std::string> conf_map;
+ auto conf_list = split(configs, ',');
+ for (const auto& conf : conf_list) {
+ auto conf_pair = split(conf, '=');
+ if (conf_pair.size() != 2) [[unlikely]] {
+ LOG(WARNING) << "failed to split config=[{}] from `k=v` pattern"
<< conf;
+ return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
+ fmt::format("config {} is invalid",
configs));
+ }
+ trim(conf_pair[0]);
+ trim(conf_pair[1]);
+ conf_map.emplace(std::move(conf_pair[0]), std::move(conf_pair[1]));
+ }
+ if (auto [succ, cause] =
+ config::set_config(std::move(conf_map), persist,
config::custom_conf_path);
+ !succ) {
+ LOG(WARNING) << cause;
+ return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, cause);
+ }
+ return http_json_reply(MetaServiceCode::OK, "");
+}
+
static HttpResponse process_decode_key(MetaServiceImpl*, brpc::Controller*
ctrl) {
auto& uri = ctrl->http_request().uri();
std::string_view key = http_query(uri, "key");
@@ -732,12 +769,14 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
{"alter_iam", process_alter_iam},
{"adjust_rate_limit", process_adjust_rate_limit},
{"list_rate_limit", process_query_rate_limit},
+ {"update_config", process_update_config},
{"v1/abort_txn", process_abort_txn},
{"v1/abort_tablet_job", process_abort_tablet_job},
{"v1/alter_ram_user", process_alter_ram_user},
{"v1/alter_iam", process_alter_iam},
{"v1/adjust_rate_limit", process_adjust_rate_limit},
{"v1/list_rate_limit", process_query_rate_limit},
+ {"v1/update_config", process_update_config},
};
auto* cntl = static_cast<brpc::Controller*>(controller);
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 9061a5c2c5d..3afb2f66bf6 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -32,9 +32,13 @@
#include <rapidjson/stringbuffer.h>
#include <cstddef>
+#include <cstdint>
+#include <filesystem>
#include <optional>
+#include <string>
#include "common/config.h"
+#include "common/configbase.h"
#include "common/logging.h"
#include "common/util.h"
#include "cpp/sync_point.h"
@@ -1639,4 +1643,122 @@ TEST(MetaServiceHttpTest, QueryRateLimit) {
}
}
+TEST(MetaServiceHttpTest, UpdateConfig) {
+ HttpContext ctx;
+ {
+ auto [status_code, content] = ctx.query<std::string>("update_config",
"");
+ ASSERT_EQ(status_code, 400);
+ std::string msg = "query param `config` should not be empty";
+ ASSERT_NE(content.find(msg), std::string::npos);
+ }
+ {
+ auto [status_code, content] = ctx.query<std::string>("update_config",
"configs=aaa");
+ ASSERT_EQ(status_code, 400);
+ std::string msg = "config aaa is invalid";
+ ASSERT_NE(content.find(msg), std::string::npos);
+ }
+ {
+ auto [status_code, content] = ctx.query<std::string>("update_config",
"configs=aaa=bbb");
+ ASSERT_EQ(status_code, 400);
+ std::string msg = "config field=aaa not exists";
+ ASSERT_NE(content.find(msg), std::string::npos);
+ }
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
"configs=custom_conf_path=./doris_conf");
+ ASSERT_EQ(status_code, 400);
+ std::string msg = "config field=custom_conf_path is immutable";
+ ASSERT_NE(content.find(msg), std::string::npos);
+ }
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
"configs=recycle_interval_seconds=3599");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::recycle_interval_seconds, 3599);
+ }
+ {
+ auto [status_code, content] = ctx.query<std::string>(
+ "update_config",
"configs=recycle_interval_seconds=3601,retention_seconds=259201");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::retention_seconds, 259201);
+ ASSERT_EQ(config::recycle_interval_seconds, 3601);
+ }
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
"configs=enable_s3_rate_limiter=true");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_TRUE(config::enable_s3_rate_limiter);
+ }
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
"enable_s3_rate_limiter=invalid");
+ ASSERT_EQ(status_code, 400);
+ }
+ {
+ auto original_conf_path = config::custom_conf_path;
+ config::custom_conf_path = "./doris_cloud_custom.conf";
+ {
+ auto [status_code, content] = ctx.query<std::string>(
+ "update_config",
+
"configs=recycle_interval_seconds=3659,retention_seconds=259219&persist=true");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::recycle_interval_seconds, 3659);
+ ASSERT_EQ(config::retention_seconds, 259219);
+ config::Properties props;
+ ASSERT_TRUE(props.load(config::custom_conf_path.c_str(), true));
+ {
+ bool new_val_set = false;
+ int64_t recycle_interval_s = 0;
+ ASSERT_TRUE(props.get_or_default("recycle_interval_seconds",
nullptr,
+ recycle_interval_s,
&new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(recycle_interval_s, 3659);
+ }
+ {
+ bool new_val_set = false;
+ int64_t retention_s = 0;
+ ASSERT_TRUE(props.get_or_default("retention_seconds", nullptr,
retention_s,
+ &new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(retention_s, 259219);
+ }
+ }
+ {
+ auto [status_code, content] = ctx.query<std::string>(
+ "update_config",
"configs=enable_s3_rate_limiter=false&persist=true");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::recycle_interval_seconds, 3659);
+ ASSERT_EQ(config::retention_seconds, 259219);
+ config::Properties props;
+ ASSERT_TRUE(props.load(config::custom_conf_path.c_str(), true));
+ {
+ bool new_val_set = false;
+ int64_t recycle_interval_s = 0;
+ ASSERT_TRUE(props.get_or_default("recycle_interval_seconds",
nullptr,
+ recycle_interval_s,
&new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(recycle_interval_s, 3659);
+ }
+ {
+ bool new_val_set = false;
+ int64_t retention_s = 0;
+ ASSERT_TRUE(props.get_or_default("retention_seconds", nullptr,
retention_s,
+ &new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(retention_s, 259219);
+ }
+ {
+ bool new_val_set = false;
+ bool enable_s3_rate_limiter = true;
+ ASSERT_TRUE(props.get_or_default("enable_s3_rate_limiter",
nullptr,
+ enable_s3_rate_limiter,
&new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_FALSE(enable_s3_rate_limiter);
+ }
+ }
+ std::filesystem::remove(config::custom_conf_path);
+ config::custom_conf_path = original_conf_path;
+ }
+}
+
} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
