This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6cbf393665 [enhance](meta action) remove useless pb field and refactor
writer cooldown meta code (#17652)
6cbf393665 is described below
commit 6cbf393665e168a6eeadbd6abc81f728a5db1da6
Author: AlexYue <[email protected]>
AuthorDate: Wed Mar 22 11:13:13 2023 +0800
[enhance](meta action) remove useless pb field and refactor writer cooldown
meta code (#17652)
---
be/src/http/action/meta_action.cpp | 48 ++++++++++------
be/src/http/action/meta_action.h | 9 +--
be/src/olap/tablet.cpp | 73 +++++++++++++-----------
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_meta.cpp | 2 -
be/src/service/http_service.cpp | 4 +-
be/test/olap/test_data/header_without_inc_rs.txt | 4 +-
gensrc/proto/olap_file.proto | 2 -
8 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/be/src/http/action/meta_action.cpp
b/be/src/http/action/meta_action.cpp
index 7531dd6a93..bb8300c8d3 100644
--- a/be/src/http/action/meta_action.cpp
+++ b/be/src/http/action/meta_action.cpp
@@ -37,6 +37,9 @@
namespace doris {
const static std::string HEADER_JSON = "application/json";
+const static std::string OP = "op";
+const static std::string DATA_SIZE = "data_size";
+const static std::string HEADER = "header";
Status MetaAction::_handle_header(HttpRequest* req, std::string* json_meta) {
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
@@ -60,26 +63,37 @@ Status MetaAction::_handle_header(HttpRequest* req,
std::string* json_meta) {
LOG(WARNING) << "no tablet for tablet_id:" << tablet_id;
return Status::InternalError("no tablet exist");
}
- TabletMetaSharedPtr tablet_meta(new TabletMeta());
- tablet->generate_tablet_meta_copy(tablet_meta);
- json2pb::Pb2JsonOptions json_options;
- json_options.pretty_json = true;
- json_options.bytes_to_base64 = enable_byte_to_base64;
- tablet_meta->to_json(json_meta, json_options);
- return Status::OK();
+ std::string operation = req->param(OP);
+ if (operation == HEADER) {
+ TabletMetaSharedPtr tablet_meta(new TabletMeta());
+ tablet->generate_tablet_meta_copy(tablet_meta);
+ json2pb::Pb2JsonOptions json_options;
+ json_options.pretty_json = true;
+ json_options.bytes_to_base64 = enable_byte_to_base64;
+ tablet_meta->to_json(json_meta, json_options);
+ return Status::OK();
+ } else if (operation == DATA_SIZE) {
+ EasyJson data_size;
+ {
+ std::shared_lock rowset_ldlock(tablet->get_header_lock());
+ data_size["local_data_size"] = tablet->tablet_local_size();
+ data_size["remote_data_size"] = tablet->tablet_remote_size();
+ }
+ *json_meta = data_size.ToString();
+ return Status::OK();
+ }
+ return Status::InternalError("invalid operation");
}
void MetaAction::handle(HttpRequest* req) {
- if (_meta_type == META_TYPE::HEADER) {
- std::string json_meta;
- Status status = _handle_header(req, &json_meta);
- std::string status_result = status.to_json();
- LOG(INFO) << "handle request result:" << status_result;
- if (status.ok()) {
- HttpChannel::send_reply(req, HttpStatus::OK, json_meta);
- } else {
- HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR,
status_result);
- }
+ std::string json_meta;
+ Status status = _handle_header(req, &json_meta);
+ std::string status_result = status.to_json();
+ LOG(INFO) << "handle request result:" << status_result;
+ if (status.ok()) {
+ HttpChannel::send_reply(req, HttpStatus::OK, json_meta);
+ } else {
+ HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR,
status_result);
}
}
diff --git a/be/src/http/action/meta_action.h b/be/src/http/action/meta_action.h
index 1ed9e11229..2349d7ec22 100644
--- a/be/src/http/action/meta_action.h
+++ b/be/src/http/action/meta_action.h
@@ -24,14 +24,10 @@ namespace doris {
class ExecEnv;
-enum META_TYPE {
- HEADER = 1,
-};
-
// Get Meta Info
class MetaAction : public HttpHandler {
public:
- MetaAction(META_TYPE meta_type) : _meta_type(meta_type) {}
+ MetaAction() = default;
virtual ~MetaAction() {}
@@ -39,9 +35,6 @@ public:
private:
Status _handle_header(HttpRequest* req, std::string* json_header);
-
-private:
- META_TYPE _meta_type;
};
} // end namespace doris
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index d30b5b0ba4..2741b858ad 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -96,24 +96,14 @@ bvar::Window<bvar::Adder<uint64_t>>
exceed_version_limit_counter_minute(
&exceed_version_limit_counter, 60);
struct WriteCooldownMetaExecutors {
- WriteCooldownMetaExecutors(size_t executor_nums = 5) :
_executor_nums(executor_nums) {
- for (size_t i = 0; i < _executor_nums; i++) {
- std::unique_ptr<ThreadPool> pool;
- ThreadPoolBuilder("AsyncWriteCooldownMetaExecutor")
- .set_min_threads(1)
- .set_max_threads(1)
- .set_max_queue_size(std::numeric_limits<int>::max())
- .build(&pool);
- _executors.emplace_back(std::move(pool));
- }
- }
+ WriteCooldownMetaExecutors(size_t executor_nums = 5);
static WriteCooldownMetaExecutors* GetInstance() {
static WriteCooldownMetaExecutors instance;
return &instance;
}
- void submit(int64_t tablet_id, std::function<void()> task);
+ void submit(TabletSharedPtr tablet);
size_t _get_executor_pos(int64_t tablet_id) const { return tablet_id %
_executor_nums; };
std::vector<std::unique_ptr<ThreadPool>> _executors;
std::unordered_set<int64_t> _pengding_tablets;
@@ -121,15 +111,49 @@ struct WriteCooldownMetaExecutors {
size_t _executor_nums;
};
-void WriteCooldownMetaExecutors::submit(int64_t tablet_id,
std::function<void()> task) {
+WriteCooldownMetaExecutors::WriteCooldownMetaExecutors(size_t executor_nums)
+ : _executor_nums(executor_nums) {
+ for (size_t i = 0; i < _executor_nums; i++) {
+ std::unique_ptr<ThreadPool> pool;
+ ThreadPoolBuilder("AsyncWriteCooldownMetaExecutor")
+ .set_min_threads(1)
+ .set_max_threads(1)
+ .set_max_queue_size(std::numeric_limits<int>::max())
+ .build(&pool);
+ _executors.emplace_back(std::move(pool));
+ }
+}
+
+void
WriteCooldownMetaExecutors::WriteCooldownMetaExecutors::submit(TabletSharedPtr
tablet) {
+ auto tablet_id = tablet->tablet_id();
+
{
+ // one tablet could at most have one cooldown task to be done
std::unique_lock<std::mutex> lck {_latch};
if (_pengding_tablets.count(tablet_id) > 0) {
return;
}
_pengding_tablets.insert(tablet_id);
}
- _executors[_get_executor_pos(tablet_id)]->submit_func([task =
std::move(task)]() { task(); });
+
+ auto async_write_task = [this, t = std::move(tablet)]() {
+ {
+ std::unique_lock<std::mutex> lck {_latch};
+ _pengding_tablets.erase(t->tablet_id());
+ }
+ auto s = t->write_cooldown_meta();
+ if (s.ok()) {
+ return;
+ }
+ LOG_WARNING("write tablet {} cooldown meta failed because: {}",
t->tablet_id(),
+ s.to_string());
+ if (!s.is<ABORTED>()) {
+ submit(t);
+ }
+ };
+
+ _executors[_get_executor_pos(tablet_id)]->submit_func(
+ [task = std::move(async_write_task)]() { task(); });
}
TabletSharedPtr Tablet::create_tablet_from_meta(TabletMetaSharedPtr
tablet_meta,
@@ -1852,28 +1876,11 @@ Status check_version_continuity(const
std::vector<RowsetMetaSharedPtr>& rs_metas
// It's guaranteed the write cooldown meta task would be invoked at the end
unless BE crashes
// one tablet would at most have one async task to be done
void Tablet::async_write_cooldown_meta(TabletSharedPtr tablet) {
- auto tablet_id = tablet->tablet_id();
- auto async_write_task = [t = std::move(tablet)]() {
- auto ex = WriteCooldownMetaExecutors::GetInstance();
- {
- std::unique_lock<std::mutex> lck {ex->_latch};
- ex->_pengding_tablets.erase(t->tablet_id());
- }
- auto s = t->_write_cooldown_meta();
- if (s.ok()) {
- return;
- }
- LOG_WARNING("write tablet {} cooldown meta failed because: {}",
t->tablet_id(),
- s.to_string());
- if (!s.is<ABORTED>()) {
- ex->submit(t->tablet_id(), [t]() {
Tablet::async_write_cooldown_meta(t); });
- }
- };
- WriteCooldownMetaExecutors::GetInstance()->submit(tablet_id,
std::move(async_write_task));
+ WriteCooldownMetaExecutors::GetInstance()->submit(std::move(tablet));
}
// hold SHARED `cooldown_conf_lock`
-Status Tablet::_write_cooldown_meta() {
+Status Tablet::write_cooldown_meta() {
auto [cooldown_replica_id, cooldown_term] = cooldown_conf();
std::shared_ptr<io::RemoteFileSystem> fs;
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 5ff1be8fdd..a20719c307 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -367,6 +367,7 @@ public:
std::shared_mutex& get_cooldown_conf_lock() { return _cooldown_conf_lock; }
static void async_write_cooldown_meta(TabletSharedPtr tablet);
+ Status write_cooldown_meta();
////////////////////////////////////////////////////////////////////////////
// end cooldown functions
////////////////////////////////////////////////////////////////////////////
@@ -478,7 +479,6 @@ private:
Status _follow_cooldowned_data();
Status _read_cooldown_meta(const std::shared_ptr<io::RemoteFileSystem>& fs,
TabletMetaPB* tablet_meta_pb);
- Status _write_cooldown_meta();
////////////////////////////////////////////////////////////////////////////
// end cooldown functions
////////////////////////////////////////////////////////////////////////////
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 7a6ec8c765..a318dc508b 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -551,8 +551,6 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
tablet_meta_pb->set_shard_id(shard_id());
tablet_meta_pb->set_creation_time(creation_time());
tablet_meta_pb->set_cumulative_layer_point(cumulative_layer_point());
- tablet_meta_pb->set_local_data_size(tablet_local_size());
- tablet_meta_pb->set_remote_data_size(tablet_remote_size());
*(tablet_meta_pb->mutable_tablet_uid()) = tablet_uid().to_proto();
tablet_meta_pb->set_tablet_type(_tablet_type);
switch (tablet_state()) {
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index b7acb1fa6a..3280ac0e6f 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -128,8 +128,8 @@ Status HttpService::start() {
_ev_http_server->register_handler(HttpMethod::GET, "/metrics", action);
}
- MetaAction* meta_action = _pool.add(new MetaAction(HEADER));
- _ev_http_server->register_handler(HttpMethod::GET,
"/api/meta/header/{tablet_id}", meta_action);
+ MetaAction* meta_action = _pool.add(new MetaAction());
+ _ev_http_server->register_handler(HttpMethod::GET,
"/api/meta/{op}/{tablet_id}", meta_action);
#ifndef BE_TEST
// Register BE checksum action
diff --git a/be/test/olap/test_data/header_without_inc_rs.txt
b/be/test/olap/test_data/header_without_inc_rs.txt
index a9da7fc827..76021315bc 100644
--- a/be/test/olap/test_data/header_without_inc_rs.txt
+++ b/be/test/olap/test_data/header_without_inc_rs.txt
@@ -108,7 +108,5 @@
"preferred_rowset_type": "BETA_ROWSET",
"tablet_type": "TABLET_TYPE_DISK",
"replica_id": 0,
- "enable_unique_key_merge_on_write": false,
- "local_data_size": 84464,
- "remote_data_size": 0
+ "enable_unique_key_merge_on_write": false
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 2a7fd32aba..03fb1a865c 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -298,8 +298,6 @@ message TabletMetaPB {
optional bool enable_unique_key_merge_on_write = 24 [default = false];
optional int64 storage_policy_id = 25;
optional PUniqueId cooldown_meta_id = 26;
- optional int64 local_data_size = 27;
- optional int64 remote_data_size = 28;
}
message OLAPRawDeltaHeaderMessage {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
