This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2389a90cd0 [enhancement](snapshot) add missed version log when
make_snapshot in engine clone task (#14284)
2389a90cd0 is described below
commit 2389a90cd08d0d9fd8af3de351f4b43b19083fa7
Author: AlexYue <[email protected]>
AuthorDate: Thu Nov 24 14:51:28 2022 +0800
[enhancement](snapshot) add missed version log when make_snapshot in engine
clone task (#14284)
---
be/src/olap/olap_common.h | 7 +++++++
be/src/olap/task/engine_clone_task.cpp | 28 ++++++++++++++--------------
be/src/olap/task/engine_clone_task.h | 4 ++--
3 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 7a396a188e..2ca2c37982 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -233,6 +233,13 @@ inline std::ostream& operator<<(std::ostream& os, const
Version& version) {
return os << version.to_string();
}
+inline std::ostream& operator<<(std::ostream& os, const Versions& versions) {
+ for (auto& version : versions) {
+ os << version;
+ }
+ return os;
+}
+
// used for hash-struct of hash_map<Version, Rowset*>.
struct HashOfVersion {
size_t operator()(const Version& version) const {
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index e78095e097..4bb86501b0 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -77,6 +77,8 @@ Status EngineCloneTask::_do_clone() {
TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(_clone_req.tablet_id);
bool is_new_tablet = tablet == nullptr;
+ // try to incremental clone
+ std::vector<Version> missed_versions;
// try to repair a tablet with missing version
if (tablet != nullptr) {
std::shared_lock migration_rlock(tablet->get_migration_lock(),
std::try_to_lock);
@@ -88,8 +90,6 @@ Status EngineCloneTask::_do_clone() {
auto local_data_path = fmt::format("{}/{}", tablet->tablet_path(),
CLONE_PREFIX);
bool allow_incremental_clone = false;
- // try to incremental clone
- std::vector<Version> missed_versions;
tablet->calc_missed_versions(_clone_req.committed_version,
&missed_versions);
// if missed version size is 0, then it is useless to clone from
remote be, it means local data is
@@ -110,7 +110,7 @@ Status EngineCloneTask::_do_clone() {
// if tablet on src backend does not contains missing version, it will
download all versions,
// and set allow_incremental_clone to false
RETURN_IF_ERROR(_make_and_download_snapshots(*(tablet->data_dir()),
local_data_path,
- &src_host,
&src_file_path, &missed_versions,
+ &src_host,
&src_file_path, missed_versions,
&allow_incremental_clone));
RETURN_IF_ERROR(_finish_clone(tablet.get(), local_data_path,
_clone_req.committed_version,
@@ -140,7 +140,7 @@ Status EngineCloneTask::_do_clone() {
bool allow_incremental_clone = false;
status = _make_and_download_snapshots(*store, tablet_dir, &src_host,
&src_file_path,
- nullptr,
&allow_incremental_clone);
+ missed_versions,
&allow_incremental_clone);
if (!status.ok()) {
return status;
}
@@ -206,7 +206,7 @@ Status EngineCloneTask::_set_tablet_info(bool
is_new_tablet) {
Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
const std::string&
local_data_path,
TBackend* src_host,
string* snapshot_path,
- const
std::vector<Version>* missed_versions,
+ const
std::vector<Version>& missed_versions,
bool*
allow_incremental_clone) {
Status status = Status::OK();
@@ -229,13 +229,15 @@ Status
EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
.tag("port", src.be_port)
.tag("tablet", _clone_req.tablet_id)
.tag("snapshot_path", *snapshot_path)
- .tag("signature", _signature);
+ .tag("signature", _signature)
+ .tag("missed_versions", missed_versions);
} else {
LOG_WARNING("failed to make snapshot in remote BE")
.tag("host", src.host)
.tag("port", src.be_port)
.tag("tablet", _clone_req.tablet_id)
.tag("signature", _signature)
+ .tag("missed_versions", missed_versions)
.error(status);
continue;
}
@@ -285,19 +287,17 @@ Status
EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
Status EngineCloneTask::_make_snapshot(const std::string& ip, int port,
TTableId tablet_id,
TSchemaHash schema_hash, int timeout_s,
- const std::vector<Version>*
missed_versions,
+ const std::vector<Version>&
missed_versions,
std::string* snapshot_path, bool*
allow_incremental_clone) {
TSnapshotRequest request;
request.__set_tablet_id(tablet_id);
request.__set_schema_hash(schema_hash);
request.__set_preferred_snapshot_version(g_Types_constants.TPREFER_SNAPSHOT_REQ_VERSION);
- if (missed_versions != nullptr) {
- // TODO: missing version composed of singleton delta.
- // if not, this place should be rewrote.
- request.__isset.missing_version = true;
- for (auto& version : *missed_versions) {
- request.missing_version.push_back(version.first);
- }
+ // TODO: missing version composed of singleton delta.
+ // if not, this place should be rewrote.
+ request.__isset.missing_version = (!missed_versions.empty());
+ for (auto& version : missed_versions) {
+ request.missing_version.push_back(version.first);
}
if (timeout_s > 0) {
request.__set_timeout(timeout_s);
diff --git a/be/src/olap/task/engine_clone_task.h
b/be/src/olap/task/engine_clone_task.h
index eae7bf0fb6..bfa6d212a6 100644
--- a/be/src/olap/task/engine_clone_task.h
+++ b/be/src/olap/task/engine_clone_task.h
@@ -52,7 +52,7 @@ private:
Status _make_and_download_snapshots(DataDir& data_dir, const std::string&
local_data_path,
TBackend* src_host, string*
src_file_path,
- const vector<Version>*
missing_versions,
+ const vector<Version>&
missing_versions,
bool* allow_incremental_clone);
Status _set_tablet_info(bool is_new_tablet);
@@ -63,7 +63,7 @@ private:
Status _make_snapshot(const std::string& ip, int port, TTableId tablet_id,
TSchemaHash schema_hash, int timeout_s,
- const std::vector<Version>* missed_versions,
std::string* snapshot_path,
+ const std::vector<Version>& missing_versions,
std::string* snapshot_path,
bool* allow_incremental_clone);
Status _release_snapshot(const std::string& ip, int port, const
std::string& snapshot_path);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
