This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new ade93477a8 [fix](merge-on-write) fix that failed to update delete
bitmap when revise tablet meta for clone (#17857)
ade93477a8 is described below
commit ade93477a87a6f215316a6681d62e383a188cc3e
Author: Xin Liao <[email protected]>
AuthorDate: Sat Mar 18 11:06:22 2023 +0800
[fix](merge-on-write) fix that failed to update delete bitmap when revise
tablet meta for clone (#17857)
---
be/src/olap/tablet.cpp | 44 +++++++++++++++++++----
be/src/olap/tablet.h | 3 +-
be/src/olap/task/engine_clone_task.cpp | 64 ++++++++--------------------------
3 files changed, 53 insertions(+), 58 deletions(-)
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 3c6dfdfdf3..76110cfc6f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -180,7 +180,8 @@ void Tablet::save_meta() {
}
Status Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
- const std::vector<Version>&
versions_to_delete) {
+ const std::vector<Version>&
versions_to_delete,
+ bool is_incremental_clone) {
LOG(INFO) << "begin to revise tablet. tablet=" << full_name()
<< ", rowsets_to_clone=" << rowsets_to_clone.size()
<< ", versions_to_delete=" << versions_to_delete.size();
@@ -213,17 +214,46 @@ Status Tablet::revise_tablet_meta(const
std::vector<RowsetMetaSharedPtr>& rowset
_rowset_tree = std::move(new_rowset_tree);
std::vector<RowsetSharedPtr> calc_delete_bitmap_rowsets;
int64_t to_add_min_version = INT64_MAX;
+ int64_t to_add_max_version = INT64_MIN;
for (auto& rs : rs_to_add) {
if (to_add_min_version > rs->start_version()) {
to_add_min_version = rs->start_version();
}
+ if (to_add_max_version < rs->end_version()) {
+ to_add_max_version = rs->end_version();
+ }
+ }
+ for (auto& [ver, rs] : _rs_version_map) {
+ if (is_incremental_clone && ver.first >= to_add_min_version) {
+ // From the rowset of to_add with smallest version, all other
rowsets
+ // need to recalculate the delete bitmap
+ // For example:
+ // local tablet: [0-1] [2-5] [6-6] [9-10]
+ // clone tablet: [7-7] [8-8]
+ // new tablet: [0-1] [2-5] [6-6] [7-7] [8-8] [9-10]
+ // [7-7] [8-8] [9-10] need to recalculate delete bitmap
+ calc_delete_bitmap_rowsets.push_back(rs);
+ } else if (!is_incremental_clone && ver.first >
to_add_max_version) {
+ // the delete bitmap of to_add's rowsets has clone from remote
when full clone.
+ // only other rowsets in local need to recalculate the delete
bitmap.
+ // For example:
+ // local tablet: [0-1]x [2-5]x [6-6]x [7-7]x [9-10]
+ // clone tablet: [0-1] [2-4] [5-6] [7-8]
+ // new tablet: [0-1] [2-4] [5-6] [7-8] [9-10]
+ // only [9-10] need to recalculate delete bitmap
+ calc_delete_bitmap_rowsets.push_back(rs);
+ }
}
- Version calc_delete_bitmap_ver;
- calc_delete_bitmap_ver = Version(to_add_min_version,
max_version().second);
- RETURN_IF_ERROR(
- capture_consistent_rowsets(calc_delete_bitmap_ver,
&calc_delete_bitmap_rowsets));
- for (auto rs : calc_delete_bitmap_rowsets) {
- RETURN_IF_ERROR(update_delete_bitmap_without_lock(rs));
+ std::sort(calc_delete_bitmap_rowsets.begin(),
calc_delete_bitmap_rowsets.end(),
+ Rowset::comparator);
+ for (size_t i = 0; i < calc_delete_bitmap_rowsets.size(); i++) {
+ CHECK(i == 0 || calc_delete_bitmap_rowsets[i]->start_version() ==
+ calc_delete_bitmap_rowsets[i -
1]->end_version() + 1)
+ << "calc_delete_bitmap_rowsets[" << i
+ << "] version: " <<
calc_delete_bitmap_rowsets[i]->version()
+ << " calc_delete_bitmap_rowsets[" << i - 1
+ << "] version: " << calc_delete_bitmap_rowsets[i -
1]->version();
+
CHECK(update_delete_bitmap_without_lock(calc_delete_bitmap_rowsets[i]).ok());
}
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index b56862df59..64ef265b0f 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -81,7 +81,8 @@ public:
void save_meta();
// Used in clone task, to update local meta when finishing a clone job
Status revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
- const std::vector<Version>& versions_to_delete);
+ const std::vector<Version>& versions_to_delete,
+ bool is_incremental_clone);
Status pick_quick_compaction_rowsets(std::vector<RowsetSharedPtr>*
input_rowsets,
int64_t* permits);
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index b77373ec6b..5c8c2f8070 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -562,7 +562,7 @@ Status EngineCloneTask::_finish_incremental_clone(Tablet*
tablet,
/// For incremental clone, nothing will be deleted.
/// So versions_to_delete is empty.
std::vector<Version> versions_to_delete;
- return tablet->revise_tablet_meta(rowsets_to_clone, versions_to_delete);
+ return tablet->revise_tablet_meta(rowsets_to_clone, versions_to_delete,
true);
}
/// This method will do:
@@ -582,8 +582,8 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet,
TabletMeta* cloned_ta
//
// after compare, the version mark with "x" will be deleted
//
- // local tablet: [0-1] [2-5]x [6-6]x [7-7]x
- // clone tablet: [0-1]x [2-4] [5-6] [7-8]
+ // local tablet: [0-1]x [2-5]x [6-6]x [7-7]x
+ // clone tablet: [0-1] [2-4] [5-6] [7-8]
// All versions deleted from local tablet will be saved in
versions_to_delete
// And these versions file will be deleted finally.
@@ -606,33 +606,12 @@ Status EngineCloneTask::_finish_full_clone(Tablet*
tablet, TabletMeta* cloned_ta
"version cross src latest. cloned_max_version={},
local_version={}",
cloned_max_version.to_string(), local_version.to_string());
} else if (local_version.second <= cloned_max_version.second) {
- // if local version smaller than src, check if existed in src,
will not clone it
- bool existed_in_src = false;
-
- // if delta labeled with local_version is same with the specified
version in clone header,
- // there is no necessity to clone it.
- for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
- if (rs_meta->version().first == local_version.first &&
- rs_meta->version().second == local_version.second) {
- existed_in_src = true;
- break;
- }
- }
-
- if (existed_in_src) {
- cloned_tablet_meta->delete_rs_meta_by_version(local_version,
-
&rs_metas_found_in_src);
- LOG(INFO) << "version exist in local tablet, no need to clone.
delete it from "
- "clone tablet"
- << ". tablet=" << tablet->full_name() << ",
version='" << local_version;
- } else {
- versions_to_delete.push_back(local_version);
- LOG(INFO) << "version not exist in local tablet. it will be
replaced by other "
- "version. "
- << "delete it from local tablet. "
- << "tablet=" << tablet->full_name() << ","
- << ", version=" << local_version;
- }
+ versions_to_delete.push_back(local_version);
+ LOG(INFO) << "version not exist in local tablet. it will be
replaced by other "
+ "version. "
+ << "delete it from local tablet. "
+ << "tablet=" << tablet->full_name() << ","
+ << ", version=" << local_version;
}
}
std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
@@ -642,32 +621,17 @@ Status EngineCloneTask::_finish_full_clone(Tablet*
tablet, TabletMeta* cloned_ta
<< tablet->full_name() << ", version=" << rs_meta->version();
}
+ if (tablet->enable_unique_key_merge_on_write()) {
+ tablet->tablet_meta()->delete_bitmap() =
cloned_tablet_meta->delete_bitmap();
+ }
+
// clone_data to tablet
// only replace rowset info, must not modify other info such as alter task
info. for example
// 1. local tablet finished alter task
// 2. local tablet has error in push
// 3. local tablet cloned rowset from other nodes
// 4. if cleared alter task info, then push will not write to new tablet,
the report info is error
- Status clone_res = tablet->revise_tablet_meta(rowsets_to_clone,
versions_to_delete);
- // in previous step, copy all files from CLONE_DIR to tablet dir
- // but some rowset is useless, so that remove them here
- for (auto& rs_meta_ptr : rs_metas_found_in_src) {
- RowsetSharedPtr rowset_to_remove;
- auto s =
-
RowsetFactory::create_rowset(cloned_tablet_meta->tablet_schema(),
- tablet->tablet_path(),
rs_meta_ptr, &rowset_to_remove);
- if (!s.ok()) {
- LOG(WARNING) << "failed to init rowset to remove: "
- << rs_meta_ptr->rowset_id().to_string();
- continue;
- }
- s = rowset_to_remove->remove();
- if (!s.ok()) {
- LOG(WARNING) << "failed to remove rowset " <<
rs_meta_ptr->rowset_id().to_string()
- << ": " << s;
- }
- }
- return clone_res;
+ return tablet->revise_tablet_meta(rowsets_to_clone, versions_to_delete,
false);
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
