This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b0d67c0358 [fix](merge-on-write) fix cu compaction correctness check
(#17347)
b0d67c0358 is described below
commit b0d67c03582ef4195ddeade073999d36b055b1d7
Author: Xin Liao <[email protected]>
AuthorDate: Mon Mar 6 21:31:48 2023 +0800
[fix](merge-on-write) fix cu compaction correctness check (#17347)
During concurrent import, the same row location may be marked delete
multiple times by different versions of rowset.
Duplicate row location need to be removed.
---
be/src/olap/compaction.cpp | 10 ++++++----
be/src/olap/tablet.cpp | 6 +++---
be/src/olap/utils.h | 14 ++++++++++++++
3 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index b405e33815..dcafd2a279 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -434,11 +434,13 @@ Status Compaction::modify_rowsets(const
Merger::Statistics* stats) {
RETURN_IF_ERROR(_tablet->check_rowid_conversion(_output_rowset,
location_map));
if (compaction_type() == READER_CUMULATIVE_COMPACTION) {
- std::string err_msg =
- "The merged rows is not equal to missed rows in rowid
conversion";
- DCHECK(stats != nullptr || stats->merged_rows == missed_rows)
<< err_msg;
+ std::string err_msg = fmt::format(
+ "cumulative compaction: the merged rows({}) is not
equal to missed "
+ "rows({}) in rowid conversion, tablet_id: {},
table_id:{}",
+ stats->merged_rows, missed_rows, _tablet->tablet_id(),
_tablet->table_id());
+ DCHECK(stats == nullptr || stats->merged_rows == missed_rows)
<< err_msg;
if (stats != nullptr && stats->merged_rows != missed_rows) {
- return Status::InternalError(err_msg);
+ LOG(WARNING) << err_msg;
}
}
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index ac24121a77..d20fa26e77 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2594,7 +2594,7 @@ uint64_t
Tablet::calc_compaction_output_rowset_delete_bitmap(
uint64_t start_version, uint64_t end_version,
std::map<RowsetSharedPtr, std::list<std::pair<RowLocation,
RowLocation>>>* location_map,
DeleteBitmap* output_rowset_delete_bitmap) {
- uint64_t missed_rows = 0;
+ std::set<RowLocation> missed_rows;
RowLocation src;
RowLocation dst;
for (auto& rowset : input_rowsets) {
@@ -2616,7 +2616,7 @@ uint64_t
Tablet::calc_compaction_output_rowset_delete_bitmap(
<< " src loaction: |" << src.rowset_id
<< "|"
<< src.segment_id << "|" << src.row_id
<< " version: " << cur_version;
- missed_rows++;
+ missed_rows.insert(src);
continue;
}
VLOG_DEBUG << "calc_compaction_output_rowset_delete_bitmap
dst location: |"
@@ -2631,7 +2631,7 @@ uint64_t
Tablet::calc_compaction_output_rowset_delete_bitmap(
}
}
}
- return missed_rows;
+ return missed_rows.size();
}
void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) {
diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h
index a0b7d61024..53cb0912bd 100644
--- a/be/src/olap/utils.h
+++ b/be/src/olap/utils.h
@@ -276,6 +276,20 @@ struct RowLocation {
RowsetId rowset_id;
uint32_t segment_id;
uint32_t row_id;
+
+ bool operator==(const RowLocation& rhs) const {
+ return rowset_id == rhs.rowset_id && segment_id == rhs.segment_id &&
row_id == rhs.row_id;
+ }
+
+ bool operator<(const RowLocation& rhs) const {
+ if (rowset_id != rhs.rowset_id) {
+ return rowset_id < rhs.rowset_id;
+ } else if (segment_id != rhs.segment_id) {
+ return segment_id < rhs.segment_id;
+ } else {
+ return row_id < rhs.row_id;
+ }
+ }
};
struct GlobalRowLoacation {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
