This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 713665930b0f5dce33ef9d6cf06c4b9981944cb4 Author: Xin Liao <[email protected]> AuthorDate: Mon Mar 6 21:31:48 2023 +0800 [fix](merge-on-write) fix cu compaction correctness check (#17347) During concurrent import, the same row location may be marked delete multiple times by different versions of rowset. Duplicate row location need to be removed. --- be/src/olap/compaction.cpp | 10 ++++++---- be/src/olap/tablet.cpp | 6 +++--- be/src/olap/utils.h | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 71aec5bad7..bbf82a390c 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -463,11 +463,13 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) { RETURN_IF_ERROR(_tablet->check_rowid_conversion(_output_rowset, location_map)); if (compaction_type() == READER_CUMULATIVE_COMPACTION) { - std::string err_msg = - "The merged rows is not equal to missed rows in rowid conversion"; - DCHECK(stats != nullptr || stats->merged_rows == missed_rows) << err_msg; + std::string err_msg = fmt::format( + "cumulative compaction: the merged rows({}) is not equal to missed " + "rows({}) in rowid conversion, tablet_id: {}, table_id:{}", + stats->merged_rows, missed_rows, _tablet->tablet_id(), _tablet->table_id()); + DCHECK(stats == nullptr || stats->merged_rows == missed_rows) << err_msg; if (stats != nullptr && stats->merged_rows != missed_rows) { - return Status::InternalError(err_msg); + LOG(WARNING) << err_msg; } } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 9e8ddfdb85..e1c84b9c90 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2232,7 +2232,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap( uint64_t start_version, uint64_t end_version, std::map<RowsetSharedPtr, std::list<std::pair<RowLocation, RowLocation>>>* location_map, DeleteBitmap* output_rowset_delete_bitmap) { - uint64_t missed_rows = 0; + std::set<RowLocation> missed_rows; RowLocation src; RowLocation dst; for (auto& rowset : input_rowsets) { @@ -2254,7 +2254,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap( << " src loaction: |" << src.rowset_id << "|" << src.segment_id << "|" << src.row_id << " version: " << cur_version; - missed_rows++; + missed_rows.insert(src); continue; } VLOG_DEBUG << "calc_compaction_output_rowset_delete_bitmap dst location: |" @@ -2269,7 +2269,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap( } } } - return missed_rows; + return missed_rows.size(); } void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) { diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index f3c0714119..bc820f8516 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -293,6 +293,20 @@ struct RowLocation { RowsetId rowset_id; uint32_t segment_id; uint32_t row_id; + + bool operator==(const RowLocation& rhs) const { + return rowset_id == rhs.rowset_id && segment_id == rhs.segment_id && row_id == rhs.row_id; + } + + bool operator<(const RowLocation& rhs) const { + if (rowset_id != rhs.rowset_id) { + return rowset_id < rhs.rowset_id; + } else if (segment_id != rhs.segment_id) { + return segment_id < rhs.segment_id; + } else { + return row_id < rhs.row_id; + } + } }; } // namespace doris --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
