This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new df78ee3d12f [fix](mow) cloud tablet should consider delete in cumu
compaction (#34479)
df78ee3d12f is described below
commit df78ee3d12f55ae9ee096d32892565fc3d88a187
Author: huanghaibin <[email protected]>
AuthorDate: Tue May 7 21:22:08 2024 +0800
[fix](mow) cloud tablet should consider delete in cumu compaction (#34479)
---
be/src/cloud/cloud_base_compaction.cpp | 3 ++-
be/src/cloud/cloud_cumulative_compaction.cpp | 3 ++-
be/src/cloud/cloud_tablet.cpp | 23 +++++++++++++----------
be/src/cloud/cloud_tablet.h | 3 ++-
4 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/be/src/cloud/cloud_base_compaction.cpp
b/be/src/cloud/cloud_base_compaction.cpp
index dc3de319fce..4f9ddf5d655 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -285,7 +285,8 @@ Status CloudBaseCompaction::modify_rowsets() {
std::numeric_limits<int64_t>::max();
RETURN_IF_ERROR(cloud_tablet()->calc_delete_bitmap_for_compaction(
_input_rowsets, _output_rowset, _rowid_conversion,
compaction_type(),
- _stats.merged_rows, initiator, output_rowset_delete_bitmap));
+ _stats.merged_rows, initiator, output_rowset_delete_bitmap,
+ _allow_delete_in_cumu_compaction));
compaction_job->set_delete_bitmap_lock_initiator(initiator);
}
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index 7f8e2cde152..f1e9f329254 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -240,7 +240,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
std::numeric_limits<int64_t>::max();
RETURN_IF_ERROR(cloud_tablet()->calc_delete_bitmap_for_compaction(
_input_rowsets, _output_rowset, _rowid_conversion,
compaction_type(),
- _stats.merged_rows, initiator, output_rowset_delete_bitmap));
+ _stats.merged_rows, initiator, output_rowset_delete_bitmap,
+ _allow_delete_in_cumu_compaction));
compaction_job->set_delete_bitmap_lock_initiator(initiator);
}
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 0da6980f250..886b4e40934 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -626,7 +626,8 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo*
txn_info, int64_t tx
Status CloudTablet::calc_delete_bitmap_for_compaction(
const std::vector<RowsetSharedPtr>& input_rowsets, const
RowsetSharedPtr& output_rowset,
const RowIdConversion& rowid_conversion, ReaderType compaction_type,
int64_t merged_rows,
- int64_t initiator, DeleteBitmapPtr& output_rowset_delete_bitmap) {
+ int64_t initiator, DeleteBitmapPtr& output_rowset_delete_bitmap,
+ bool allow_delete_in_cumu_compaction) {
output_rowset_delete_bitmap = std::make_shared<DeleteBitmap>(tablet_id());
std::set<RowLocation> missed_rows;
std::map<RowsetSharedPtr, std::list<std::pair<RowLocation, RowLocation>>>
location_map;
@@ -638,15 +639,17 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
input_rowsets, rowid_conversion, 0, version.second + 1,
&missed_rows, &location_map,
tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get());
std::size_t missed_rows_size = missed_rows.size();
- if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
- tablet_state() == TABLET_RUNNING) {
- if (merged_rows >= 0 && merged_rows != missed_rows_size) {
- std::string err_msg = fmt::format(
- "cumulative compaction: the merged rows({}) is not equal
to missed "
- "rows({}) in rowid conversion, tablet_id: {}, table_id:{}",
- merged_rows, missed_rows_size, tablet_id(), table_id());
- DCHECK(false) << err_msg;
- LOG(WARNING) << err_msg;
+ if (!allow_delete_in_cumu_compaction) {
+ if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
+ tablet_state() == TABLET_RUNNING) {
+ if (merged_rows >= 0 && merged_rows != missed_rows_size) {
+ std::string err_msg = fmt::format(
+ "cumulative compaction: the merged rows({}) is not
equal to missed "
+ "rows({}) in rowid conversion, tablet_id: {},
table_id:{}",
+ merged_rows, missed_rows_size, tablet_id(),
table_id());
+ DCHECK(false) << err_msg;
+ LOG(WARNING) << err_msg;
+ }
}
}
if (config::enable_rowid_conversion_correctness_check) {
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index 08d11385479..e0ab3519787 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -186,7 +186,8 @@ public:
const RowIdConversion&
rowid_conversion,
ReaderType compaction_type,
int64_t merged_rows,
int64_t initiator,
- DeleteBitmapPtr&
output_rowset_delete_bitmap);
+ DeleteBitmapPtr&
output_rowset_delete_bitmap,
+ bool
allow_delete_in_cumu_compaction);
std::mutex& get_rowset_update_lock() { return _rowset_update_lock; }
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
