This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0fb1f0f0170 branch-3.0: [Opt](compaction) Prune rows with delete
sign=1 in full compaction #51874 (#52256)
0fb1f0f0170 is described below
commit 0fb1f0f0170188052a2781018ec42cd51114bc50
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Jun 30 15:51:25 2025 +0800
branch-3.0: [Opt](compaction) Prune rows with delete sign=1 in full
compaction #51874 (#52256)
Cherry-picked from #51874
Co-authored-by: bobhan1 <[email protected]>
---
be/src/cloud/cloud_tablet.cpp | 7 +++++--
be/src/olap/tablet_reader.cpp | 3 ++-
.../data/compaction/test_full_compaction.out | Bin 336 -> 332 bytes
.../compaction/test_full_compaction_by_table_id.out | Bin 336 -> 332 bytes
4 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 582a4b77126..4344b085a83 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -950,7 +950,9 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
if ((config::enable_missing_rows_correctness_check ||
config::enable_mow_compaction_correctness_check_core) &&
!allow_delete_in_cumu_compaction &&
- compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION) {
+ (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION ||
+ !config::enable_prune_delete_sign_when_base_compaction)) {
+ // also check duplicate key for base compaction when
config::enable_prune_delete_sign_when_base_compaction==false
missed_rows = std::make_unique<RowLocationSet>();
LOG(INFO) << "RowLocation Set inited succ for tablet:" << tablet_id();
}
@@ -971,7 +973,8 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
if (missed_rows) {
missed_rows_size = missed_rows->size();
if (!allow_delete_in_cumu_compaction) {
- if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
+ if ((compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION ||
+ !config::enable_prune_delete_sign_when_base_compaction) &&
tablet_state() == TABLET_RUNNING) {
if (merged_rows >= 0 && merged_rows != missed_rows_size) {
std::string err_msg = fmt::format(
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index a0c3a9c5eda..a453634ca83 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -635,7 +635,8 @@ Status TabletReader::_init_delete_condition(const
ReaderParams& read_params) {
// Delete sign could not be applied when delete on cumu compaction is
enabled, bucause it is meant for delete with predicates.
// If delete design is applied on cumu compaction, it will lose effect
when doing base compaction.
// `_delete_sign_available` indicates the condition where we could apply
delete signs to data.
- _delete_sign_available = ((read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION &&
+ _delete_sign_available = (((read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION ||
+ read_params.reader_type ==
ReaderType::READER_FULL_COMPACTION) &&
config::enable_prune_delete_sign_when_base_compaction) ||
read_params.reader_type ==
ReaderType::READER_COLD_DATA_COMPACTION ||
read_params.reader_type ==
ReaderType::READER_CHECKSUM);
diff --git a/regression-test/data/compaction/test_full_compaction.out
b/regression-test/data/compaction/test_full_compaction.out
index b25fdad9314..118191b6cc8 100644
Binary files a/regression-test/data/compaction/test_full_compaction.out and
b/regression-test/data/compaction/test_full_compaction.out differ
diff --git
a/regression-test/data/compaction/test_full_compaction_by_table_id.out
b/regression-test/data/compaction/test_full_compaction_by_table_id.out
index b25fdad9314..118191b6cc8 100644
Binary files
a/regression-test/data/compaction/test_full_compaction_by_table_id.out and
b/regression-test/data/compaction/test_full_compaction_by_table_id.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
