This is an automated email from the ASF dual-hosted git repository.
Yukang-Lian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e43a387eaa6 [fix](compaction) Count empty rowsets in approximate
compaction score (#66610)
e43a387eaa6 is described below
commit e43a387eaa646ea36bb3689ab2c324bc9824bf9b
Author: Jamie <[email protected]>
AuthorDate: Tue Aug 11 15:54:12 2026 +0800
[fix](compaction) Count empty rowsets in approximate compaction score
(#66610)
Problem Summary:
Cloud cumulative compaction scheduling maintains an approximate score
between metadata synchronizations. The online write path used the raw
segment count, so a zero-segment rowset added no score even though
metadata synchronization counts every empty rowset as one. Empty rowsets
could therefore accumulate without increasing tablet scheduling
priority.
This PR clamps the online contribution of a load rowset to at least one
and adds the same contribution for zero-segment delete-predicate
rowsets, which bypass `CloudRowsetBuilder`. Non-empty PREPARED rowsets
keep the existing segment-based estimate because their versions have not
been assigned yet.
The unit test models that PREPARED, versionless lifecycle and verifies
both cases: a zero-segment rowset contributes one, while a three-segment
rowset still contributes three.
### Release note
Fix cloud cumulative compaction scheduling for empty rowsets.
---
be/src/cloud/cloud_delete_task.cpp | 1 +
be/src/cloud/cloud_rowset_builder.cpp | 4 ++-
be/test/cloud/cloud_compaction_test.cpp | 52 +++++++++++++++++++++++++++++++++
3 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/be/src/cloud/cloud_delete_task.cpp
b/be/src/cloud/cloud_delete_task.cpp
index 82c52b968cc..34f9354285e 100644
--- a/be/src/cloud/cloud_delete_task.cpp
+++ b/be/src/cloud/cloud_delete_task.cpp
@@ -112,6 +112,7 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine,
const TPushReq& requ
// Update tablet stats
tablet->fetch_add_approximate_num_rowsets(1);
tablet->fetch_add_approximate_cumu_num_rowsets(1);
+ tablet->fetch_add_approximate_cumu_num_deltas(1);
// TODO(liaoxin) delete operator don't send calculate delete bitmap task
from fe,
// then we don't need to set_txn_related_info here.
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index be36abf5775..d36048a6646 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -17,6 +17,8 @@
#include "cloud/cloud_rowset_builder.h"
+#include <algorithm>
+
#include "cloud/cloud_meta_mgr.h"
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
@@ -222,7 +224,7 @@ void CloudRowsetBuilder::update_tablet_stats() {
tablet->fetch_add_approximate_num_rows(_rowset->num_rows());
tablet->fetch_add_approximate_data_size(_rowset->total_disk_size());
tablet->fetch_add_approximate_cumu_num_rowsets(1);
- tablet->fetch_add_approximate_cumu_num_deltas(_rowset->num_segments());
+
tablet->fetch_add_approximate_cumu_num_deltas(std::max<int64_t>(_rowset->num_segments(),
1));
tablet->write_count.fetch_add(1, std::memory_order_relaxed);
}
diff --git a/be/test/cloud/cloud_compaction_test.cpp
b/be/test/cloud/cloud_compaction_test.cpp
index 78440a82de3..9ebf0c8d01e 100644
--- a/be/test/cloud/cloud_compaction_test.cpp
+++ b/be/test/cloud/cloud_compaction_test.cpp
@@ -26,6 +26,7 @@
#include "cloud/cloud_base_compaction.h"
#include "cloud/cloud_cluster_info.h"
+#include "cloud/cloud_rowset_builder.h"
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
#include "cloud/cloud_tablet_mgr.h"
@@ -405,6 +406,57 @@ static RowsetSharedPtr create_rowset(Version version, int
num_segments, bool ove
return rowset;
}
+static RowsetSharedPtr create_prepared_rowset(int num_segments, int data_size)
{
+ auto rs_meta = std::make_shared<RowsetMeta>();
+ rs_meta->set_rowset_type(BETA_ROWSET);
+ rs_meta->set_rowset_state(PREPARED);
+ rs_meta->set_num_segments(num_segments);
+ rs_meta->set_segments_overlap(OVERLAPPING);
+ rs_meta->set_total_disk_size(data_size);
+ RowsetSharedPtr rowset;
+ Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset);
+ if (!st.ok()) {
+ return nullptr;
+ }
+ return rowset;
+}
+
+class TestableCloudRowsetBuilder : public CloudRowsetBuilder {
+public:
+ using CloudRowsetBuilder::CloudRowsetBuilder;
+
+ void set_tablet_and_rowset(const CloudTabletSPtr& tablet, const
RowsetSharedPtr& rowset) {
+ _tablet = tablet;
+ _rowset = rowset;
+ }
+};
+
+TEST_F(CloudCompactionTest, update_tablet_stats_counts_zero_segment_rowset) {
+ auto tablet = std::make_shared<CloudTablet>(_engine, _tablet_meta);
+ {
+ std::unique_lock lock(tablet->get_header_lock());
+ tablet->reset_approximate_stats(0, 0, 0, 0);
+ }
+
+ TestableCloudRowsetBuilder builder(_engine, WriteRequest {}, nullptr);
+
+ auto empty_rowset = create_prepared_rowset(0, 0);
+ ASSERT_NE(empty_rowset, nullptr);
+ ASSERT_TRUE(empty_rowset->is_pending());
+ ASSERT_FALSE(empty_rowset->rowset_meta()->has_version());
+ builder.set_tablet_and_rowset(tablet, empty_rowset);
+ builder.update_tablet_stats();
+ EXPECT_EQ(tablet->fetch_add_approximate_cumu_num_deltas(0), 1);
+
+ auto overlapping_rowset = create_prepared_rowset(3, 41);
+ ASSERT_NE(overlapping_rowset, nullptr);
+ ASSERT_TRUE(overlapping_rowset->is_pending());
+ ASSERT_FALSE(overlapping_rowset->rowset_meta()->has_version());
+ builder.set_tablet_and_rowset(tablet, overlapping_rowset);
+ builder.update_tablet_stats();
+ EXPECT_EQ(tablet->fetch_add_approximate_cumu_num_deltas(0), 4);
+}
+
class TestableCloudCompaction : public CloudCompactionMixin {
public:
TestableCloudCompaction(CloudStorageEngine& engine, CloudTabletSPtr tablet)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
