This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1837ae5691e6121a9f5d02c809dd6e8889d11424 Author: Jamie <[email protected]> AuthorDate: Thu Sep 10 07:43:48 2026 +0800 branch-4.1: [fix](compaction) Keep latest 10 versions unmerged during schema change #66337 (#67589) ### What problem does this PR solve? Issue Number: None Related PR: #66337 Problem Summary: Backport #66337 to `branch-4.1`. While a schema change target tablet is `TABLET_NOTREADY`, cumulative compaction should merge only older rowsets and leave the newest 10 versions unmerged. The filter used the inverse comparison, so a compaction output could cross the base tablet's maximum version and prevent incremental schema-change conversion with `VERSION_ALREADY_MERGED`. This change reverses the comparison in both local and Cloud size-based cumulative compaction policies. It also adds focused unit-test coverage adapted to the branch-4.1 test layout. ### Release note Fix schema changes that could fail when cumulative compaction merged the latest versions on the new tablet. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - Focused local and Cloud unit tests are included; hosted CI is requested with `run buildall`. - `build-support/check-format.sh`: passed - `git diff --check`: passed - [ ] Manual test - [ ] No need to test or manual test - Behavior changed: Yes. `TABLET_NOTREADY` cumulative compaction now leaves the latest 10 versions unmerged. - Does this need documentation? No --------- Co-authored-by: Siyang Tang <[email protected]> --- .../cloud/cloud_cumulative_compaction_policy.cpp | 2 +- .../compaction/cumulative_compaction_policy.cpp | 2 +- .../cloud_cumulative_compaction_policy_test.cpp | 34 ++++++++++++++++ .../cumulative_compaction_policy_test.cpp | 47 ++++++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 53d336e5c70..0f7371307c2 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -160,7 +160,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( if (tablet->tablet_state() == TABLET_NOTREADY) { // If tablet under alter, keep latest 10 version so that base tablet max version // not merged in new tablet, and then we can copy data from base tablet - if (rowset->version().second < max_version - 10) { + if (rowset->version().second > max_version - 10) { continue; } } diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp b/be/src/storage/compaction/cumulative_compaction_policy.cpp index 5326d25bd5f..ade1fe46acb 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp @@ -304,7 +304,7 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets( if (tablet->tablet_state() == TABLET_NOTREADY) { // If tablet under alter, keep latest 10 version so that base tablet max version // not merged in new tablet, and then we can copy data from base tablet - if (rowset->version().second < max_version - 10) { + if (rowset->version().second > max_version - 10) { continue; } } diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp index 29358eaaee0..6e9fa1e1aaa 100644 --- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp +++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp @@ -151,6 +151,40 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, new_cumulative_point) { EXPECT_EQ(policy.new_cumulative_point(&_tablet, output_rowset, version, 2), 6); } +TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, + pick_input_rowsets_notready_keeps_latest_versions) { + auto base_rowset = create_rowset(Version(0, 1), 1, false, 1024 * 1024 * 1024); + ASSERT_NE(nullptr, base_rowset); + ASSERT_TRUE(_tablet_meta->add_rs_meta(base_rowset->rowset_meta()).ok()); + + std::vector<RowsetSharedPtr> candidate_rowsets; + for (int64_t version = 2; version <= 20; ++version) { + auto rowset = create_rowset(Version(version, version), 1, true, 1024 * 1024); + ASSERT_NE(nullptr, rowset); + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset->rowset_meta()).ok()); + candidate_rowsets.push_back(rowset); + } + + CloudTablet tablet(_engine, _tablet_meta); + ASSERT_TRUE(tablet.set_tablet_state(TABLET_NOTREADY).ok()); + tablet._base_size = 1024L * 1024 * 1024; + + std::vector<RowsetSharedPtr> input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + + CloudSizeBasedCumulativeCompactionPolicy policy; + auto picked_size = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 5, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(9, picked_size); + ASSERT_EQ(9, input_rowsets.size()); + EXPECT_EQ(9, compaction_score); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(10, input_rowsets.back()->end_version()); + EXPECT_EQ(Version(-1, -1), last_delete_version); +} + // Test case: Empty rowset compaction with skip_trim TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_empty_rowset_compaction) { // Save original config values diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp index 81a2d1705fa..6bff25b9ff1 100644 --- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp +++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp @@ -476,6 +476,53 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_normal) { EXPECT_EQ(-1, last_delete_version.second); } +TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_notready_keeps_latest_versions) { + std::vector<RowsetMetaSharedPtr> rs_metas; + + RowsetMetaSharedPtr base_rowset(new RowsetMeta()); + init_rs_meta(base_rowset, 0, 1); + base_rowset->set_total_disk_size(1024L * 1024 * 1024); + base_rowset->set_segments_overlap(NONOVERLAPPING); + rs_metas.push_back(base_rowset); + + for (int64_t version = 2; version <= 20; ++version) { + RowsetMetaSharedPtr rowset(new RowsetMeta()); + init_rs_meta(rowset, version, version); + rowset->set_total_disk_size(1 * 1024 * 1024); + rowset->set_num_segments(1); + rowset->set_segments_overlap(OVERLAPPING); + rs_metas.push_back(rowset); + } + + for (const auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr tablet( + new Tablet(_engine, _tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY)); + ASSERT_TRUE(tablet->init().ok()); + ASSERT_TRUE(tablet->set_tablet_state(TABLET_NOTREADY).ok()); + tablet->calculate_cumulative_point(); + + EXPECT_EQ(2, tablet->cumulative_layer_point()); + auto candidate_rowsets = tablet->pick_candidate_rowsets_to_cumulative_compaction(); + ASSERT_EQ(19, candidate_rowsets.size()); + + std::vector<RowsetSharedPtr> input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + auto picked_size = tablet->_cumulative_compaction_policy->pick_input_rowsets( + tablet.get(), candidate_rowsets, 100, 5, &input_rowsets, &last_delete_version, + &compaction_score, config::enable_delete_when_cumu_compaction); + + EXPECT_EQ(9, picked_size); + ASSERT_EQ(9, input_rowsets.size()); + EXPECT_EQ(9, compaction_score); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(10, input_rowsets.back()->end_version()); + EXPECT_EQ(Version(-1, -1), last_delete_version); +} + TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_big_base) { std::vector<RowsetMetaSharedPtr> rs_metas; init_rs_meta_big_base(&rs_metas); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
