This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 8c20c2703a0 branch-4.2: [fix](fe) Enforce minimum buckets for
automatic partition fallbacks (#69353) (#69653)
8c20c2703a0 is described below
commit 8c20c2703a0d9cbb4b0afdbd5ff730de4ef9aa8a
Author: TengJianPing <[email protected]>
AuthorDate: Tue Oct 27 18:53:36 2026 +0800
branch-4.2: [fix](fe) Enforce minimum buckets for automatic partition
fallbacks (#69353) (#69653)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #69353
Problem Summary: Raising autobucket_min_buckets after creating an AUTO
RANGE partitioned table could leave new partitions below the minimum.
The shared calculator returned the stored default without applying the
current minimum when history was absent, empty, or unavailable. It also
reused a historical bucket count directly when size statistics had not
arrived.
Apply the current minimum to automatic default and historical bucket
counts in the shared calculator used by automatic range and dynamic
partitions. Keep fixed bucket counts and existing partitions unchanged.
Cover the fallback paths with unit tests and the configuration change
with the original SQL regression.
Resolve the branch-4.2 unit test conflict by using Mockito throughout
AutoBucketCalculatorTest while retaining its existing test cases.
### Release note
New automatic range and dynamic partitions respect
autobucket_min_buckets when reusing defaults or historical bucket counts
without size statistics. Existing partitions and fixed bucket
distributions retain their bucket counts.
### Check List (For Author)
- Test: Unit Test / Checkstyle
- run-fe-ut.sh --run
'org.apache.doris.common.util.AutoBucketCalculatorTest,org.apache.doris.catalog.DynamicPartitionTableTest#testAutoBuckets':
12 tests passed
- mvn checkstyle:check -pl fe-core: passed with zero violations
- SQL regression files preserved from the original PR; SQL regression
not rerun for this backport
- Behavior changed: Yes (new automatically bucketed partitions enforce
the current minimum on fallback paths)
- Does this need documentation: No (restore the existing minimum-bucket
configuration semantics)
(cherry picked from commit a1b1cf722fba9526123214c574394c3d885dc973)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../doris/common/util/AutoBucketCalculator.java | 12 +-
.../common/util/AutoBucketCalculatorTest.java | 196 ++++++++++++++++-----
.../autobucket/test_autobucket_min_buckets.out | 11 ++
.../autobucket/test_autobucket_min_buckets.groovy | 109 ++++++++++++
4 files changed, 279 insertions(+), 49 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
index 4434c404a7c..1765db49930 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketCalculator.java
@@ -20,6 +20,7 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.clone.DynamicPartitionScheduler;
+import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.rpc.RpcException;
@@ -130,10 +131,14 @@ public class AutoBucketCalculator {
String partitionName = context.getPartitionName();
String nowPartitionName = context.getNowPartitionName();
boolean executeFirstTime = context.isExecuteFirstTime();
- int defaultBuckets = context.getDefaultBuckets();
+ boolean autoBucket = table.isAutoBucket();
+ int minBuckets = Config.autobucket_min_buckets;
+ // Stored defaults may predate a change to the minimum. Fixed-bucket
tables keep their configured count.
+ int defaultBuckets = autoBucket
+ ? Math.max(context.getDefaultBuckets(), minBuckets) :
context.getDefaultBuckets();
// if execute first time or not auto bucket, use default buckets
- if (!table.isAutoBucket() || executeFirstTime) {
+ if (!autoBucket || executeFirstTime) {
return AutoBucketResult.fallback(defaultBuckets,
executeFirstTime ? "executeFirstTime" : "not auto bucket
table");
}
@@ -173,7 +178,8 @@ public class AutoBucketCalculator {
// Calculate buckets based on historical data
Pair<Integer, Integer> calc =
DynamicPartitionScheduler.calculateBuckets(hasDataPartitions);
- int candidateBuckets = calc.first;
+ // Historical bucket counts may also predate a change to the minimum
when size statistics are unavailable.
+ int candidateBuckets = Math.max(calc.first, minBuckets);
int previousBuckets = calc.second;
return AutoBucketResult.success(candidateBuckets, previousBuckets);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
index e23ec904252..d49b0c4b54f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketCalculatorTest.java
@@ -17,36 +17,162 @@
package org.apache.doris.common.util;
+import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.RangePartitionInfo;
+import org.apache.doris.clone.DynamicPartitionScheduler;
+import org.apache.doris.common.Config;
import org.apache.doris.common.util.AutoBucketCalculator.AutoBucketContext;
import org.apache.doris.common.util.AutoBucketCalculator.AutoBucketResult;
+import org.apache.doris.rpc.RpcException;
-import mockit.Expectations;
-import mockit.Mocked;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
public class AutoBucketCalculatorTest {
- @Mocked
- OlapTable table;
+ OlapTable table = Mockito.mock(OlapTable.class);
+ private int originalMinBuckets;
+
+ @BeforeEach
+ public void setUp() {
+ originalMinBuckets = Config.autobucket_min_buckets;
+ Config.autobucket_min_buckets = 4;
+ RangePartitionInfo partitionInfo =
Mockito.mock(RangePartitionInfo.class);
+
Mockito.when(partitionInfo.getIdToItem(false)).thenReturn(Collections.emptyMap());
+ Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo);
+ }
+
+ @AfterEach
+ public void tearDown() {
+ Config.autobucket_min_buckets = originalMinBuckets;
+ }
+
+ @Test
+ public void testFixedBucketsBelowMinimum() {
+ for (int buckets : new int[] {1, 2}) {
+ AutoBucketContext context = new AutoBucketContext(table, "p1",
"p2", false, buckets);
+ Assertions.assertEquals(buckets,
AutoBucketCalculator.calculateAutoBuckets(context).getBuckets());
+ Assertions.assertEquals(buckets,
AutoBucketCalculator.calculateAutoBucketsWithBoundsCheck(context));
+ }
+ }
@Test
- public void testCalculateAutoBucketsNotAutoBucket() {
- new Expectations() {
- {
- table.isAutoBucket();
- result = false;
-
- table.getName();
- result = "test_table";
- minTimes = 0;
-
- table.getId();
- result = 1L;
- minTimes = 0;
+ public void testFirstExecutionUsesCurrentMinimum() {
+ Mockito.when(table.isAutoBucket()).thenReturn(true);
+ for (int buckets : new int[] {1, 2, 8}) {
+ AutoBucketContext context = new AutoBucketContext(table, "p1",
"p2", true, buckets);
+ Assertions.assertEquals(Math.max(4, buckets),
+
AutoBucketCalculator.calculateAutoBuckets(context).getBuckets());
+ Assertions.assertEquals(Math.max(4, buckets),
+
AutoBucketCalculator.calculateAutoBucketsWithBoundsCheck(context));
+ }
+ }
+
+ @Test
+ public void testNoHistoricalPartitionsUsesCurrentMinimum() throws
RpcException {
+ assertFallbackUsesCurrentMinimum(Collections.emptyList(),
Collections.emptyList(), "no data partitions");
+ }
+
+ @Test
+ public void testEmptyHistoricalPartitionUsesCurrentMinimum() throws
RpcException {
+
assertFallbackUsesCurrentMinimum(Collections.singletonList(Mockito.mock(Partition.class)),
+ Collections.singletonList(1L), "no data partitions");
+ }
+
+ @Test
+ public void testVisibleVersionsMismatchUsesCurrentMinimum() throws
RpcException {
+
assertFallbackUsesCurrentMinimum(Collections.singletonList(Mockito.mock(Partition.class)),
+ Collections.emptyList(), "visible versions mismatch");
+ }
+
+ @Test
+ public void testMissingVisibleVersionsUsesCurrentMinimum() throws
RpcException {
+
assertFallbackUsesCurrentMinimum(Collections.singletonList(Mockito.mock(Partition.class)),
+ null, "visible versions mismatch");
+ }
+
+ @Test
+ public void testVisibleVersionRpcFailureUsesCurrentMinimum() throws
RpcException {
+ Mockito.when(table.isAutoBucket()).thenReturn(true);
+ List<Partition> partitions =
Collections.singletonList(Mockito.mock(Partition.class));
+ try (MockedStatic<DynamicPartitionScheduler> scheduler =
Mockito.mockStatic(
+ DynamicPartitionScheduler.class,
Mockito.CALLS_REAL_METHODS);
+ MockedStatic<Partition> partitionMethods =
Mockito.mockStatic(Partition.class)) {
+ scheduler.when(() ->
DynamicPartitionScheduler.getHistoricalPartitions(table, "p2"))
+ .thenReturn(partitions);
+ partitionMethods.when(() ->
Partition.getVisibleVersions(partitions))
+ .thenThrow(new RpcException("", "test failure"));
+ assertFallbackBuckets("RpcException: test failure");
+ }
+ }
+
+ @Test
+ public void testUnknownHistoricalSizesUsesCurrentMinimum() throws
RpcException {
+ Mockito.when(table.isAutoBucket()).thenReturn(true);
+ Partition partition = Mockito.mock(Partition.class);
+ HashDistributionInfo distribution =
Mockito.mock(HashDistributionInfo.class);
+ Mockito.when(partition.getDistributionInfo()).thenReturn(distribution);
+
Mockito.when(partition.getDataSizeExcludeEmptyReplica(true)).thenReturn(0L);
+ List<Partition> partitions = Collections.singletonList(partition);
+ try (MockedStatic<DynamicPartitionScheduler> scheduler =
Mockito.mockStatic(
+ DynamicPartitionScheduler.class,
Mockito.CALLS_REAL_METHODS);
+ MockedStatic<Partition> partitionMethods =
Mockito.mockStatic(Partition.class)) {
+ scheduler.when(() ->
DynamicPartitionScheduler.getHistoricalPartitions(table, "p2"))
+ .thenReturn(partitions);
+ partitionMethods.when(() ->
Partition.getVisibleVersions(partitions))
+ .thenReturn(Collections.singletonList(2L));
+ for (int buckets : new int[] {1, 2, 8}) {
+ Mockito.when(distribution.getBucketNum()).thenReturn(buckets);
+ AutoBucketContext context = new AutoBucketContext(table, "p1",
"p2", false, 1);
+ AutoBucketResult result =
AutoBucketCalculator.calculateAutoBuckets(context);
+ Assertions.assertTrue(result.isSuccess());
+ Assertions.assertEquals(Math.max(4, buckets),
result.getBuckets());
+ Assertions.assertEquals(buckets, result.getPreviousBuckets());
+ Assertions.assertEquals(Math.max(4, buckets),
+
AutoBucketCalculator.calculateAutoBucketsWithBoundsCheck(context));
}
- };
+ }
+ }
+
+ private void assertFallbackUsesCurrentMinimum(List<Partition> partitions,
List<Long> versions, String reason)
+ throws RpcException {
+ Mockito.when(table.isAutoBucket()).thenReturn(true);
+ try (MockedStatic<DynamicPartitionScheduler> scheduler =
Mockito.mockStatic(
+ DynamicPartitionScheduler.class,
Mockito.CALLS_REAL_METHODS);
+ MockedStatic<Partition> partitionMethods =
Mockito.mockStatic(Partition.class)) {
+ scheduler.when(() ->
DynamicPartitionScheduler.getHistoricalPartitions(table, "p2"))
+ .thenReturn(partitions);
+ partitionMethods.when(() ->
Partition.getVisibleVersions(partitions)).thenReturn(versions);
+ assertFallbackBuckets(reason);
+ }
+ }
+
+ private void assertFallbackBuckets(String reason) {
+ for (int buckets : new int[] {1, 2, 8}) {
+ AutoBucketContext context = new AutoBucketContext(table, "p1",
"p2", false, buckets);
+ AutoBucketResult result =
AutoBucketCalculator.calculateAutoBuckets(context);
+ Assertions.assertFalse(result.isSuccess());
+ Assertions.assertEquals(reason, result.getErrorMessage());
+ Assertions.assertEquals(Math.max(4, buckets), result.getBuckets());
+ Assertions.assertEquals(Math.max(4, buckets),
+
AutoBucketCalculator.calculateAutoBucketsWithBoundsCheck(context));
+ }
+ }
+
+ @Test
+ public void testCalculateAutoBucketsNotAutoBucket() {
+ Mockito.when(table.isAutoBucket()).thenReturn(false);
+ Mockito.when(table.getName()).thenReturn("test_table");
+ Mockito.when(table.getId()).thenReturn(1L);
AutoBucketContext context = new AutoBucketContext(table, "p1", "p2",
false, 10);
AutoBucketResult result =
AutoBucketCalculator.calculateAutoBuckets(context);
@@ -58,20 +184,9 @@ public class AutoBucketCalculatorTest {
@Test
public void testCalculateAutoBucketsExecuteFirstTime() {
- new Expectations() {
- {
- table.isAutoBucket();
- result = true;
-
- table.getName();
- result = "test_table";
- minTimes = 0;
-
- table.getId();
- result = 1L;
- minTimes = 0;
- }
- };
+ Mockito.when(table.isAutoBucket()).thenReturn(true);
+ Mockito.when(table.getName()).thenReturn("test_table");
+ Mockito.when(table.getId()).thenReturn(1L);
AutoBucketContext context = new AutoBucketContext(table, "p1", "p2",
true, 10);
AutoBucketResult result =
AutoBucketCalculator.calculateAutoBuckets(context);
@@ -83,20 +198,9 @@ public class AutoBucketCalculatorTest {
@Test
public void testCalculateAutoBucketsWithBoundsCheck() {
- new Expectations() {
- {
- table.isAutoBucket();
- result = false;
-
- table.getName();
- result = "test_table";
- minTimes = 0;
-
- table.getId();
- result = 1L;
- minTimes = 0;
- }
- };
+ Mockito.when(table.isAutoBucket()).thenReturn(false);
+ Mockito.when(table.getName()).thenReturn("test_table");
+ Mockito.when(table.getId()).thenReturn(1L);
AutoBucketContext context = new AutoBucketContext(table, "p1", "p2",
false, 10);
int buckets =
AutoBucketCalculator.calculateAutoBucketsWithBoundsCheck(context);
diff --git a/regression-test/data/autobucket/test_autobucket_min_buckets.out
b/regression-test/data/autobucket/test_autobucket_min_buckets.out
new file mode 100644
index 00000000000..5be8e8a659c
--- /dev/null
+++ b/regression-test/data/autobucket/test_autobucket_min_buckets.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !buckets --
+autobucket_min_fixed p20261021000000 2
+autobucket_min_history p20261020000000 1
+autobucket_min_history p20261021000000 4
+autobucket_min_one p20261021000000 4
+autobucket_min_two p20261021000000 4
+
+-- !dynamic_buckets --
+4
+
diff --git
a/regression-test/suites/autobucket/test_autobucket_min_buckets.groovy
b/regression-test/suites/autobucket/test_autobucket_min_buckets.groovy
new file mode 100644
index 00000000000..b95ef4b5002
--- /dev/null
+++ b/regression-test/suites/autobucket/test_autobucket_min_buckets.groovy
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_autobucket_min_buckets", "nonConcurrent") {
+ setFeConfigTemporary([
+ "autobucket_min_buckets": "1",
+ "autobucket_partition_size_per_bucket_gb": "5"
+ ]) {
+ sql "DROP TABLE IF EXISTS autobucket_min_one"
+ sql """
+ CREATE TABLE autobucket_min_one (
+ dt DATETIMEV2(0) NOT NULL,
+ logid BIGINT NOT NULL,
+ content VARIANT NULL
+ ) UNIQUE KEY(dt, logid)
+ AUTO PARTITION BY RANGE(date_trunc(dt, 'day')) ()
+ DISTRIBUTED BY HASH(logid) BUCKETS AUTO
+ PROPERTIES (
+ "replication_num" = "1",
+ "estimate_partition_size" = "1M",
+ "enable_unique_key_merge_on_write" = "true"
+ )
+ """
+
+ sql "DROP TABLE IF EXISTS autobucket_min_two"
+ sql """
+ CREATE TABLE autobucket_min_two (
+ dt DATETIMEV2(0) NOT NULL,
+ logid BIGINT NOT NULL,
+ content VARIANT NULL
+ ) UNIQUE KEY(dt, logid)
+ AUTO PARTITION BY RANGE(date_trunc(dt, 'day')) ()
+ DISTRIBUTED BY HASH(logid) BUCKETS AUTO
+ PROPERTIES (
+ "replication_num" = "1",
+ "estimate_partition_size" = "30G",
+ "enable_unique_key_merge_on_write" = "true"
+ )
+ """
+
+ sql "DROP TABLE IF EXISTS autobucket_min_history"
+ sql "CREATE TABLE autobucket_min_history LIKE autobucket_min_one"
+ sql """INSERT INTO autobucket_min_history VALUES ('2026-10-20', 1,
CAST('{"v":1}' AS VARIANT))"""
+
+ sql "DROP TABLE IF EXISTS autobucket_min_fixed"
+ sql """
+ CREATE TABLE autobucket_min_fixed (
+ dt DATETIMEV2(0) NOT NULL,
+ logid BIGINT NOT NULL
+ ) DUPLICATE KEY(dt, logid)
+ AUTO PARTITION BY RANGE(date_trunc(dt, 'day')) ()
+ DISTRIBUTED BY HASH(logid) BUCKETS 2
+ PROPERTIES ("replication_num" = "1")
+ """
+
+ // Defaults and the existing one-bucket partition were created before
raising the minimum.
+ setFeConfig("autobucket_min_buckets", "4")
+ sql """INSERT INTO autobucket_min_one VALUES ('2026-10-21', 1,
CAST('{"v":1}' AS VARIANT))"""
+ sql """INSERT INTO autobucket_min_two VALUES ('2026-10-21', 1,
CAST('{"v":1}' AS VARIANT))"""
+ sql """INSERT INTO autobucket_min_history VALUES ('2026-10-21', 2,
CAST('{"v":2}' AS VARIANT))"""
+ sql "INSERT INTO autobucket_min_fixed VALUES ('2026-10-21', 1)"
+
+ // New AUTO partitions respect the new minimum; existing and
fixed-bucket partitions retain their counts.
+ order_qt_buckets """
+ SELECT TABLE_NAME, PARTITION_NAME, BUCKET_NUM
+ FROM information_schema.partitions
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME IN ('autobucket_min_one', 'autobucket_min_two',
+ 'autobucket_min_history', 'autobucket_min_fixed')
+ """
+
+ // The dynamic partition scheduler uses the same minimum, including
its first execution.
+ sql "DROP TABLE IF EXISTS autobucket_min_dynamic"
+ sql """
+ CREATE TABLE autobucket_min_dynamic (
+ dt DATEV2 NOT NULL,
+ logid BIGINT NOT NULL
+ ) DUPLICATE KEY(dt, logid)
+ PARTITION BY RANGE(dt) ()
+ DISTRIBUTED BY HASH(logid) BUCKETS AUTO
+ PROPERTIES (
+ "replication_num" = "1",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.end" = "1",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "1"
+ )
+ """
+ order_qt_dynamic_buckets """
+ SELECT DISTINCT BUCKET_NUM FROM information_schema.partitions
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME =
'autobucket_min_dynamic'
+ """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
