This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 9e0fccbc5d6 branch-4.0: [Fix](auto bucket) Enhance auto bucket
robustness calculation #53317 (#56767)
9e0fccbc5d6 is described below
commit 9e0fccbc5d602a3c9a8957a458975a914502e102
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Oct 11 16:09:32 2025 +0800
branch-4.0: [Fix](auto bucket) Enhance auto bucket robustness calculation
#53317 (#56767)
Cherry-picked from #53317
Co-authored-by: deardeng <[email protected]>
---
.../doris/clone/DynamicPartitionScheduler.java | 43 +++++++++++++++++-----
.../doris/catalog/DynamicPartitionTableTest.java | 2 +
2 files changed, 36 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 3e95ebac173..1a96280ea30 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -254,7 +254,7 @@ public class DynamicPartitionScheduler extends MasterDaemon
{
for (Partition hasDataPartition : hasDataPartitions) {
long partitionSize =
hasDataPartition.getDataSizeExcludeEmptyReplica(true);
- if (partitionSize == 0) {
+ if (partitionSize <= 0) {
sizeUnknownArray.add(partitionSize);
} else {
partitionSizeArray.add(partitionSize);
@@ -395,7 +395,11 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
int bucketsNum = ret.first;
int previousPartitionBucketsNum = ret.second;
if (olapTable.isAutoBucket()) {
- checkAutoBucketCalcNumIsValid(bucketsNum,
previousPartitionBucketsNum);
+ int afterCheckAndFixBucketNum =
checkAndFixAutoBucketCalcNumIsValid(bucketsNum,
+ previousPartitionBucketsNum, olapTable.getName(),
partitionName);
+ if (afterCheckAndFixBucketNum > 0) {
+ bucketsNum = afterCheckAndFixBucketNum;
+ }
}
if (distributionInfo.getType() ==
DistributionInfo.DistributionInfoType.HASH) {
HashDistributionInfo hashDistributionInfo =
(HashDistributionInfo) distributionInfo;
@@ -413,15 +417,36 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
return addPartitionClauses;
}
- private void checkAutoBucketCalcNumIsValid(int calcNum, int
previousPartitionBucketsNum) {
+ private int checkAndFixAutoBucketCalcNumIsValid(int
currentPartitionNumBuckets, int previousPartitionNumBuckets,
+ String tableName, String
partitionName) {
// previousPartitionBucketsNum == 0, some abnormal case, ignore it
- if (previousPartitionBucketsNum != 0
- && (calcNum > previousPartitionBucketsNum * (1 +
Config.autobucket_out_of_bounds_percent_threshold))
- || (calcNum < previousPartitionBucketsNum * (1 -
Config.autobucket_out_of_bounds_percent_threshold))) {
- LOG.warn("auto bucket calc num may be err, plz check. "
- + "calc bucket num {}, previous partition bucket num {},
percent {}",
- calcNum, previousPartitionBucketsNum,
Config.autobucket_out_of_bounds_percent_threshold);
+ if (currentPartitionNumBuckets != 0) {
+ // currentPartitionNumBuckets can be too big
+ if (currentPartitionNumBuckets
+ > previousPartitionNumBuckets * (1 +
Config.autobucket_out_of_bounds_percent_threshold)) {
+ LOG.warn("tabletName {}, partitionName {} auto bucket calc num
may be err, "
+ + "bigger than previous too much, plz check. "
+ + "calc bucket num {}, previous partition bucket num
{}, percent {}",
+ tableName, partitionName,
+ currentPartitionNumBuckets,
previousPartitionNumBuckets,
+ Config.autobucket_out_of_bounds_percent_threshold);
+ return currentPartitionNumBuckets;
+ }
+ // currentPartitionNumBuckets not too small.
+ // If it is too small, the program will intervene. use
previousPartitionNumBuckets
+ if (currentPartitionNumBuckets
+ < previousPartitionNumBuckets * (1 -
Config.autobucket_out_of_bounds_percent_threshold)) {
+ LOG.warn("tabletName {}, partitionName {} auto bucket calc num
may be err, "
+ + "smaller than previous too much, plz check. "
+ + "calc bucket num {}, previous partition bucket num
{}, percent {}",
+ tableName, partitionName,
+ currentPartitionNumBuckets,
previousPartitionNumBuckets,
+ Config.autobucket_out_of_bounds_percent_threshold);
+ return previousPartitionNumBuckets;
+ }
}
+ LOG.info("previousPartitionBucketsNum eq 0, check before log");
+ return -1;
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 2d19a4a3fc0..a4ee2797321 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -1794,6 +1794,7 @@ public class DynamicPartitionTableTest {
}
RebalancerTestUtil.updateReplicaDataSize(1, 1, 1);
+ Config.autobucket_out_of_bounds_percent_threshold = 0.99;
String alterStmt1 =
"alter table test.test_autobucket_dynamic_partition set
('dynamic_partition.end' = '2')";
ExceptionChecker.expectThrowsNoException(() -> alterTable(alterStmt1));
@@ -1804,6 +1805,7 @@ public class DynamicPartitionTableTest {
partitions.sort(Comparator.comparing(Partition::getId));
Assert.assertEquals(53, partitions.size());
Assert.assertEquals(1, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
+ Config.autobucket_out_of_bounds_percent_threshold = 0.5;
table.readLock();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
