This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch 4.1_performance
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/4.1_performance by this push:
new e432090810b Adjust decompose_repeat (#66490)
e432090810b is described below
commit e432090810ba9d6e97739c8d72c59838f40789a5
Author: feiniaofeiafei <[email protected]>
AuthorDate: Thu Aug 6 14:15:20 2026 +0800
Adjust decompose_repeat (#66490)
when there is no hotvalue stats, still choose one shuffle key in
decompose_repeat
---
.../rules/rewrite/DecomposeRepeatWithPreAggregation.java | 13 ++++++++++---
.../org/apache/doris/statistics/util/StatisticsUtil.java | 7 ++++---
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
index 4f6fa967f5f..ef40c53e75f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
@@ -97,6 +97,7 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
private static final Set<Class<? extends AggregateFunction>>
SUPPORT_AGG_FUNCTIONS =
ImmutableSet.of(Sum.class, Sum0.class, Min.class, Max.class,
AnyValue.class, Count.class);
private static final int DECOMPOSE_REPEAT_THRESHOLD = 3;
+ private static final int BALANCE_MULTIPLIER = 128;
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
@@ -546,11 +547,10 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
}
for (Expression candidate : candidates) {
ColumnStatistic columnStatistic =
inputStats.findColumnStatistics(candidate);
- if (columnStatistic == null || columnStatistic.isUnKnown() ||
columnStatistic.hotValues == null) {
+ if (columnStatistic == null || columnStatistic.isUnKnown()) {
continue;
}
- if (StatisticsUtil.isBalanced(columnStatistic, totalInstanceNum,
- ShuffleKeyPruneUtils.shuffleKeyHotValueThreshold,
inputStats.getRowCount())) {
+ if (isBalanced(columnStatistic, totalInstanceNum,
inputStats.getRowCount())) {
return Optional.of(candidate);
}
}
@@ -620,4 +620,11 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
return repeat.withNormalizedExpr(replacedNewGroupingSets,
replacedRepeatOutputs,
repeat.getGroupingId().get(), child);
}
+
+ private static boolean isBalanced(ColumnStatistic columnStatistic, int
instanceNum, double rowCount) {
+ double ndv = columnStatistic.ndv;
+ return ndv > instanceNum * BALANCE_MULTIPLIER
+ && !StatisticsUtil.hasSignificantHotValues(columnStatistic,
+ ShuffleKeyPruneUtils.shuffleKeyHotValueThreshold, rowCount,
false);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index c59992082ad..0bc6ea8a699 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -1402,10 +1402,11 @@ public class StatisticsUtil {
* Used by shuffle key prune and skew detection rules.
* Returns false when hotValues is null (not collected) or empty
(collected but no hot values).
*/
- public static boolean hasSignificantHotValues(ColumnStatistic
columnStatistic, double minRatio, double rowCount) {
+ public static boolean hasSignificantHotValues(ColumnStatistic
columnStatistic, double minRatio, double rowCount,
+ boolean strictWhenHotValueUnknow) {
Map<Literal, Float> hotValues = columnStatistic.getHotValues();
if (hotValues == null) {
- return true;
+ return strictWhenHotValueUnknow;
}
return columnStatistic.numNulls / rowCount > minRatio
|| hotValues.values().stream().anyMatch(ratio -> ratio >=
minRatio);
@@ -1415,6 +1416,6 @@ public class StatisticsUtil {
double rowCount) {
double ndv = columnStatistic.ndv;
return ndv > instanceNum *
AggregateUtils.NDV_INSTANCE_BALANCE_MULTIPLIER
- && !hasSignificantHotValues(columnStatistic, minRatio,
rowCount);
+ && !hasSignificantHotValues(columnStatistic, minRatio,
rowCount, true);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
