This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9319d173dd7 [refactor](planner) filter empty partitions in a unified
location (#27190)
9319d173dd7 is described below
commit 9319d173dd75adfa0108cb27d20edfafdaa5fe9a
Author: walter <[email protected]>
AuthorDate: Fri Nov 17 23:58:21 2023 +0800
[refactor](planner) filter empty partitions in a unified location (#27190)
---
.../main/java/org/apache/doris/catalog/OlapTable.java | 12 ++++++++++++
.../nereids/rules/rewrite/PruneEmptyPartition.java | 6 +-----
.../java/org/apache/doris/planner/OlapScanNode.java | 17 ++---------------
3 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 661a7893b03..a1ca88bf7c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -965,6 +965,18 @@ public class OlapTable extends Table {
return partition;
}
+ // select the non-empty partition ids belonging to this table.
+ //
+ // ATTN: partitions not belonging to this table will be filtered.
+ public List<Long> selectNonEmptyPartitionIds(Collection<Long>
partitionIds) {
+ return partitionIds.stream()
+ .map(this::getPartition)
+ .filter(p -> p != null)
+ .filter(Partition::hasData)
+ .map(Partition::getId)
+ .collect(Collectors.toList());
+ }
+
public int getPartitionNum() {
return idToPartition.size();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
index 5b3154e29f2..a408ea95a2c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
@@ -22,8 +22,6 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
-import java.util.stream.Collectors;
-
/**
* Used to prune empty partition.
*/
@@ -34,9 +32,7 @@ public class PruneEmptyPartition extends
OneRewriteRuleFactory {
return logicalOlapScan().thenApply(ctx -> {
LogicalOlapScan scan = ctx.root;
OlapTable table = scan.getTable();
- return
scan.withSelectedPartitionIds(scan.getSelectedPartitionIds().stream()
- .filter(partitionId ->
table.getPartition(partitionId).hasData())
- .collect(Collectors.toList()));
+ return
scan.withSelectedPartitionIds(table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds()));
}).toRule(RuleType.PRUNE_EMPTY_PARTITION);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 1164e84f7ea..6208d7f39ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -873,22 +873,9 @@ public class OlapScanNode extends ScanNode {
if (partitionInfo.getType() == PartitionType.RANGE ||
partitionInfo.getType() == PartitionType.LIST) {
selectedPartitionIds = partitionPrune(partitionInfo,
partitionNames);
} else {
- selectedPartitionIds = null;
- }
-
- if (selectedPartitionIds == null) {
- selectedPartitionIds = Lists.newArrayList();
- for (Partition partition : olapTable.getPartitions()) {
- if (!partition.hasData()) {
- continue;
- }
- selectedPartitionIds.add(partition.getId());
- }
- } else {
- selectedPartitionIds = selectedPartitionIds.stream()
- .filter(id -> olapTable.getPartition(id).hasData())
- .collect(Collectors.toList());
+ selectedPartitionIds = olapTable.getPartitionIds();
}
+ selectedPartitionIds =
olapTable.selectNonEmptyPartitionIds(selectedPartitionIds);
selectedPartitionNum = selectedPartitionIds.size();
for (long id : selectedPartitionIds) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
