This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/nested_column_prune by this
push:
new c3db83ed54f should not prune project nested column if the bottom table
can not prune, e.g. hive(textfile)
c3db83ed54f is described below
commit c3db83ed54f4e218dbbecf5b41a797fce740c847
Author: 924060929 <[email protected]>
AuthorDate: Fri Oct 31 12:36:22 2025 +0800
should not prune project nested column if the bottom table can not prune,
e.g. hive(textfile)
---
.../nereids/rules/rewrite/NestedColumnPruning.java | 3 +-
.../nereids/rules/rewrite/SlotTypeReplacer.java | 53 ++++++++++++++++++++--
.../org/apache/doris/nereids/trees/TreeNode.java | 2 +-
3 files changed, 50 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index de22e9d2101..b03f183b115 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -92,8 +92,7 @@ public class NestedColumnPruning implements CustomRewriter {
AccessPathInfo accessPathInfo = kv.getValue();
slotIdToPruneType.put(slotId, accessPathInfo);
}
- SlotTypeReplacer typeReplacer = new
SlotTypeReplacer(slotIdToPruneType);
- return plan.accept(typeReplacer, null);
+ return new SlotTypeReplacer(slotIdToPruneType, plan).replace();
}
return plan;
} catch (Throwable t) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index 9ce5de48061..9f6a02391eb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -77,9 +77,11 @@ import com.google.common.collect.ImmutableMultimap.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -89,10 +91,32 @@ import java.util.Set;
/** SlotTypeReplacer */
public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
+ private Plan plan;
private Map<Integer, AccessPathInfo> replacedDataTypes;
+ private IdentityHashMap<Plan, Void> shouldPrunePlans = new
IdentityHashMap();
- public SlotTypeReplacer(Map<Integer, AccessPathInfo>
bottomReplacedDataTypes) {
+ public SlotTypeReplacer(Map<Integer, AccessPathInfo>
bottomReplacedDataTypes, Plan plan) {
this.replacedDataTypes =
Maps.newLinkedHashMap(bottomReplacedDataTypes);
+ this.plan = plan;
+ }
+
+ /** replace */
+ public Plan replace() {
+ Set<Integer> shouldReplaceSlots = Sets.newLinkedHashSet();
+ plan.foreachUp(p -> {
+ if (p instanceof LogicalTVFRelation) {
+ TableValuedFunction function = ((LogicalTVFRelation)
p).getFunction();
+ tryRecordReplaceSlots((Plan) p, function, shouldReplaceSlots);
+ } else {
+ tryRecordReplaceSlots((Plan) p, p, shouldReplaceSlots);
+ }
+ });
+ replacedDataTypes.keySet().retainAll(shouldReplaceSlots);
+
+ if (replacedDataTypes.isEmpty()) {
+ return plan;
+ }
+ return plan.accept(this, null);
}
@Override
@@ -376,7 +400,7 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
- if (!fileScan.supportPruneNestedColumn()) {
+ if (!shouldPrunePlans.containsKey(fileScan)) {
return fileScan;
}
@@ -413,9 +437,7 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void
context) {
- TableValuedFunction function = tvfRelation.getFunction();
- if (!(function instanceof SupportPruneNestedColumn)
- || !((SupportPruneNestedColumn)
function).supportPruneNestedColumn()) {
+ if (!shouldPrunePlans.containsKey(tvfRelation)) {
return tvfRelation;
}
Pair<Boolean, List<Slot>> replaced
@@ -428,6 +450,9 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
+ if (!shouldPrunePlans.containsKey(olapScan)) {
+ return olapScan;
+ }
Pair<Boolean, List<Slot>> replaced =
replaceExpressions(olapScan.getOutput(), false, true);
if (replaced.first) {
return olapScan.withPrunedTypeSlots(replaced.second);
@@ -682,4 +707,22 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
}
}
}
+
+ private void tryRecordReplaceSlots(Plan plan, Object checkObj,
Set<Integer> shouldReplaceSlots) {
+ if (checkObj instanceof SupportPruneNestedColumn
+ && ((SupportPruneNestedColumn)
checkObj).supportPruneNestedColumn()) {
+ List<Slot> output = plan.getOutput();
+ boolean shouldPrune = false;
+ for (Slot slot : output) {
+ int slotId = slot.getExprId().asInt();
+ if (replacedDataTypes.containsKey(slotId)) {
+ shouldReplaceSlots.add(slotId);
+ shouldPrune = true;
+ }
+ }
+ if (shouldPrune) {
+ shouldPrunePlans.put(plan, null);
+ }
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
index a5e3ee9c5e1..2e59b283f93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -236,7 +236,7 @@ public interface TreeNode<NODE_TYPE extends
TreeNode<NODE_TYPE>> {
default void foreachUp(Consumer<TreeNode<NODE_TYPE>> func) {
for (NODE_TYPE child : children()) {
- child.foreach(func);
+ child.foreachUp(func);
}
func.accept(this);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
