This is an automated email from the ASF dual-hosted git repository. huajianlan pushed a commit to branch nested_column_prune in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9efa81907b5b7e546c5fa268bd14b9712f007a42 Author: 924060929 <[email protected]> AuthorDate: Fri Oct 31 12:36:22 2025 +0800 should not prune project nested column if the bottom table can not prune, e.g. hive(textfile) --- .../nereids/rules/rewrite/NestedColumnPruning.java | 3 +- .../nereids/rules/rewrite/SlotTypeReplacer.java | 53 ++++++++++++++++++++-- .../org/apache/doris/nereids/trees/TreeNode.java | 2 +- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index de22e9d2101..b03f183b115 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -92,8 +92,7 @@ public class NestedColumnPruning implements CustomRewriter { AccessPathInfo accessPathInfo = kv.getValue(); slotIdToPruneType.put(slotId, accessPathInfo); } - SlotTypeReplacer typeReplacer = new SlotTypeReplacer(slotIdToPruneType); - return plan.accept(typeReplacer, null); + return new SlotTypeReplacer(slotIdToPruneType, plan).replace(); } return plan; } catch (Throwable t) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index 9ce5de48061..9f6a02391eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -77,9 +77,11 @@ import com.google.common.collect.ImmutableMultimap.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.Collection; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -89,10 +91,32 @@ import java.util.Set; /** SlotTypeReplacer */ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> { + private Plan plan; private Map<Integer, AccessPathInfo> replacedDataTypes; + private IdentityHashMap<Plan, Void> shouldPrunePlans = new IdentityHashMap(); - public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes) { + public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes, Plan plan) { this.replacedDataTypes = Maps.newLinkedHashMap(bottomReplacedDataTypes); + this.plan = plan; + } + + /** replace */ + public Plan replace() { + Set<Integer> shouldReplaceSlots = Sets.newLinkedHashSet(); + plan.foreachUp(p -> { + if (p instanceof LogicalTVFRelation) { + TableValuedFunction function = ((LogicalTVFRelation) p).getFunction(); + tryRecordReplaceSlots((Plan) p, function, shouldReplaceSlots); + } else { + tryRecordReplaceSlots((Plan) p, p, shouldReplaceSlots); + } + }); + replacedDataTypes.keySet().retainAll(shouldReplaceSlots); + + if (replacedDataTypes.isEmpty()) { + return plan; + } + return plan.accept(this, null); } @Override @@ -376,7 +400,7 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> { @Override public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) { - if (!fileScan.supportPruneNestedColumn()) { + if (!shouldPrunePlans.containsKey(fileScan)) { return fileScan; } @@ -413,9 +437,7 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> { @Override public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) { - TableValuedFunction function = tvfRelation.getFunction(); - if (!(function instanceof SupportPruneNestedColumn) - || !((SupportPruneNestedColumn) function).supportPruneNestedColumn()) { + if (!shouldPrunePlans.containsKey(tvfRelation)) { return tvfRelation; } Pair<Boolean, List<Slot>> replaced @@ -428,6 +450,9 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> { @Override public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) { + if (!shouldPrunePlans.containsKey(olapScan)) { + return olapScan; + } Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true); if (replaced.first) { return olapScan.withPrunedTypeSlots(replaced.second); @@ -682,4 +707,22 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> { } } } + + private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set<Integer> shouldReplaceSlots) { + if (checkObj instanceof SupportPruneNestedColumn + && ((SupportPruneNestedColumn) checkObj).supportPruneNestedColumn()) { + List<Slot> output = plan.getOutput(); + boolean shouldPrune = false; + for (Slot slot : output) { + int slotId = slot.getExprId().asInt(); + if (replacedDataTypes.containsKey(slotId)) { + shouldReplaceSlots.add(slotId); + shouldPrune = true; + } + } + if (shouldPrune) { + shouldPrunePlans.put(plan, null); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index a5e3ee9c5e1..2e59b283f93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -236,7 +236,7 @@ public interface TreeNode<NODE_TYPE extends TreeNode<NODE_TYPE>> { default void foreachUp(Consumer<TreeNode<NODE_TYPE>> func) { for (NODE_TYPE child : children()) { - child.foreach(func); + child.foreachUp(func); } func.accept(this); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
