This is an automated email from the ASF dual-hosted git repository. huajianlan pushed a commit to branch nested_column_prune in repository https://gitbox.apache.org/repos/asf/doris.git
commit bbc61b4cdab71e64e7b27376eb8b7ab3cbc4925d Author: 924060929 <[email protected]> AuthorDate: Wed Oct 29 19:24:51 2025 +0800 support push project in aggregate/join --- .../org/apache/doris/datasource/FileScanNode.java | 2 +- .../org/apache/doris/nereids/rules/RuleSet.java | 4 +- .../org/apache/doris/nereids/rules/RuleType.java | 11 ++- .../nereids/rules/analysis/NormalizeAggregate.java | 3 +- ...hDownNestedColumn.java => PushDownProject.java} | 107 +++++++++++++++++--- .../apache/doris/planner/MaterializationNode.java | 2 + .../org/apache/doris/planner/OlapScanNode.java | 2 +- .../java/org/apache/doris/planner/PlanNode.java | 109 +++++++++++++++++++++ .../java/org/apache/doris/planner/ScanNode.java | 109 --------------------- .../rules/rewrite/PruneNestedColumnTest.java | 27 +++++ 10 files changed, 245 insertions(+), 131 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index f1205069e7c..46c69247bf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -186,7 +186,7 @@ public abstract class FileScanNode extends ExternalScanNode { } output.append(String.format("numNodes=%s", numNodes)).append("\n"); - printNestedColumns(output, prefix); + printNestedColumns(output, prefix, getTupleDesc()); // pushdown agg output.append(prefix).append(String.format("pushdown agg=%s", pushDownAggNoGroupingOp)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index a2268581a47..3d490930b97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -115,7 +115,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; -import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn; +import org.apache.doris.nereids.rules.rewrite.PushDownProject; import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit; import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin; @@ -167,7 +167,7 @@ public class RuleSet { new PushDownFilterThroughSetOperation(), new PushDownFilterThroughGenerate(), new PushDownProjectThroughLimit(), - new PushDownNestedColumn(), + new PushDownProject(), new EliminateOuterJoin(), new MergeProjectable(), new MergeFilters(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 70bedd1b85b..0e187084b4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -196,11 +196,12 @@ public enum RuleType { PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE), PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE), // Pushdown nested column project - PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE), - PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE), - PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE), - PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE), - PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE), + PUSH_DOWN_NESTED_COLUMN_THROUGH_AGGREGATE(RuleTypeClass.REWRITE), + PUSH_DOWN_PROJECT_THROUGH_JOIN(RuleTypeClass.REWRITE), + PUSH_DOWN_PROJECT_THROUGH_UNION(RuleTypeClass.REWRITE), + PUSH_DOWN_PROJECT_THROUGH_WINDOW(RuleTypeClass.REWRITE), + PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE), + // PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE), // Pushdown filter PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE), PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java index fd5ae6931d9..8af7236860c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.PreferPushDownProject; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -175,7 +176,7 @@ public class NormalizeAggregate implements RewriteRuleFactory, NormalizeToSlot { if (arg instanceof Literal) { continue; } - if (arg.containsType(SubqueryExpr.class, WindowExpression.class)) { + if (arg.containsType(SubqueryExpr.class, WindowExpression.class, PreferPushDownProject.class)) { needPushDownSelfExprs.add(arg); } else { needPushDownInputs.add(arg); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java similarity index 71% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java index 6ef42183c7b..3a307d26ec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -50,24 +51,27 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; -/** PushDownNestedColumnThroughJoin */ -public class PushDownNestedColumn implements RewriteRuleFactory { +/** push down project if the expression instance of PreferPushDownProject */ +public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot { @Override public List<Rule> buildRules() { return ImmutableList.of( - RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build( - logicalProject(logicalJoin()).thenApply(this::defaultPushDown) + RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build( + logicalJoin().thenApply(this::pushDownJoinExpressions) ), - RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build( - logicalProject(logicalWindow()).thenApply(this::defaultPushDown) + RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build( + logicalProject(logicalJoin()).thenApply(this::defaultPushDownProject) ), - RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build( - logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown) + RuleType.PUSH_DOWN_PROJECT_THROUGH_WINDOW.build( + logicalProject(logicalWindow()).thenApply(this::defaultPushDownProject) ), - // RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build( - // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown) + RuleType.PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N.build( + logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDownProject) + ), + // RuleType.PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N.build( + // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDownProject) // ), - RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build( + RuleType.PUSH_DOWN_PROJECT_THROUGH_UNION.build( logicalProject( logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL) ).thenApply(this::pushThroughUnion) @@ -75,7 +79,86 @@ public class PushDownNestedColumn implements RewriteRuleFactory { ); } - private <C extends LogicalPlan> Plan defaultPushDown(MatchingContext<LogicalProject<C>> ctx) { + private Plan pushDownJoinExpressions(MatchingContext<LogicalJoin<Plan, Plan>> ctx) { + LogicalJoin<Plan, Plan> join = ctx.root; + Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> rewriteHashJoinConjunctsResult + = pushDownProjectInExpressions(join, join.getHashJoinConjuncts(), ctx.statementContext); + Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> rewriteOtherJoinConjunctsResult + = pushDownProjectInExpressions(join, join.getOtherJoinConjuncts(), ctx.statementContext); + if (!rewriteHashJoinConjunctsResult.isPresent() && !rewriteOtherJoinConjunctsResult.isPresent()) { + return join; + } + + List<Expression> newHashJoinConjuncts = rewriteHashJoinConjunctsResult.isPresent() + ? rewriteHashJoinConjunctsResult.get().first : join.getHashJoinConjuncts(); + List<Expression> newOtherJoinConjuncts = rewriteOtherJoinConjunctsResult.isPresent() + ? rewriteOtherJoinConjunctsResult.get().first : join.getOtherJoinConjuncts(); + + List<List<NamedExpression>> pushedOutput = new ArrayList<>(); + pushedOutput.add(new ArrayList<>(join.left().getOutput())); + pushedOutput.add(new ArrayList<>(join.right().getOutput())); + + if (rewriteHashJoinConjunctsResult.isPresent()) { + pushedOutput.get(0).addAll(rewriteHashJoinConjunctsResult.get().second.get(0)); + pushedOutput.get(1).addAll(rewriteHashJoinConjunctsResult.get().second.get(1)); + } + if (rewriteOtherJoinConjunctsResult.isPresent()) { + pushedOutput.get(0).addAll(rewriteOtherJoinConjunctsResult.get().second.get(0)); + pushedOutput.get(1).addAll(rewriteOtherJoinConjunctsResult.get().second.get(1)); + } + + Plan newLeft = join.left(); + Plan newRight = join.right(); + if (pushedOutput.get(0).size() != newLeft.getOutput().size()) { + newLeft = new LogicalProject<>(pushedOutput.get(0), newLeft); + } + if (pushedOutput.get(1).size() != newRight.getOutput().size()) { + newRight = new LogicalProject<>(pushedOutput.get(1), newRight); + } + + return join.withJoinConjuncts( + newHashJoinConjuncts, newOtherJoinConjuncts, + join.getMarkJoinConjuncts(), join.getJoinReorderContext() + ).withChildren(newLeft, newRight); + } + + // return: + // key: rewrite the PreferPushDownProject to slot + // value: the pushed down project outputs which contains the Alias(PreferPushDownProject) + private Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>> pushDownProjectInExpressions( + Plan plan, Collection<Expression> expressions, StatementContext context) { + + boolean changed = false; + Map<Integer, List<NamedExpression>> childIndexToPushedAlias = new LinkedHashMap<>(); + List<Expression> newExpressions = new ArrayList<>(); + for (Expression expression : expressions) { + Expression newExpression = expression.rewriteDownShortCircuit(e -> { + if (e instanceof PreferPushDownProject) { + List<Plan> children = plan.children(); + for (int i = 0; i < children.size(); i++) { + Plan child = children.get(i); + if (child.getOutputSet().containsAll(e.getInputSlots())) { + Alias alias = new Alias(context.getNextExprId(), e); + Slot slot = alias.toSlot(); + List<NamedExpression> namedExpressions + = childIndexToPushedAlias.computeIfAbsent(i, k -> new ArrayList<>()); + namedExpressions.add(alias); + return slot; + } + } + } + return e; + }); + newExpressions.add(newExpression); + changed |= newExpression != expression; + } + if (changed) { + return Optional.of(Pair.of(newExpressions, childIndexToPushedAlias)); + } + return Optional.empty(); + } + + private <C extends LogicalPlan> Plan defaultPushDownProject(MatchingContext<LogicalProject<C>> ctx) { if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) { return ctx.root; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java index 89918c014d0..29f6baf0e77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java @@ -127,6 +127,8 @@ public class MaterializationNode extends PlanNode { output.append(detailPrefix).append("table_idxs: ").append(idxs).append("\n"); output.append(detailPrefix).append("row_ids: ").append(rowIds).append("\n"); output.append(detailPrefix).append("isTopMaterializeNode: ").append(isTopMaterializeNode).append("\n"); + printNestedColumns(output, detailPrefix, outputTupleDesc); + return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index f9c6ed0b237..658bc08add9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1089,7 +1089,7 @@ public class OlapScanNode extends ScanNode { getExplainString(rewrittenProjectList)).append("\n"); } - printNestedColumns(output, prefix); + printNestedColumns(output, prefix, getTupleDesc()); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 382782c393a..ad5f0037a1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -32,11 +32,14 @@ import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.planner.normalize.Normalizer; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsDeriveResult; +import org.apache.doris.thrift.TAccessPathType; +import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TNormalizedPlanNode; @@ -50,6 +53,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -928,4 +932,109 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats { } return children.stream().anyMatch(PlanNode::hasSerialScanChildren); } + + protected void printNestedColumns(StringBuilder output, String prefix, TupleDescriptor tupleDesc) { + boolean printNestedColumnsHeader = true; + for (SlotDescriptor slot : tupleDesc.getSlots()) { + String prunedType = null; + if (!slot.getType().equals(slot.getColumn().getType())) { + prunedType = slot.getType().toString(); + } + String displayAllAccessPathsString = null; + if (slot.getDisplayAllAccessPaths() != null + && slot.getDisplayAllAccessPaths() != null + && !slot.getDisplayAllAccessPaths().isEmpty()) { + if (this instanceof IcebergScanNode) { + displayAllAccessPathsString = mergeIcebergAccessPathsWithId( + slot.getAllAccessPaths(), + slot.getDisplayAllAccessPaths() + ); + } else { + displayAllAccessPathsString = slot.getDisplayAllAccessPaths() + .stream() + .map(a -> { + if (a.type == TAccessPathType.DATA) { + return StringUtils.join(a.data_access_path.path, "."); + } else { + return StringUtils.join(a.meta_access_path.path, "."); + } + }) + .collect(Collectors.joining(", ")); + } + } + String displayPredicateAccessPathsString = null; + if (slot.getDisplayPredicateAccessPaths() != null + && slot.getDisplayPredicateAccessPaths() != null + && !slot.getDisplayPredicateAccessPaths().isEmpty()) { + if (this instanceof IcebergScanNode) { + displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId( + slot.getPredicateAccessPaths(), + slot.getDisplayPredicateAccessPaths() + ); + } else { + displayPredicateAccessPathsString = slot.getPredicateAccessPaths() + .stream() + .map(a -> { + if (a.type == TAccessPathType.DATA) { + return StringUtils.join(a.data_access_path.path, "."); + } else { + return StringUtils.join(a.meta_access_path.path, "."); + } + }) + .collect(Collectors.joining(", ")); + } + } + + + if (prunedType == null + && displayAllAccessPathsString == null + && displayPredicateAccessPathsString == null) { + continue; + } + + if (printNestedColumnsHeader) { + output.append(prefix).append("nested columns:\n"); + printNestedColumnsHeader = false; + } + output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n"); + output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n"); + if (prunedType != null) { + output.append(prefix).append(" pruned type: ").append(prunedType).append("\n"); + } + if (displayAllAccessPathsString != null) { + output.append(prefix).append(" all access paths: [") + .append(displayAllAccessPathsString).append("]\n"); + } + if (displayPredicateAccessPathsString != null) { + output.append(prefix).append(" predicate access paths: [") + .append(displayPredicateAccessPathsString).append("]\n"); + } + } + } + + private String mergeIcebergAccessPathsWithId( + List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) { + List<String> mergeDisplayAccessPaths = Lists.newArrayList(); + for (int i = 0; i < displayAccessPaths.size(); i++) { + TColumnAccessPath displayAccessPath = displayAccessPaths.get(i); + TColumnAccessPath idAccessPath = accessPaths.get(i); + List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA + ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path; + List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA + ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path; + + List<String> mergedPath = new ArrayList<>(); + for (int j = 0; j < idAccessPathStrings.size(); j++) { + String name = nameAccessPathStrings.get(j); + String id = idAccessPathStrings.get(j); + if (name.equals(id)) { + mergedPath.add(name); + } else { + mergedPath.add(name + "(" + id + ")"); + } + } + mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, ".")); + } + return StringUtils.join(mergeDisplayAccessPaths, ", "); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index ee8270fbd79..fee9b0f8afc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -47,13 +47,10 @@ import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.SplitAssignment; import org.apache.doris.datasource.SplitGenerator; import org.apache.doris.datasource.SplitSource; -import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.system.Backend; -import org.apache.doris.thrift.TAccessPathType; -import org.apache.doris.thrift.TColumnAccessPath; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TScanRange; @@ -68,7 +65,6 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import org.apache.commons.collections.map.CaseInsensitiveMap; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -543,111 +539,6 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator { .addValue(super.debugString()).toString(); } - protected void printNestedColumns(StringBuilder output, String prefix) { - boolean printNestedColumnsHeader = true; - for (SlotDescriptor slot : getTupleDesc().getSlots()) { - String prunedType = null; - if (!slot.getType().equals(slot.getColumn().getType())) { - prunedType = slot.getType().toString(); - } - String displayAllAccessPathsString = null; - if (slot.getDisplayAllAccessPaths() != null - && slot.getDisplayAllAccessPaths() != null - && !slot.getDisplayAllAccessPaths().isEmpty()) { - if (this instanceof IcebergScanNode) { - displayAllAccessPathsString = mergeIcebergAccessPathsWithId( - slot.getAllAccessPaths(), - slot.getDisplayAllAccessPaths() - ); - } else { - displayAllAccessPathsString = slot.getDisplayAllAccessPaths() - .stream() - .map(a -> { - if (a.type == TAccessPathType.DATA) { - return StringUtils.join(a.data_access_path.path, "."); - } else { - return StringUtils.join(a.meta_access_path.path, "."); - } - }) - .collect(Collectors.joining(", ")); - } - } - String displayPredicateAccessPathsString = null; - if (slot.getDisplayPredicateAccessPaths() != null - && slot.getDisplayPredicateAccessPaths() != null - && !slot.getDisplayPredicateAccessPaths().isEmpty()) { - if (this instanceof IcebergScanNode) { - displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId( - slot.getPredicateAccessPaths(), - slot.getDisplayPredicateAccessPaths() - ); - } else { - displayPredicateAccessPathsString = slot.getPredicateAccessPaths() - .stream() - .map(a -> { - if (a.type == TAccessPathType.DATA) { - return StringUtils.join(a.data_access_path.path, "."); - } else { - return StringUtils.join(a.meta_access_path.path, "."); - } - }) - .collect(Collectors.joining(", ")); - } - } - - - if (prunedType == null - && displayAllAccessPathsString == null - && displayPredicateAccessPathsString == null) { - continue; - } - - if (printNestedColumnsHeader) { - output.append(prefix).append("nested columns:\n"); - printNestedColumnsHeader = false; - } - output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n"); - output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n"); - if (prunedType != null) { - output.append(prefix).append(" pruned type: ").append(prunedType).append("\n"); - } - if (displayAllAccessPathsString != null) { - output.append(prefix).append(" all access paths: [") - .append(displayAllAccessPathsString).append("]\n"); - } - if (displayPredicateAccessPathsString != null) { - output.append(prefix).append(" predicate access paths: [") - .append(displayPredicateAccessPathsString).append("]\n"); - } - } - } - - private String mergeIcebergAccessPathsWithId( - List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) { - List<String> mergeDisplayAccessPaths = Lists.newArrayList(); - for (int i = 0; i < displayAccessPaths.size(); i++) { - TColumnAccessPath displayAccessPath = displayAccessPaths.get(i); - TColumnAccessPath idAccessPath = accessPaths.get(i); - List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA - ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path; - List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA - ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path; - - List<String> mergedPath = new ArrayList<>(); - for (int j = 0; j < idAccessPathStrings.size(); j++) { - String name = nameAccessPathStrings.get(j); - String id = idAccessPathStrings.get(j); - if (name.equals(id)) { - mergedPath.add(name); - } else { - mergedPath.add(name + "(" + id + ")"); - } - } - mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, ".")); - } - return StringUtils.join(mergeDisplayAccessPaths, ", "); - } - public List<TupleId> getOutputTupleIds() { if (outputTupleDesc != null) { return Lists.newArrayList(outputTupleDesc.getId()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 8e8d7c44d16..d2240a2727e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -428,6 +428,33 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt ); } + @Test + public void testAggregate() throws Exception { + assertColumn("select count(struct_element(s, 'city')) from tbl", + "struct<city:text>", + ImmutableList.of(path("s", "city")), + ImmutableList.of() + ); + } + + @Test + public void testJoin() throws Exception { + assertColumns("select 100 from tbl t1 join tbl t2 on struct_element(t1.s, 'city')=struct_element(t2.s, 'city')", + ImmutableList.of( + Triple.of( + "struct<city:text>", + ImmutableList.of(path("s", "city")), + ImmutableList.of() + ), + Triple.of( + "struct<city:text>", + ImmutableList.of(path("s", "city")), + ImmutableList.of() + ) + ) + ); + } + @Test public void testPushDownThroughWindow() { PlanChecker.from(connectContext) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
