This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/nested_column_prune by this
push:
new a2d2f3e4167 support push project in aggregate/join
a2d2f3e4167 is described below
commit a2d2f3e416774c02402129ac4345b1ba7b590e9e
Author: 924060929 <[email protected]>
AuthorDate: Wed Oct 29 19:24:51 2025 +0800
support push project in aggregate/join
---
.../org/apache/doris/datasource/FileScanNode.java | 2 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 11 ++-
.../nereids/rules/analysis/NormalizeAggregate.java | 3 +-
...hDownNestedColumn.java => PushDownProject.java} | 107 +++++++++++++++++---
.../apache/doris/planner/MaterializationNode.java | 2 +
.../org/apache/doris/planner/OlapScanNode.java | 2 +-
.../java/org/apache/doris/planner/PlanNode.java | 109 +++++++++++++++++++++
.../java/org/apache/doris/planner/ScanNode.java | 109 ---------------------
.../rules/rewrite/PruneNestedColumnTest.java | 27 +++++
10 files changed, 245 insertions(+), 131 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 08a757ed94e..15ed4a93118 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -172,7 +172,7 @@ public abstract class FileScanNode extends ExternalScanNode
{
}
output.append(String.format("numNodes=%s", numNodes)).append("\n");
- printNestedColumns(output, prefix);
+ printNestedColumns(output, prefix, getTupleDesc());
// pushdown agg
output.append(prefix).append(String.format("pushdown agg=%s",
pushDownAggNoGroupingOp));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 630202de254..90a664473e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -115,7 +115,7 @@ import
org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
-import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
+import org.apache.doris.nereids.rules.rewrite.PushDownProject;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -167,7 +167,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
- new PushDownNestedColumn(),
+ new PushDownProject(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 5d4143fbb02..f0909402cf5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -196,11 +196,12 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
// Pushdown nested column project
- PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE),
- PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
-
PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_AGGREGATE(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_UNION(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_WINDOW(RuleTypeClass.REWRITE),
+ PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
+ //
PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
index fd5ae6931d9..8af7236860c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -175,7 +176,7 @@ public class NormalizeAggregate implements
RewriteRuleFactory, NormalizeToSlot {
if (arg instanceof Literal) {
continue;
}
- if (arg.containsType(SubqueryExpr.class,
WindowExpression.class)) {
+ if (arg.containsType(SubqueryExpr.class,
WindowExpression.class, PreferPushDownProject.class)) {
needPushDownSelfExprs.add(arg);
} else {
needPushDownInputs.add(arg);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
similarity index 71%
rename from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
rename to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
index 6ef42183c7b..3a307d26ec8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -50,24 +51,27 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
-/** PushDownNestedColumnThroughJoin */
-public class PushDownNestedColumn implements RewriteRuleFactory {
+/** push down project if the expression instance of PreferPushDownProject */
+public class PushDownProject implements RewriteRuleFactory, NormalizeToSlot {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
- logicalProject(logicalJoin()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build(
+ logicalJoin().thenApply(this::pushDownJoinExpressions)
),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build(
-
logicalProject(logicalWindow()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_JOIN.build(
+
logicalProject(logicalJoin()).thenApply(this::defaultPushDownProject)
),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build(
-
logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_WINDOW.build(
+
logicalProject(logicalWindow()).thenApply(this::defaultPushDownProject)
),
- //
RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
- //
logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown)
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_PARTITION_TOP_N.build(
+
logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDownProject)
+ ),
+ //
RuleType.PUSH_DOWN_PROJECT_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
+ //
logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDownProject)
// ),
- RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ RuleType.PUSH_DOWN_PROJECT_THROUGH_UNION.build(
logicalProject(
logicalUnion().when(u -> u.getQualifier() ==
Qualifier.ALL)
).thenApply(this::pushThroughUnion)
@@ -75,7 +79,86 @@ public class PushDownNestedColumn implements
RewriteRuleFactory {
);
}
- private <C extends LogicalPlan> Plan
defaultPushDown(MatchingContext<LogicalProject<C>> ctx) {
+ private Plan pushDownJoinExpressions(MatchingContext<LogicalJoin<Plan,
Plan>> ctx) {
+ LogicalJoin<Plan, Plan> join = ctx.root;
+ Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>>
rewriteHashJoinConjunctsResult
+ = pushDownProjectInExpressions(join,
join.getHashJoinConjuncts(), ctx.statementContext);
+ Optional<Pair<List<Expression>, Map<Integer, List<NamedExpression>>>>
rewriteOtherJoinConjunctsResult
+ = pushDownProjectInExpressions(join,
join.getOtherJoinConjuncts(), ctx.statementContext);
+ if (!rewriteHashJoinConjunctsResult.isPresent() &&
!rewriteOtherJoinConjunctsResult.isPresent()) {
+ return join;
+ }
+
+ List<Expression> newHashJoinConjuncts =
rewriteHashJoinConjunctsResult.isPresent()
+ ? rewriteHashJoinConjunctsResult.get().first :
join.getHashJoinConjuncts();
+ List<Expression> newOtherJoinConjuncts =
rewriteOtherJoinConjunctsResult.isPresent()
+ ? rewriteOtherJoinConjunctsResult.get().first :
join.getOtherJoinConjuncts();
+
+ List<List<NamedExpression>> pushedOutput = new ArrayList<>();
+ pushedOutput.add(new ArrayList<>(join.left().getOutput()));
+ pushedOutput.add(new ArrayList<>(join.right().getOutput()));
+
+ if (rewriteHashJoinConjunctsResult.isPresent()) {
+
pushedOutput.get(0).addAll(rewriteHashJoinConjunctsResult.get().second.get(0));
+
pushedOutput.get(1).addAll(rewriteHashJoinConjunctsResult.get().second.get(1));
+ }
+ if (rewriteOtherJoinConjunctsResult.isPresent()) {
+
pushedOutput.get(0).addAll(rewriteOtherJoinConjunctsResult.get().second.get(0));
+
pushedOutput.get(1).addAll(rewriteOtherJoinConjunctsResult.get().second.get(1));
+ }
+
+ Plan newLeft = join.left();
+ Plan newRight = join.right();
+ if (pushedOutput.get(0).size() != newLeft.getOutput().size()) {
+ newLeft = new LogicalProject<>(pushedOutput.get(0), newLeft);
+ }
+ if (pushedOutput.get(1).size() != newRight.getOutput().size()) {
+ newRight = new LogicalProject<>(pushedOutput.get(1), newRight);
+ }
+
+ return join.withJoinConjuncts(
+ newHashJoinConjuncts, newOtherJoinConjuncts,
+ join.getMarkJoinConjuncts(), join.getJoinReorderContext()
+ ).withChildren(newLeft, newRight);
+ }
+
+ // return:
+ // key: rewrite the PreferPushDownProject to slot
+ // value: the pushed down project outputs which contains the
Alias(PreferPushDownProject)
+ private Optional<Pair<List<Expression>, Map<Integer,
List<NamedExpression>>>> pushDownProjectInExpressions(
+ Plan plan, Collection<Expression> expressions, StatementContext
context) {
+
+ boolean changed = false;
+ Map<Integer, List<NamedExpression>> childIndexToPushedAlias = new
LinkedHashMap<>();
+ List<Expression> newExpressions = new ArrayList<>();
+ for (Expression expression : expressions) {
+ Expression newExpression = expression.rewriteDownShortCircuit(e ->
{
+ if (e instanceof PreferPushDownProject) {
+ List<Plan> children = plan.children();
+ for (int i = 0; i < children.size(); i++) {
+ Plan child = children.get(i);
+ if
(child.getOutputSet().containsAll(e.getInputSlots())) {
+ Alias alias = new Alias(context.getNextExprId(),
e);
+ Slot slot = alias.toSlot();
+ List<NamedExpression> namedExpressions
+ =
childIndexToPushedAlias.computeIfAbsent(i, k -> new ArrayList<>());
+ namedExpressions.add(alias);
+ return slot;
+ }
+ }
+ }
+ return e;
+ });
+ newExpressions.add(newExpression);
+ changed |= newExpression != expression;
+ }
+ if (changed) {
+ return Optional.of(Pair.of(newExpressions,
childIndexToPushedAlias));
+ }
+ return Optional.empty();
+ }
+
+ private <C extends LogicalPlan> Plan
defaultPushDownProject(MatchingContext<LogicalProject<C>> ctx) {
if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns)
{
return ctx.root;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
index 89918c014d0..29f6baf0e77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
@@ -127,6 +127,8 @@ public class MaterializationNode extends PlanNode {
output.append(detailPrefix).append("table_idxs:
").append(idxs).append("\n");
output.append(detailPrefix).append("row_ids:
").append(rowIds).append("\n");
output.append(detailPrefix).append("isTopMaterializeNode:
").append(isTopMaterializeNode).append("\n");
+ printNestedColumns(output, detailPrefix, outputTupleDesc);
+
return output.toString();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index bd7b006f3ea..4a882d62097 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1089,7 +1089,7 @@ public class OlapScanNode extends ScanNode {
getExplainString(rewrittenProjectList)).append("\n");
}
- printNestedColumns(output, prefix);
+ printNestedColumns(output, prefix, getTupleDesc());
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 382782c393a..ad5f0037a1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -32,11 +32,14 @@ import org.apache.doris.common.Id;
import org.apache.doris.common.Pair;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.planner.normalize.Normalizer;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.PlanStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TNormalizedPlanNode;
@@ -50,6 +53,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -928,4 +932,109 @@ public abstract class PlanNode extends TreeNode<PlanNode>
implements PlanStats {
}
return children.stream().anyMatch(PlanNode::hasSerialScanChildren);
}
+
+ protected void printNestedColumns(StringBuilder output, String prefix,
TupleDescriptor tupleDesc) {
+ boolean printNestedColumnsHeader = true;
+ for (SlotDescriptor slot : tupleDesc.getSlots()) {
+ String prunedType = null;
+ if (!slot.getType().equals(slot.getColumn().getType())) {
+ prunedType = slot.getType().toString();
+ }
+ String displayAllAccessPathsString = null;
+ if (slot.getDisplayAllAccessPaths() != null
+ && slot.getDisplayAllAccessPaths() != null
+ && !slot.getDisplayAllAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayAllAccessPathsString =
mergeIcebergAccessPathsWithId(
+ slot.getAllAccessPaths(),
+ slot.getDisplayAllAccessPaths()
+ );
+ } else {
+ displayAllAccessPathsString =
slot.getDisplayAllAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return
StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return
StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+ String displayPredicateAccessPathsString = null;
+ if (slot.getDisplayPredicateAccessPaths() != null
+ && slot.getDisplayPredicateAccessPaths() != null
+ && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayPredicateAccessPathsString =
mergeIcebergAccessPathsWithId(
+ slot.getPredicateAccessPaths(),
+ slot.getDisplayPredicateAccessPaths()
+ );
+ } else {
+ displayPredicateAccessPathsString =
slot.getPredicateAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return
StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return
StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+
+
+ if (prunedType == null
+ && displayAllAccessPathsString == null
+ && displayPredicateAccessPathsString == null) {
+ continue;
+ }
+
+ if (printNestedColumnsHeader) {
+ output.append(prefix).append("nested columns:\n");
+ printNestedColumnsHeader = false;
+ }
+ output.append(prefix).append("
").append(slot.getColumn().getName()).append(":\n");
+ output.append(prefix).append(" origin type:
").append(slot.getColumn().getType()).append("\n");
+ if (prunedType != null) {
+ output.append(prefix).append(" pruned type:
").append(prunedType).append("\n");
+ }
+ if (displayAllAccessPathsString != null) {
+ output.append(prefix).append(" all access paths: [")
+ .append(displayAllAccessPathsString).append("]\n");
+ }
+ if (displayPredicateAccessPathsString != null) {
+ output.append(prefix).append(" predicate access paths: [")
+
.append(displayPredicateAccessPathsString).append("]\n");
+ }
+ }
+ }
+
+ private String mergeIcebergAccessPathsWithId(
+ List<TColumnAccessPath> accessPaths, List<TColumnAccessPath>
displayAccessPaths) {
+ List<String> mergeDisplayAccessPaths = Lists.newArrayList();
+ for (int i = 0; i < displayAccessPaths.size(); i++) {
+ TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
+ TColumnAccessPath idAccessPath = accessPaths.get(i);
+ List<String> nameAccessPathStrings = displayAccessPath.type ==
TAccessPathType.DATA
+ ? displayAccessPath.data_access_path.path :
displayAccessPath.meta_access_path.path;
+ List<String> idAccessPathStrings = idAccessPath.type ==
TAccessPathType.DATA
+ ? idAccessPath.data_access_path.path :
idAccessPath.meta_access_path.path;
+
+ List<String> mergedPath = new ArrayList<>();
+ for (int j = 0; j < idAccessPathStrings.size(); j++) {
+ String name = nameAccessPathStrings.get(j);
+ String id = idAccessPathStrings.get(j);
+ if (name.equals(id)) {
+ mergedPath.add(name);
+ } else {
+ mergedPath.add(name + "(" + id + ")");
+ }
+ }
+ mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
+ }
+ return StringUtils.join(mergeDisplayAccessPaths, ", ");
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 465a11d152c..4a9a1945d9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -48,13 +48,10 @@ import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
-import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
-import org.apache.doris.thrift.TAccessPathType;
-import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRange;
@@ -69,7 +66,6 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import org.apache.commons.collections.map.CaseInsensitiveMap;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -561,111 +557,6 @@ public abstract class ScanNode extends PlanNode
implements SplitGenerator {
.addValue(super.debugString()).toString();
}
- protected void printNestedColumns(StringBuilder output, String prefix) {
- boolean printNestedColumnsHeader = true;
- for (SlotDescriptor slot : getTupleDesc().getSlots()) {
- String prunedType = null;
- if (!slot.getType().equals(slot.getColumn().getType())) {
- prunedType = slot.getType().toString();
- }
- String displayAllAccessPathsString = null;
- if (slot.getDisplayAllAccessPaths() != null
- && slot.getDisplayAllAccessPaths() != null
- && !slot.getDisplayAllAccessPaths().isEmpty()) {
- if (this instanceof IcebergScanNode) {
- displayAllAccessPathsString =
mergeIcebergAccessPathsWithId(
- slot.getAllAccessPaths(),
- slot.getDisplayAllAccessPaths()
- );
- } else {
- displayAllAccessPathsString =
slot.getDisplayAllAccessPaths()
- .stream()
- .map(a -> {
- if (a.type == TAccessPathType.DATA) {
- return
StringUtils.join(a.data_access_path.path, ".");
- } else {
- return
StringUtils.join(a.meta_access_path.path, ".");
- }
- })
- .collect(Collectors.joining(", "));
- }
- }
- String displayPredicateAccessPathsString = null;
- if (slot.getDisplayPredicateAccessPaths() != null
- && slot.getDisplayPredicateAccessPaths() != null
- && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
- if (this instanceof IcebergScanNode) {
- displayPredicateAccessPathsString =
mergeIcebergAccessPathsWithId(
- slot.getPredicateAccessPaths(),
- slot.getDisplayPredicateAccessPaths()
- );
- } else {
- displayPredicateAccessPathsString =
slot.getPredicateAccessPaths()
- .stream()
- .map(a -> {
- if (a.type == TAccessPathType.DATA) {
- return
StringUtils.join(a.data_access_path.path, ".");
- } else {
- return
StringUtils.join(a.meta_access_path.path, ".");
- }
- })
- .collect(Collectors.joining(", "));
- }
- }
-
-
- if (prunedType == null
- && displayAllAccessPathsString == null
- && displayPredicateAccessPathsString == null) {
- continue;
- }
-
- if (printNestedColumnsHeader) {
- output.append(prefix).append("nested columns:\n");
- printNestedColumnsHeader = false;
- }
- output.append(prefix).append("
").append(slot.getColumn().getName()).append(":\n");
- output.append(prefix).append(" origin type:
").append(slot.getColumn().getType()).append("\n");
- if (prunedType != null) {
- output.append(prefix).append(" pruned type:
").append(prunedType).append("\n");
- }
- if (displayAllAccessPathsString != null) {
- output.append(prefix).append(" all access paths: [")
- .append(displayAllAccessPathsString).append("]\n");
- }
- if (displayPredicateAccessPathsString != null) {
- output.append(prefix).append(" predicate access paths: [")
-
.append(displayPredicateAccessPathsString).append("]\n");
- }
- }
- }
-
- private String mergeIcebergAccessPathsWithId(
- List<TColumnAccessPath> accessPaths, List<TColumnAccessPath>
displayAccessPaths) {
- List<String> mergeDisplayAccessPaths = Lists.newArrayList();
- for (int i = 0; i < displayAccessPaths.size(); i++) {
- TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
- TColumnAccessPath idAccessPath = accessPaths.get(i);
- List<String> nameAccessPathStrings = displayAccessPath.type ==
TAccessPathType.DATA
- ? displayAccessPath.data_access_path.path :
displayAccessPath.meta_access_path.path;
- List<String> idAccessPathStrings = idAccessPath.type ==
TAccessPathType.DATA
- ? idAccessPath.data_access_path.path :
idAccessPath.meta_access_path.path;
-
- List<String> mergedPath = new ArrayList<>();
- for (int j = 0; j < idAccessPathStrings.size(); j++) {
- String name = nameAccessPathStrings.get(j);
- String id = idAccessPathStrings.get(j);
- if (name.equals(id)) {
- mergedPath.add(name);
- } else {
- mergedPath.add(name + "(" + id + ")");
- }
- }
- mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
- }
- return StringUtils.join(mergeDisplayAccessPaths, ", ");
- }
-
public List<TupleId> getOutputTupleIds() {
if (outputTupleDesc != null) {
return Lists.newArrayList(outputTupleDesc.getId());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 8e8d7c44d16..d2240a2727e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -428,6 +428,33 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
);
}
+ @Test
+ public void testAggregate() throws Exception {
+ assertColumn("select count(struct_element(s, 'city')) from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ assertColumns("select 100 from tbl t1 join tbl t2 on
struct_element(t1.s, 'city')=struct_element(t2.s, 'city')",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ )
+ )
+ );
+ }
+
@Test
public void testPushDownThroughWindow() {
PlanChecker.from(connectContext)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
