This is an automated email from the ASF dual-hosted git repository. lingmiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new e6da00bb26 [feature](nereides) support sort translator (#10678) e6da00bb26 is described below commit e6da00bb261ef6a04207336e318e64b2558193c3 Author: zhengshiJ <32082872+zhengs...@users.noreply.github.com> AuthorDate: Fri Jul 8 19:22:48 2022 +0800 [feature](nereides) support sort translator (#10678) Physical sort: * 1. Build sortInfo * There are two types of slotRef: * one is generated by the previous node, collectively called old. * the other is newly generated by the sort node, collectively called new. * Filling of sortInfo related data structures, * a. ordering use newSlotRef. * b. sortTupleSlotExprs use oldSlotRef. * 2. Create sortNode * 3. Create mergeFragment TODO: 1.Currently, columns that do not exist in select but exist in order by cannot be parsed. eg: select key from table order by value; 2.For the combination of Literal and slotRefrance in select, there is a problem with parsing, eg: select key ,(10-value) from table; --- .../java/org/apache/doris/analysis/SortInfo.java | 4 ++ .../org/apache/doris/nereids/NereidsPlanner.java | 6 +-- .../glue/translator/PhysicalPlanTranslator.java | 56 +++++++++++++++++++--- .../glue/translator/PlanTranslatorContext.java | 13 ++++- .../java/org/apache/doris/planner/PlanNode.java | 7 +++ .../java/org/apache/doris/planner/SortNode.java | 23 +++++++++ .../java/org/apache/doris/qe/StmtExecutor.java | 1 - 7 files changed, 98 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java index 05763e8d07..63394db4cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java @@ -141,6 +141,10 @@ public class SortInfo { this.sortTupleSlotExprs = sortTupleSlotExprs; } + public void setSortTupleDesc(TupleDescriptor tupleDesc) { + sortTupleDesc = tupleDesc; + } + public TupleDescriptor getSortTupleDescriptor() { return sortTupleDesc; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index bc7aed0bf1..23fafa518d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -25,7 +25,6 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Id; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; @@ -39,7 +38,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -104,8 +102,8 @@ public class NereidsPlanner extends Planner { outputCandidates.put(slotDescriptor.getId().asInt(), slotRef); } } - physicalPlan.getOutput().stream().map(Slot::getExprId) - .map(Id::asInt).forEach(i -> outputExprs.add(outputCandidates.get(i))); + physicalPlan.getOutput().stream() + .forEach(i -> outputExprs.add(planTranslatorContext.findExpr(i))); root.setOutputExprs(outputExprs); root.getPlanRoot().convertToVectoriezd(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 511ac61fb4..0dd6b9e859 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -204,6 +204,26 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl return planFragment; } + /** + * Physical sort: + * 1. Build sortInfo + * There are two types of slotRef: + * one is generated by the previous node, collectively called old. + * the other is newly generated by the sort node, collectively called new. + * Filling of sortInfo related data structures, + * a. ordering use newSlotRef. + * b. sortTupleSlotExprs use oldSlotRef. + * 2. Create sortNode + * 3. Create mergeFragment + * TODO: When the slotRef of sort is currently generated, + * it will be based on the expression in select and orderBy expression in to ensure the uniqueness of slotRef. + * But eg: + * select a+1 from table order by a+1; + * the expressions of the two are inconsistent. + * The former will perform an additional Alisa. + * Currently we cannot test whether this will have any effect. + * After a+1 can be parsed , reprocessing. + */ @Override public PlanFragment visitPhysicalSort(PhysicalUnaryPlan<PhysicalHeapSort, Plan> sort, PlanTranslatorContext context) { @@ -211,24 +231,35 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl PhysicalHeapSort physicalHeapSort = sort.getOperator(); long limit = physicalHeapSort.getLimit(); // TODO: need to discuss how to process field: SortNode::resolvedTupleExprs - List<Expr> execOrderingExprList = Lists.newArrayList(); + List<Expr> oldOrderingExprList = Lists.newArrayList(); List<Boolean> ascOrderList = Lists.newArrayList(); List<Boolean> nullsFirstParamList = Lists.newArrayList(); List<OrderKey> orderKeyList = physicalHeapSort.getOrderKeys(); + // 1.Get previous slotRef orderKeyList.forEach(k -> { - execOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); ascOrderList.add(k.isAsc()); nullsFirstParamList.add(k.isNullFirst()); }); - + List<Expr> sortTupleOutputList = new ArrayList<>(); List<Slot> outputList = sort.getOutput(); - TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, null); - SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); - + outputList.forEach(k -> { + sortTupleOutputList.add(ExpressionTranslator.translate(k, context)); + }); + // 2. Generate new Tuple + TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, context, null); + // 3. Get current slotRef + List<Expr> newOrderingExprList = Lists.newArrayList(); + orderKeyList.forEach(k -> { + newOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + }); + // 4. fill in SortInfo members + SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); PlanNode childNode = childFragment.getPlanRoot(); // TODO: notice topN SortNode sortNode = new SortNode(context.nextNodeId(), childNode, sortInfo, true, physicalHeapSort.hasLimit(), physicalHeapSort.getOffset()); + sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList); childFragment.addPlanRoot(sortNode); if (!childFragment.isPartitioned()) { return childFragment; @@ -362,6 +393,19 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl return tupleDescriptor; } + private TupleDescriptor generateTupleDesc(List<Slot> slotList, List<OrderKey> orderKeyList, + PlanTranslatorContext context, Table table) { + TupleDescriptor tupleDescriptor = context.generateTupleDesc(); + tupleDescriptor.setTable(table); + for (Slot slot : slotList) { + context.createSlotDesc(tupleDescriptor, (SlotReference) slot); + } + for (OrderKey orderKey : orderKeyList) { + context.createSlotDesc(tupleDescriptor, orderKey.getExpr()); + } + return tupleDescriptor; + } + private PlanFragment createParentFragment(PlanFragment childFragment, DataPartition parentPartition, PlanTranslatorContext ctx) { ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), childFragment.getPlanRoot(), false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index afde344669..cc3337bbd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -106,7 +106,7 @@ public class PlanTranslatorContext { * Create SlotDesc and add it to the mappings from expression to the stales epxr */ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference slotReference) { - SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc, slotReference.getExprId().asInt()); + SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc); Column column = slotReference.getColumn(); // Only the SlotDesc that in the tuple generated for scan node would have corresponding column. if (column != null) { @@ -118,6 +118,17 @@ public class PlanTranslatorContext { return slotDescriptor; } + /** + * Create slotDesc with Expression. + */ + public void createSlotDesc(TupleDescriptor tupleDesc, Expression expression) { + if (!expressionToExecExpr.containsKey(expression)) { + SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc); + slotDescriptor.setType(expression.getDataType().toCatalogDataType()); + this.addSlotRefMapping(expression, new SlotRef(slotDescriptor)); + } + } + public TupleDescriptor getTupleDesc(TupleId tupleId) { return descTable.getTupleDesc(tupleId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index c2dc140d90..074a52888f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -999,4 +999,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats { output.append("\n"); } } + + /** + * Supplement the information be needed for nodes generated by the new optimizer. + */ + public void finalizeForNereids() { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index db08f4eb48..232e703555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -22,11 +22,13 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprId; import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.SortInfo; +import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.statistics.StatisticalType; @@ -44,6 +46,7 @@ import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -264,4 +267,24 @@ public class SortNode extends PlanNode { Expr.getIds(resolvedTupleExprs, null, result); return new HashSet<>(result); } + + /** + * Supplement the information needed by be for the sort node. + */ + public void finalizeForNereids(TupleDescriptor tupleDescriptor, + List<Expr> outputList, List<Expr> orderingExpr) { + List<Expr> sortTupleSlotExprs = new ArrayList<>(); + sortTupleSlotExprs.addAll(outputList); + sortTupleSlotExprs.addAll(orderingExpr); + List<Expr> afterDeduplication = new ArrayList<>(); + Set<ExprId> exprIds = new HashSet<>(); + for (int i = 0; i < sortTupleSlotExprs.size(); i++) { + Expr expr = sortTupleSlotExprs.get(i); + if (!exprIds.contains(expr.getId())) { + afterDeduplication.add(expr); + } + } + info.setSortTupleDesc(tupleDescriptor); + info.setSortTupleSlotExprs(afterDeduplication); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 34321cee03..6478551678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -978,7 +978,6 @@ public class StmtExecutor implements ProfileWriter { return; } - MysqlChannel channel = context.getMysqlChannel(); boolean isOutfileQuery = queryStmt.hasOutFileClause(); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org