This is an automated email from the ASF dual-hosted git repository.

lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new e6da00bb26 [feature](nereides) support sort translator (#10678)
e6da00bb26 is described below

commit e6da00bb261ef6a04207336e318e64b2558193c3
Author: zhengshiJ <32082872+zhengs...@users.noreply.github.com>
AuthorDate: Fri Jul 8 19:22:48 2022 +0800

    [feature](nereides) support sort translator (#10678)
    
    Physical sort:
         * 1. Build sortInfo
         *    There are two types of slotRef:
         *    one is generated by the previous node, collectively called old.
         *    the other is newly generated by the sort node, collectively 
called new.
         *    Filling of sortInfo related data structures,
         *    a. ordering use newSlotRef.
         *    b. sortTupleSlotExprs use oldSlotRef.
         * 2. Create sortNode
         * 3. Create mergeFragment
    
    TODO:
    1.Currently, columns that do not exist in select but exist in order by 
cannot be parsed.
    eg: select key from table order by value;
    
    2.For the combination of Literal and slotRefrance in select, there is a 
problem with parsing,
    eg: select key ,(10-value) from table;
---
 .../java/org/apache/doris/analysis/SortInfo.java   |  4 ++
 .../org/apache/doris/nereids/NereidsPlanner.java   |  6 +--
 .../glue/translator/PhysicalPlanTranslator.java    | 56 +++++++++++++++++++---
 .../glue/translator/PlanTranslatorContext.java     | 13 ++++-
 .../java/org/apache/doris/planner/PlanNode.java    |  7 +++
 .../java/org/apache/doris/planner/SortNode.java    | 23 +++++++++
 .../java/org/apache/doris/qe/StmtExecutor.java     |  1 -
 7 files changed, 98 insertions(+), 12 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java 
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
index 05763e8d07..63394db4cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
@@ -141,6 +141,10 @@ public class SortInfo {
         this.sortTupleSlotExprs = sortTupleSlotExprs;
     }
 
+    public void setSortTupleDesc(TupleDescriptor tupleDesc) {
+        sortTupleDesc = tupleDesc;
+    }
+
     public TupleDescriptor getSortTupleDescriptor() {
         return sortTupleDesc;
     }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index bc7aed0bf1..23fafa518d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -25,7 +25,6 @@ import org.apache.doris.analysis.StatementBase;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.analysis.TupleId;
 import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Id;
 import org.apache.doris.common.UserException;
 import org.apache.doris.nereids.glue.LogicalPlanAdapter;
 import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
@@ -39,7 +38,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.memo.Memo;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -104,8 +102,8 @@ public class NereidsPlanner extends Planner {
                 outputCandidates.put(slotDescriptor.getId().asInt(), slotRef);
             }
         }
-        physicalPlan.getOutput().stream().map(Slot::getExprId)
-                .map(Id::asInt).forEach(i -> 
outputExprs.add(outputCandidates.get(i)));
+        physicalPlan.getOutput().stream()
+                .forEach(i -> 
outputExprs.add(planTranslatorContext.findExpr(i)));
         root.setOutputExprs(outputExprs);
         root.getPlanRoot().convertToVectoriezd();
 
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 511ac61fb4..0dd6b9e859 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -204,6 +204,26 @@ public class PhysicalPlanTranslator extends 
PlanOperatorVisitor<PlanFragment, Pl
         return planFragment;
     }
 
+    /**
+     * Physical sort:
+     * 1. Build sortInfo
+     *    There are two types of slotRef:
+     *    one is generated by the previous node, collectively called old.
+     *    the other is newly generated by the sort node, collectively called 
new.
+     *    Filling of sortInfo related data structures,
+     *    a. ordering use newSlotRef.
+     *    b. sortTupleSlotExprs use oldSlotRef.
+     * 2. Create sortNode
+     * 3. Create mergeFragment
+     * TODO: When the slotRef of sort is currently generated,
+     *       it will be based on the expression in select and orderBy 
expression in to ensure the uniqueness of slotRef.
+     *       But eg:
+     *       select a+1 from table order by a+1;
+     *       the expressions of the two are inconsistent.
+     *       The former will perform an additional Alisa.
+     *       Currently we cannot test whether this will have any effect.
+     *       After a+1 can be parsed , reprocessing.
+     */
     @Override
     public PlanFragment visitPhysicalSort(PhysicalUnaryPlan<PhysicalHeapSort, 
Plan> sort,
             PlanTranslatorContext context) {
@@ -211,24 +231,35 @@ public class PhysicalPlanTranslator extends 
PlanOperatorVisitor<PlanFragment, Pl
         PhysicalHeapSort physicalHeapSort = sort.getOperator();
         long limit = physicalHeapSort.getLimit();
         // TODO: need to discuss how to process field: 
SortNode::resolvedTupleExprs
-        List<Expr> execOrderingExprList = Lists.newArrayList();
+        List<Expr> oldOrderingExprList = Lists.newArrayList();
         List<Boolean> ascOrderList = Lists.newArrayList();
         List<Boolean> nullsFirstParamList = Lists.newArrayList();
         List<OrderKey> orderKeyList = physicalHeapSort.getOrderKeys();
+        // 1.Get previous slotRef
         orderKeyList.forEach(k -> {
-            
execOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
+            
oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
             ascOrderList.add(k.isAsc());
             nullsFirstParamList.add(k.isNullFirst());
         });
-
+        List<Expr> sortTupleOutputList = new ArrayList<>();
         List<Slot> outputList = sort.getOutput();
-        TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, 
null);
-        SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, 
nullsFirstParamList, tupleDesc);
-
+        outputList.forEach(k -> {
+            sortTupleOutputList.add(ExpressionTranslator.translate(k, 
context));
+        });
+        // 2. Generate new Tuple
+        TupleDescriptor tupleDesc = generateTupleDesc(outputList, 
orderKeyList, context, null);
+        // 3. Get current slotRef
+        List<Expr> newOrderingExprList = Lists.newArrayList();
+        orderKeyList.forEach(k -> {
+            
newOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
+        });
+        // 4. fill in SortInfo members
+        SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, 
nullsFirstParamList, tupleDesc);
         PlanNode childNode = childFragment.getPlanRoot();
         // TODO: notice topN
         SortNode sortNode = new SortNode(context.nextNodeId(), childNode, 
sortInfo, true,
                 physicalHeapSort.hasLimit(), physicalHeapSort.getOffset());
+        sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, 
oldOrderingExprList);
         childFragment.addPlanRoot(sortNode);
         if (!childFragment.isPartitioned()) {
             return childFragment;
@@ -362,6 +393,19 @@ public class PhysicalPlanTranslator extends 
PlanOperatorVisitor<PlanFragment, Pl
         return tupleDescriptor;
     }
 
+    private TupleDescriptor generateTupleDesc(List<Slot> slotList, 
List<OrderKey> orderKeyList,
+            PlanTranslatorContext context, Table table) {
+        TupleDescriptor tupleDescriptor = context.generateTupleDesc();
+        tupleDescriptor.setTable(table);
+        for (Slot slot : slotList) {
+            context.createSlotDesc(tupleDescriptor, (SlotReference) slot);
+        }
+        for (OrderKey orderKey : orderKeyList) {
+            context.createSlotDesc(tupleDescriptor, orderKey.getExpr());
+        }
+        return tupleDescriptor;
+    }
+
     private PlanFragment createParentFragment(PlanFragment childFragment, 
DataPartition parentPartition,
             PlanTranslatorContext ctx) {
         ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), 
childFragment.getPlanRoot(), false);
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index afde344669..cc3337bbd3 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -106,7 +106,7 @@ public class PlanTranslatorContext {
      * Create SlotDesc and add it to the mappings from expression to the 
stales epxr
      */
     public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, 
SlotReference slotReference) {
-        SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc, 
slotReference.getExprId().asInt());
+        SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
         Column column = slotReference.getColumn();
         // Only the SlotDesc that in the tuple generated for scan node would 
have corresponding column.
         if (column != null) {
@@ -118,6 +118,17 @@ public class PlanTranslatorContext {
         return slotDescriptor;
     }
 
+    /**
+     * Create slotDesc with Expression.
+     */
+    public void createSlotDesc(TupleDescriptor tupleDesc, Expression 
expression) {
+        if (!expressionToExecExpr.containsKey(expression)) {
+            SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
+            
slotDescriptor.setType(expression.getDataType().toCatalogDataType());
+            this.addSlotRefMapping(expression, new SlotRef(slotDescriptor));
+        }
+    }
+
     public TupleDescriptor getTupleDesc(TupleId tupleId) {
         return descTable.getTupleDesc(tupleId);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java 
b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index c2dc140d90..074a52888f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -999,4 +999,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> 
implements PlanStats {
             output.append("\n");
         }
     }
+
+    /**
+     * Supplement the information be needed for nodes generated by the new 
optimizer.
+     */
+    public void finalizeForNereids() {
+
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java 
b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
index db08f4eb48..232e703555 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
@@ -22,11 +22,13 @@ package org.apache.doris.planner;
 
 import org.apache.doris.analysis.Analyzer;
 import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.ExprId;
 import org.apache.doris.analysis.ExprSubstitutionMap;
 import org.apache.doris.analysis.SlotDescriptor;
 import org.apache.doris.analysis.SlotId;
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.analysis.SortInfo;
+import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.statistics.StatisticalType;
@@ -44,6 +46,7 @@ import com.google.common.collect.Lists;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -264,4 +267,24 @@ public class SortNode extends PlanNode {
         Expr.getIds(resolvedTupleExprs, null, result);
         return new HashSet<>(result);
     }
+
+    /**
+     * Supplement the information needed by be for the sort node.
+     */
+    public void finalizeForNereids(TupleDescriptor tupleDescriptor,
+            List<Expr> outputList, List<Expr> orderingExpr) {
+        List<Expr> sortTupleSlotExprs = new ArrayList<>();
+        sortTupleSlotExprs.addAll(outputList);
+        sortTupleSlotExprs.addAll(orderingExpr);
+        List<Expr> afterDeduplication = new ArrayList<>();
+        Set<ExprId> exprIds = new HashSet<>();
+        for (int i = 0; i < sortTupleSlotExprs.size(); i++) {
+            Expr expr = sortTupleSlotExprs.get(i);
+            if (!exprIds.contains(expr.getId())) {
+                afterDeduplication.add(expr);
+            }
+        }
+        info.setSortTupleDesc(tupleDescriptor);
+        info.setSortTupleSlotExprs(afterDeduplication);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 34321cee03..6478551678 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -978,7 +978,6 @@ public class StmtExecutor implements ProfileWriter {
             return;
         }
 
-
         MysqlChannel channel = context.getMysqlChannel();
         boolean isOutfileQuery = queryStmt.hasOutFileClause();
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Reply via email to