This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 108085330a7f0ff8083f501e53e12ae92d74905b Author: EmmyMiao87 <[email protected]> AuthorDate: Fri Jun 24 12:29:02 2022 +0800 [hotfix](dev-1.0.1) fix inline view bug for vec engine --- be/src/vec/exec/join/vhash_join_node.cpp | 6 ++-- .../java/org/apache/doris/analysis/Analyzer.java | 13 +++++-- .../apache/doris/analysis/ExprSubstitutionMap.java | 4 +++ .../java/org/apache/doris/analysis/FromClause.java | 23 ++++++++++++- .../org/apache/doris/analysis/InlineViewRef.java | 1 + .../org/apache/doris/planner/HashJoinNode.java | 40 ++++++++++++++++++++-- .../apache/doris/planner/SingleNodePlanner.java | 12 +++---- 7 files changed, 84 insertions(+), 15 deletions(-) diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp index 37d31a0a13..c8d7f6e77c 100644 --- a/be/src/vec/exec/join/vhash_join_node.cpp +++ b/be/src/vec/exec/join/vhash_join_node.cpp @@ -1172,15 +1172,15 @@ Status HashJoinNode::_extract_probe_join_column(Block& block, NullMap& null_map, Status HashJoinNode::_process_build_block(RuntimeState* state, Block& block, uint8_t offset) { SCOPED_TIMER(_build_table_timer); + if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) { + _convert_block_to_null(block); + } size_t rows = block.rows(); if (UNLIKELY(rows == 0)) { return Status::OK(); } COUNTER_UPDATE(_build_rows_counter, rows); - if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) { - _convert_block_to_null(block); - } ColumnRawPtrs raw_ptrs(_build_expr_ctxs.size()); NullMap null_map_val(rows); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index d05cb2685d..fa40b57d2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -217,8 +217,9 @@ public class Analyzer { private final Map<TupleId, List<ExprId>> eqJoinConjuncts = Maps.newHashMap(); // set of conjuncts that have been assigned to some PlanNode - private Set<ExprId> assignedConjuncts = - Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>()); + private Set<ExprId> assignedConjuncts = Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>()); + + private Set<TupleId> inlineViewTupleIds = Sets.newHashSet(); // map from outer-joined tuple id, ie, one that is nullable in this select block, // to the last Join clause (represented by its rhs table ref) that outer-joined it @@ -841,6 +842,10 @@ public class Analyzer { return result; } + public void registerInlineViewTupleId(TupleId tupleId) { + globalState.inlineViewTupleIds.add(tupleId); + } + /** * Register conjuncts that are outer joined by a full outer join. For a given * predicate, we record the last full outer join that outer-joined any of its @@ -2068,6 +2073,10 @@ public class Analyzer { return globalState.outerJoinedTupleIds.containsKey(tid); } + public boolean isInlineView(TupleId tid) { + return globalState.inlineViewTupleIds.contains(tid); + } + public boolean containSubquery() { return globalState.containsSubquery; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java index 20e68cf5c3..062eef4df8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java @@ -106,6 +106,10 @@ public final class ExprSubstitutionMap { } } + public void updateLhsExprs(List<Expr> lhsExprList) { + lhs_ = lhsExprList; + } + /** * Return a map which is equivalent to applying f followed by g, * i.e., g(f()). diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java index e77fd6ac21..fa6242f01f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java @@ -147,8 +147,11 @@ public class FromClause implements ParseNode, Iterable<TableRef> { tblRef.analyze(analyzer); leftTblRef = tblRef; } - checkExternalTable(analyzer); + + // Fix the problem of column nullable attribute error caused by inline view + outer join + changeTblRefToNullable(analyzer); + // TODO: remove when query from hive table is supported checkFromHiveTable(analyzer); @@ -184,6 +187,24 @@ public class FromClause implements ParseNode, Iterable<TableRef> { } } + // set null-side inlinve view column + // For example: select * from (select a as k1 from t) tmp right join b on tmp.k1=b.k1 + // The columns from tmp should be nullable. + // The table ref tmp will be used by HashJoinNode.computeOutputTuple() + private void changeTblRefToNullable(Analyzer analyzer) { + for (TableRef tableRef : tableRefs_) { + if (!(tableRef instanceof InlineViewRef)) { + continue; + } + InlineViewRef inlineViewRef = (InlineViewRef) tableRef; + if (analyzer.isOuterJoined(inlineViewRef.getId())) { + for (SlotDescriptor slotDescriptor : inlineViewRef.getDesc().getSlots()) { + slotDescriptor.setIsNullable(true); + } + } + } + } + public FromClause clone() { ArrayList<TableRef> clone = Lists.newArrayList(); for (TableRef tblRef : tableRefs_) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java index 7bf43b78b6..01791ed17f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -286,6 +286,7 @@ public class InlineViewRef extends TableRef { TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor(); result.setIsMaterialized(false); result.setTable(inlineView); + analyzer.registerInlineViewTupleId(result.getId()); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 0534209275..52c623f3e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -28,10 +28,10 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.analysis.TupleIsNullPredicate; import org.apache.doris.catalog.ColumnStats; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.CheckedMath; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.Pair; @@ -320,7 +320,7 @@ public class HashJoinNode extends PlanNode { } } - private void computeOutputTuple(Analyzer analyzer) throws AnalysisException { + private void computeOutputTuple(Analyzer analyzer) throws UserException { // 1. create new tuple vOutputTupleDesc = analyzer.getDescTbl().createTupleDescriptor(); boolean copyLeft = false; @@ -362,6 +362,8 @@ public class HashJoinNode extends PlanNode { break; } ExprSubstitutionMap srcTblRefToOutputTupleSmap = new ExprSubstitutionMap(); + int leftNullableNumber = 0; + int rightNullableNumber = 0; if (copyLeft) { for (TupleDescriptor leftTupleDesc : analyzer.getDescTbl().getTupleDesc(getChild(0).getOutputTblRefIds())) { for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) { @@ -372,6 +374,7 @@ public class HashJoinNode extends PlanNode { analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, leftSlotDesc); if (leftNullable) { outputSlotDesc.setIsNullable(true); + leftNullableNumber++; } srcTblRefToOutputTupleSmap.put(new SlotRef(leftSlotDesc), new SlotRef(outputSlotDesc)); } @@ -388,6 +391,7 @@ public class HashJoinNode extends PlanNode { analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, rightSlotDesc); if (rightNullable) { outputSlotDesc.setIsNullable(true); + rightNullableNumber++; } srcTblRefToOutputTupleSmap.put(new SlotRef(rightSlotDesc), new SlotRef(outputSlotDesc)); } @@ -406,7 +410,37 @@ public class HashJoinNode extends PlanNode { } } vOutputTupleDesc.computeStatAndMemLayout(); - // 3. change the outputSmap + // 3. add tupleisnull in null-side + Preconditions.checkState(srcTblRefToOutputTupleSmap.getLhs().size() == vSrcToOutputSMap.getLhs().size()); + // Condition1: the left child is null-side + // Condition2: the left child is a inline view + // Then: add tuple is null in left child columns + if (leftNullable && getChild(0).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(0).tblRefIds.get(0))) { + List<Expr> tupleIsNullLhs = TupleIsNullPredicate + .wrapExprs(vSrcToOutputSMap.getLhs().subList(0, leftNullableNumber), getChild(0).getTupleIds(), + analyzer); + tupleIsNullLhs + .addAll(vSrcToOutputSMap.getLhs().subList(leftNullableNumber, vSrcToOutputSMap.getLhs().size())); + vSrcToOutputSMap.updateLhsExprs(tupleIsNullLhs); + } + // Condition1: the right child is null-side + // Condition2: the right child is a inline view + // Then: add tuple is null in right child columns + if (rightNullable && getChild(1).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(1).tblRefIds.get(0))) { + if (rightNullableNumber != 0) { + int rightBeginIndex = vSrcToOutputSMap.size() - rightNullableNumber; + List<Expr> tupleIsNullLhs = TupleIsNullPredicate + .wrapExprs(vSrcToOutputSMap.getLhs().subList(rightBeginIndex, vSrcToOutputSMap.size()), + getChild(1).getTupleIds(), analyzer); + List<Expr> newLhsList = Lists.newArrayList(); + if (rightBeginIndex > 0) { + newLhsList.addAll(vSrcToOutputSMap.getLhs().subList(0, rightBeginIndex)); + } + newLhsList.addAll(tupleIsNullLhs); + vSrcToOutputSMap.updateLhsExprs(newLhsList); + } + } + // 4. change the outputSmap outputSmap = ExprSubstitutionMap.combineAndReplace(outputSmap, srcTblRefToOutputTupleSmap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 1f4a90a3ee..c804ed624f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -59,6 +59,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.VectorizedUtil; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; @@ -66,7 +67,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1347,10 +1347,10 @@ public class SingleNodePlanner { unionNode.addConstExprList(selectStmt.getBaseTblResultExprs()); unionNode.init(analyzer); //set outputSmap to substitute literal in outputExpr - unionNode.setWithoutTupleIsNullOutputSmap(inlineViewRef.getSmap()); - if (analyzer.isOuterJoined(inlineViewRef.getId())) { - List<Expr> nullableRhs = TupleIsNullPredicate.wrapExprs( - inlineViewRef.getSmap().getRhs(), unionNode.getTupleIds(), analyzer); + if (analyzer.isOuterJoined(inlineViewRef.getId()) && !VectorizedUtil.isVectorized()) { + unionNode.setWithoutTupleIsNullOutputSmap(inlineViewRef.getSmap()); + List<Expr> nullableRhs = TupleIsNullPredicate + .wrapExprs(inlineViewRef.getSmap().getRhs(), unionNode.getTupleIds(), analyzer); unionNode.setOutputSmap(new ExprSubstitutionMap(inlineViewRef.getSmap().getLhs(), nullableRhs)); } return unionNode; @@ -1370,7 +1370,7 @@ public class SingleNodePlanner { ExprSubstitutionMap outputSmap = ExprSubstitutionMap.compose( inlineViewRef.getSmap(), rootNode.getOutputSmap(), analyzer); - if (analyzer.isOuterJoined(inlineViewRef.getId())) { + if (analyzer.isOuterJoined(inlineViewRef.getId()) && !VectorizedUtil.isVectorized()) { rootNode.setWithoutTupleIsNullOutputSmap(outputSmap); // Exprs against non-matched rows of an outer join should always return NULL. // Make the rhs exprs of the output smap nullable, if necessary. This expr wrapping --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
