HIVE-19358 : CBO decorrelation logic should generate Hive operators (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f567a823 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f567a823 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f567a823 Branch: refs/heads/master Commit: f567a8231f4329e77d8eceb68b40d0cee8022a47 Parents: 0ebf04c Author: Jesus Camacho Rodriguez <jcama...@apache.org> Authored: Sun Apr 29 14:14:00 2018 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Mon Jun 4 21:00:15 2018 -0700 ---------------------------------------------------------------------- .../ql/optimizer/calcite/HiveRelOptUtil.java | 113 +++ .../calcite/cost/HiveVolcanoPlanner.java | 5 +- .../calcite/rules/HiveRelDecorrelator.java | 442 +++++------ .../clientpositive/llap/explainuser_1.q.out | 112 ++- .../llap/groupby_groupingset_bug.q.out | 16 +- .../results/clientpositive/llap/lineage3.q.out | 4 +- .../clientpositive/llap/subquery_in.q.out | 100 ++- .../llap/subquery_in_having.q.out | 60 +- .../clientpositive/llap/subquery_multi.q.out | 212 ++++-- .../clientpositive/llap/subquery_notin.q.out | 738 ++++++++++--------- .../clientpositive/llap/subquery_scalar.q.out | 545 +++++++------- .../clientpositive/llap/subquery_select.q.out | 100 ++- .../clientpositive/llap/subquery_views.q.out | 93 +-- .../clientpositive/perf/spark/query1.q.out | 16 +- .../clientpositive/perf/spark/query30.q.out | 18 +- .../clientpositive/perf/spark/query32.q.out | 18 +- .../clientpositive/perf/spark/query6.q.out | 22 +- .../clientpositive/perf/spark/query81.q.out | 18 +- .../clientpositive/perf/spark/query92.q.out | 18 +- .../clientpositive/perf/tez/query1.q.out | 8 +- .../clientpositive/perf/tez/query30.q.out | 10 +- .../clientpositive/perf/tez/query32.q.out | 8 +- .../clientpositive/perf/tez/query6.q.out | 12 +- .../clientpositive/perf/tez/query81.q.out | 50 +- .../clientpositive/perf/tez/query92.q.out | 8 +- .../spark/spark_explainuser_1.q.out | 120 +-- .../clientpositive/spark/subquery_in.q.out | 84 +-- .../clientpositive/spark/subquery_multi.q.out | 87 +-- .../clientpositive/spark/subquery_notin.q.out | 512 ++++++------- .../clientpositive/spark/subquery_scalar.q.out | 537 +++++++------- .../clientpositive/spark/subquery_select.q.out | 90 +-- .../clientpositive/spark/subquery_views.q.out | 85 ++- .../clientpositive/subquery_notexists.q.out | 42 +- .../clientpositive/subquery_notin_having.q.out | 43 +- .../subquery_unqualcolumnrefs.q.out | 25 +- 35 files changed, 2233 insertions(+), 2138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index 50fbb78..268284a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -21,13 +21,19 @@ import java.util.AbstractList; import java.util.ArrayList; import java.util.List; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; @@ -347,4 +353,111 @@ public class HiveRelOptUtil extends RelOptUtil { }, true, relBuilder); } + public static RexNode splitCorrelatedFilterCondition( + Filter filter, + List<RexNode> joinKeys, + List<RexNode> correlatedJoinKeys, + boolean extractCorrelatedFieldAccess) { + final List<RexNode> nonEquiList = new ArrayList<>(); + + splitCorrelatedFilterCondition( + filter, + filter.getCondition(), + joinKeys, + correlatedJoinKeys, + nonEquiList, + extractCorrelatedFieldAccess); + + // Convert the remainders into a list that are AND'ed together. + return RexUtil.composeConjunction( + filter.getCluster().getRexBuilder(), nonEquiList, true); + } + + private static void splitCorrelatedFilterCondition( + Filter filter, + RexNode condition, + List<RexNode> joinKeys, + List<RexNode> correlatedJoinKeys, + List<RexNode> nonEquiList, + boolean extractCorrelatedFieldAccess) { + if (condition instanceof RexCall) { + RexCall call = (RexCall) condition; + if (call.getOperator().getKind() == SqlKind.AND) { + for (RexNode operand : call.getOperands()) { + splitCorrelatedFilterCondition( + filter, + operand, + joinKeys, + correlatedJoinKeys, + nonEquiList, + extractCorrelatedFieldAccess); + } + return; + } + + if (call.getOperator().getKind() == SqlKind.EQUALS) { + final List<RexNode> operands = call.getOperands(); + RexNode op0 = operands.get(0); + RexNode op1 = operands.get(1); + + if (extractCorrelatedFieldAccess) { + if (!RexUtil.containsFieldAccess(op0) + && (op1 instanceof RexFieldAccess)) { + joinKeys.add(op0); + correlatedJoinKeys.add(op1); + return; + } else if ( + (op0 instanceof RexFieldAccess) + && !RexUtil.containsFieldAccess(op1)) { + correlatedJoinKeys.add(op0); + joinKeys.add(op1); + return; + } + } else { + if (!(RexUtil.containsInputRef(op0)) + && (op1 instanceof RexInputRef)) { + correlatedJoinKeys.add(op0); + joinKeys.add(op1); + return; + } else if ( + (op0 instanceof RexInputRef) + && !(RexUtil.containsInputRef(op1))) { + joinKeys.add(op0); + correlatedJoinKeys.add(op1); + return; + } + } + } + } + + // The operator is not of RexCall type + // So we fail. Fall through. + // Add this condition to the list of non-equi-join conditions. + nonEquiList.add(condition); + } + + /** + * Creates a LogicalAggregate that removes all duplicates from the result of + * an underlying relational expression. + * + * @param rel underlying rel + * @return rel implementing SingleValueAgg + */ + public static RelNode createSingleValueAggRel( + RelOptCluster cluster, + RelNode rel, + RelFactories.AggregateFactory aggregateFactory) { + // assert (rel.getRowType().getFieldCount() == 1); + final int aggCallCnt = rel.getRowType().getFieldCount(); + final List<AggregateCall> aggCalls = new ArrayList<>(); + + for (int i = 0; i < aggCallCnt; i++) { + aggCalls.add( + AggregateCall.create( + SqlStdOperatorTable.SINGLE_VALUE, false, false, + ImmutableList.of(i), -1, 0, rel, null, null)); + } + + return aggregateFactory.createAggregate(rel, false, ImmutableBitSet.of(), null, aggCalls); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index 23ff518..5dcbff6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -97,7 +97,10 @@ public class HiveVolcanoPlanner extends VolcanoPlanner { if (rel instanceof RelSubset) { // Get cost of the subset, best rel may have been chosen or not RelSubset subset = (RelSubset) rel; - return getCost(Util.first(subset.getBest(), subset.getOriginal()), mq); + if (subset.getBest() != null) { + return getCost(subset.getBest(), mq); + } + return costFactory.makeInfiniteCost(); } if (rel.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == Convention.NONE) { http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index 24e22a0..238ae4e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -53,22 +53,14 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Values; -import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalCorrelate; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalIntersect; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.FilterJoinRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -104,6 +96,8 @@ import org.apache.calcite.util.ReflectiveVisitor; import org.apache.calcite.util.Stacks; import org.apache.calcite.util.Util; import org.apache.calcite.util.mapping.Mappings; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -137,7 +131,7 @@ import java.util.Stack; * NOTE: this whole logic is replicated from Calcite's RelDecorrelator * and is exteneded to make it suitable for HIVE * We should get rid of this and replace it with Calcite's RelDecorrelator - * once that works with Join, Project etc instead of LogicalJoin, LogicalProject. + * once that works with Join, Project etc instead of Join, Project. * At this point this has differed from Calcite's version significantly so cannot * get rid of this. * @@ -198,8 +192,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { this.cm = cm; this.rexBuilder = cluster.getRexBuilder(); this.context = context; - relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster, null); - + relBuilder = HiveRelFactories.HIVE_BUILDER.create(cluster, null); } //~ Methods ---------------------------------------------------------------- @@ -245,8 +238,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { HepProgram program = HepProgram.builder() .addRuleInstance(new AdjustProjectForCountAggregateRule(false)) .addRuleInstance(new AdjustProjectForCountAggregateRule(true)) - .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN) - .addRuleInstance(FilterProjectTransposeRule.INSTANCE) + .addRuleInstance(HiveFilterJoinRule.FILTER_ON_JOIN) + .addRuleInstance(HiveFilterProjectTransposeRule.INSTANCE) // FilterCorrelateRule rule mistakenly pushes a FILTER, consiting of correlated vars, // on top of LogicalCorrelate to within left input for scalar corr queries // which causes exception during decorrelation. This has been disabled for now. @@ -265,8 +258,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { if (frame != null) { // has been rewritten; apply rules post-decorrelation final HepProgram program2 = HepProgram.builder() - .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN) - .addRuleInstance(FilterJoinRule.JOIN) + .addRuleInstance(HiveFilterJoinRule.FILTER_ON_JOIN) + .addRuleInstance(HiveFilterJoinRule.JOIN) .build(); final HepPlanner planner2 = createPlanner(program2); @@ -504,11 +497,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } /** - * Rewrites a {@link LogicalAggregate}. + * Rewrites a {@link Aggregate}. * * @param rel Aggregate to rewrite */ - public Frame decorrelateRel(LogicalAggregate rel) throws SemanticException{ + public Frame decorrelateRel(Aggregate rel) throws SemanticException{ if (rel.getGroupType() != Aggregate.Group.SIMPLE) { throw new AssertionError(Bug.CALCITE_461_FIXED); } @@ -654,12 +647,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { newInputOutputFieldCount + i); } - relBuilder.push( - LogicalAggregate.create(newProject, - false, - newGroupSet, - null, - newAggCalls)); + relBuilder.push(newProject) + .aggregate(relBuilder.groupKey(newGroupSet, null), newAggCalls); if (!omittedConstants.isEmpty()) { final List<RexNode> postProjects = new ArrayList<>(relBuilder.fields()); @@ -877,17 +866,17 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { final List<RexNode> oldProjects = rel.getProjects(); final List<RelDataTypeField> relOutput = rel.getRowType().getFieldList(); - // LogicalProject projects the original expressions, + // Project projects the original expressions, // plus any correlated variables the input wants to pass along. final List<Pair<RexNode, String>> projects = Lists.newArrayList(); - // If this LogicalProject has correlated reference, create value generator + // If this Project has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); } - // LogicalProject projects the original expressions + // Project projects the original expressions final Map<Integer, Integer> mapOldToNewOutputs = new HashMap<>(); int newPos; for (newPos = 0; newPos < oldProjects.size(); newPos++) { @@ -917,11 +906,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { corDefOutputs); } /** - * Rewrite LogicalProject. + * Rewrite Project. * * @param rel the project rel to rewrite */ - public Frame decorrelateRel(LogicalProject rel) throws SemanticException{ + public Frame decorrelateRel(Project rel) throws SemanticException{ // // Rewrite logic: // @@ -937,17 +926,17 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { final List<RexNode> oldProjects = rel.getProjects(); final List<RelDataTypeField> relOutput = rel.getRowType().getFieldList(); - // LogicalProject projects the original expressions, + // Project projects the original expressions, // plus any correlated variables the input wants to pass along. final List<Pair<RexNode, String>> projects = Lists.newArrayList(); - // If this LogicalProject has correlated reference, create value generator + // If this Project has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); } - // LogicalProject projects the original expressions + // Project projects the original expressions final Map<Integer, Integer> mapOldToNewOutputs = new HashMap<>(); int newPos; for (newPos = 0; newPos < oldProjects.size(); newPos++) { @@ -977,13 +966,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { /** * Create RelNode tree that produces a list of correlated variables. - * - * @param correlations correlated variables to generate - * @param valueGenFieldOffset offset in the output that generated columns - * will start - * @param mapCorVarToOutputPos output positions for the correlated variables - * generated - * @return RelNode the root of the resultant RelNode tree */ private RelNode createValueGenerator( Iterable<CorRef> correlations, @@ -1039,11 +1021,9 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { assert newInput != null; if (!joinedInputs.contains(newInput)) { - RelNode project = - RelOptUtil.createProject( - newInput, - mapNewInputToOutputs.get(newInput)); - RelNode distinct = RelOptUtil.createDistinctRel(project); + RelNode project = RelOptUtil.createProject( + HiveRelFactories.HIVE_PROJECT_FACTORY, newInput, mapNewInputToOutputs.get(newInput)); + RelNode distinct = relBuilder.push(project).distinct().build(); RelOptCluster cluster = distinct.getCluster(); joinedInputs.add(newInput); @@ -1053,10 +1033,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { if (r == null) { r = distinct; } else { - r = - LogicalJoin.create(r, distinct, - cluster.getRexBuilder().makeLiteral(true), - ImmutableSet.<CorrelationId>of(), JoinRelType.INNER); + r = relBuilder.push(r).push(distinct) + .join(JoinRelType.INNER, cluster.getRexBuilder().makeLiteral(true)).build(); } } } @@ -1153,13 +1131,12 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { leftInputOutputCount, corDefOutputs); - RelNode join = - LogicalJoin.create(frame.r, valueGenRel, rexBuilder.makeLiteral(true), - ImmutableSet.<CorrelationId>of(), JoinRelType.INNER); + RelNode join = relBuilder.push(frame.r).push(valueGenRel) + .join(JoinRelType.INNER, rexBuilder.makeLiteral(true)).build(); - // LogicalJoin or LogicalFilter does not change the old input ordering. All + // Join or Filter does not change the old input ordering. All // input fields from newLeftInput(i.e. the original input to the old - // LogicalFilter) are in the output and in the same position. + // Filter) are in the output and in the same position. return register(oldInput, join, frame.oldToNewOutputs, corDefOutputs); } @@ -1242,16 +1219,16 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // Rewrite logic: // - // 1. If a LogicalFilter references a correlated field in its filter - // condition, rewrite the LogicalFilter to be - // LogicalFilter - // LogicalJoin(cross product) + // 1. If a Filter references a correlated field in its filter + // condition, rewrite the Filter to be + // Filter + // Join(cross product) // OriginalFilterInput // ValueGenerator(produces distinct sets of correlated variables) // and rewrite the correlated fieldAccess in the filter condition to - // reference the LogicalJoin output. + // reference the Join output. // - // 2. If LogicalFilter does not reference correlated variables, simply + // 2. If Filter does not reference correlated variables, simply // rewrite the filter condition using new input. // @@ -1263,7 +1240,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } Frame oldInputFrame = frame; - // If this LogicalFilter has correlated reference, create value generator + // If this Filter has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); @@ -1306,24 +1283,24 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } /** - * Rewrite LogicalFilter. + * Rewrite Filter. * * @param rel the filter rel to rewrite */ - public Frame decorrelateRel(LogicalFilter rel) { + public Frame decorrelateRel(Filter rel) { // // Rewrite logic: // - // 1. If a LogicalFilter references a correlated field in its filter - // condition, rewrite the LogicalFilter to be - // LogicalFilter - // LogicalJoin(cross product) + // 1. If a Filter references a correlated field in its filter + // condition, rewrite the Filter to be + // Filter + // Join(cross product) // OriginalFilterInput // ValueGenerator(produces distinct sets of correlated variables) // and rewrite the correlated fieldAccess in the filter condition to - // reference the LogicalJoin output. + // reference the Join output. // - // 2. If LogicalFilter does not reference correlated variables, simply + // 2. If Filter does not reference correlated variables, simply // rewrite the filter condition using new input. // @@ -1334,7 +1311,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return null; } - // If this LogicalFilter has correlated reference, create value generator + // If this Filter has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); @@ -1500,9 +1477,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); } - newJoin = LogicalJoin.create(leftFrame.r, rightFrame.r, condition, - ImmutableSet.<CorrelationId>of(), rel.getJoinType().toJoinType()); - + newJoin = relBuilder.push(leftFrame.r).push(rightFrame.r) + .join(rel.getJoinType().toJoinType(), condition).build(); } valueGen.pop(); @@ -1564,11 +1540,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); } /** - * Rewrite LogicalJoin. + * Rewrite Join. * - * @param rel LogicalJoin + * @param rel Join */ - public Frame decorrelateRel(LogicalJoin rel) { + public Frame decorrelateRel(Join rel) { // // Rewrite logic: // @@ -1684,11 +1660,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { * @param join Join * @param project Original project as the right-hand input of the join * @param nullIndicatorPos Position of null indicator - * @return the subtree with the new LogicalProject at the root + * @return the subtree with the new Project at the root */ private RelNode projectJoinOutputWithNullability( - LogicalJoin join, - LogicalProject project, + Join join, + Project project, int nullIndicatorPos) { final RelDataTypeFactory typeFactory = join.getCluster().getTypeFactory(); final RelNode left = join.getLeft(); @@ -1730,7 +1706,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { newProjExprs.add(Pair.of(newProjExpr, pair.right)); } - return RelOptUtil.createProject(join, newProjExprs, false); + return RelOptUtil.createProject(join, Pair.left(newProjExprs), Pair.right(newProjExprs), + false, relBuilder); } /** @@ -1741,11 +1718,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { * @param project the original project as the RHS input of the join * @param isCount Positions which are calls to the <code>COUNT</code> * aggregation function - * @return the subtree with the new LogicalProject at the root + * @return the subtree with the new Project at the root */ private RelNode aggregateCorrelatorOutput( Correlate correlate, - LogicalProject project, + Project project, Set<Integer> isCount) { final RelNode left = correlate.getLeft(); final JoinRelType joinType = correlate.getJoinType().toJoinType(); @@ -1777,7 +1754,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { newProjects.add(Pair.of(newProjExpr, pair.right)); } - return RelOptUtil.createProject(correlate, newProjects, false); + return RelOptUtil.createProject(correlate, Pair.left(newProjects), Pair.right(newProjects), + false, relBuilder); } /** @@ -1792,8 +1770,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { */ private boolean checkCorVars( LogicalCorrelate correlate, - LogicalProject project, - LogicalFilter filter, + Project project, + Filter filter, List<RexFieldAccess> correlatedJoinKeys) { if (filter != null) { assert correlatedJoinKeys != null; @@ -1852,7 +1830,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { * * @param input Input relational expression * @param additionalExprs Additional expressions and names - * @return the new LogicalProject + * @return the new Project */ private RelNode createProjectWithAdditionalExprs( RelNode input, @@ -1868,7 +1846,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { field.e.getName())); } projects.addAll(additionalExprs); - return RelOptUtil.createProject(input, projects, false); + return RelOptUtil.createProject(input, Pair.left(projects), Pair.right(projects), + false, relBuilder); } /* Returns an immutable map with the identity [0: 0, .., count-1: count-1]. */ @@ -2157,7 +2136,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } } - final RelDataType newType; if (!isSpecialCast) { // TODO: ideally this only needs to be called if the result // type will also change. However, since that requires @@ -2165,7 +2143,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // decides return type based on input types, for now all // operators will be recreated with new type if any operand // changed, unless the operator has "built-in" type. - newType = rexBuilder.deriveReturnType(operator, clonedOperands); + newCall = rexBuilder.ensureType( + call.getType(), + rexBuilder.makeCall( + rexBuilder.deriveReturnType(operator, clonedOperands), operator, clonedOperands), + true); } else { // Use the current return type when creating a new call, for // operators with return type built into the operator @@ -2175,13 +2157,12 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // TODO: Comments in RexShuttle.visitCall() mention other // types in this category. Need to resolve those together // and preferably in the base class RexShuttle. - newType = call.getType(); + newCall = + rexBuilder.makeCall( + call.getType(), + operator, + clonedOperands); } - newCall = - rexBuilder.makeCall( - newType, - operator, - clonedOperands); } else { newCall = call; } @@ -2206,16 +2187,16 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { RemoveSingleAggregateRule() { super( operand( - LogicalAggregate.class, + Aggregate.class, operand( - LogicalProject.class, - operand(LogicalAggregate.class, any())))); + Project.class, + operand(Aggregate.class, any())))); } public void onMatch(RelOptRuleCall call) { - LogicalAggregate singleAggregate = call.rel(0); - LogicalProject project = call.rel(1); - LogicalAggregate aggregate = call.rel(2); + Aggregate singleAggregate = call.rel(0); + Project project = call.rel(1); + Aggregate aggregate = call.rel(2); // check singleAggRel is single_value agg if ((!singleAggregate.getGroupSet().isEmpty()) @@ -2241,15 +2222,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // singleAggRel produces a nullable type, so create the new // projection that casts proj expr to a nullable type. final RelOptCluster cluster = project.getCluster(); - RelNode newProject = - RelOptUtil.createProject(aggregate, - ImmutableList.of( - rexBuilder.makeCast( - cluster.getTypeFactory().createTypeWithNullability( - projExprs.get(0).getType(), - true), - projExprs.get(0))), - null); + RelNode newProject = RelOptUtil.createProject(aggregate, + ImmutableList.of(rexBuilder.makeCast( + cluster.getTypeFactory().createTypeWithNullability(projExprs.get(0).getType(), true), + projExprs.get(0))), + null, false, relBuilder); call.transformTo(newProject); } } @@ -2260,16 +2237,16 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { super( operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalAggregate.class, - operand(LogicalProject.class, + operand(Aggregate.class, + operand(Project.class, operand(RelNode.class, any()))))); } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalAggregate aggregate = call.rel(2); - final LogicalProject project = call.rel(3); + final Aggregate aggregate = call.rel(2); + final Project project = call.rel(3); RelNode right = call.rel(4); final RelOptCluster cluster = correlate.getCluster(); @@ -2281,9 +2258,12 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalAggregate (groupby (0) single_value()) - // LogicalProject-A (may reference coVar) + // Aggregate (groupby (0) single_value()) + // Project-A (may reference coVar) // RightInputRel + if(correlate.getJoinType() != SemiJoinType.LEFT) { + return; + } final JoinRelType joinType = correlate.getJoinType().toJoinType(); // corRel.getCondition was here, however Correlate was updated so it @@ -2311,18 +2291,18 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { int nullIndicatorPos; - if ((right instanceof LogicalFilter) + if ((right instanceof Filter) && cm.mapRefRelToCorRef.containsKey(right)) { // rightInputRel has this shape: // - // LogicalFilter (references corvar) + // Filter (references corvar) // FilterInputRel // If rightInputRel is a filter and contains correlated // reference, make sure the correlated keys in the filter // condition forms a unique key of the RHS. - LogicalFilter filter = (LogicalFilter) right; + Filter filter = (Filter) right; right = filter.getInput(); assert right instanceof HepRelVertex; @@ -2342,7 +2322,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // refs. These comparisons are AND'ed together. List<RexNode> tmpRightJoinKeys = Lists.newArrayList(); List<RexNode> correlatedJoinKeys = Lists.newArrayList(); - RelOptUtil.splitCorrelatedFilterCondition( + HiveRelOptUtil.splitCorrelatedFilterCondition( filter, tmpRightJoinKeys, correlatedJoinKeys, @@ -2386,8 +2366,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // Change the plan to this structure. // Note that the aggregateRel is removed. // - // LogicalProject-A' (replace corvar to input ref from the LogicalJoin) - // LogicalJoin (replace corvar to input ref from LeftInputRel) + // Project-A' (replace corvar to input ref from the Join) + // Join (replace corvar to input ref from LeftInputRel) // LeftInputRel // RightInputRel(oreviously FilterInputRel) @@ -2410,11 +2390,11 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // Change the plan to this structure. // - // LogicalProject-A' (replace corvar to input ref from LogicalJoin) - // LogicalJoin (left, condition = true) + // Project-A' (replace corvar to input ref from Join) + // Join (left, condition = true) // LeftInputRel - // LogicalAggregate(groupby(0), single_value(0), s_v(1)....) - // LogicalProject-B (everything from input plus literal true) + // Aggregate(groupby(0), single_value(0), s_v(1)....) + // Project-B (everything from input plus literal true) // ProjInputRel // make the new projRel to provide a null indicator @@ -2426,7 +2406,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // make the new aggRel right = - RelOptUtil.createSingleValueAggRel(cluster, right); + HiveRelOptUtil.createSingleValueAggRel(cluster, right, HiveRelFactories.HIVE_AGGREGATE_FACTORY); // The last field: // single_value(true) @@ -2439,9 +2419,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } // make the new join rel - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.<CorrelationId>of(), joinType); + Join join = (Join) relBuilder.push(left).push(right) + .join(joinType, joinCond).build(); RelNode newProject = projectJoinOutputWithNullability(join, project, nullIndicatorPos); @@ -2459,18 +2438,18 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { super( operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalProject.class, - operand(LogicalAggregate.class, null, Aggregate.IS_SIMPLE, - operand(LogicalProject.class, + operand(Project.class, + operand(Aggregate.class, null, Aggregate.IS_SIMPLE, + operand(Project.class, operand(RelNode.class, any())))))); } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalProject aggOutputProject = call.rel(2); - final LogicalAggregate aggregate = call.rel(3); - final LogicalProject aggInputProject = call.rel(4); + final Project aggOutputProject = call.rel(2); + final Aggregate aggregate = call.rel(3); + final Project aggInputProject = call.rel(4); RelNode right = call.rel(5); final RelOptCluster cluster = correlate.getCluster(); @@ -2482,9 +2461,9 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby (0), agg0(), agg1()...) - // LogicalProject-B (references coVar) + // Project-A (a RexNode) + // Aggregate (groupby (0), agg0(), agg1()...) + // Project-B (references coVar) // rightInputRel // check aggOutputProject projects only one expression @@ -2493,6 +2472,10 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return; } + if(correlate.getJoinType() != SemiJoinType.LEFT) { + return; + } + final JoinRelType joinType = correlate.getJoinType().toJoinType(); // corRel.getCondition was here, however Correlate was updated so it // never includes a join condition. The code was not modified for brevity. @@ -2523,13 +2506,13 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } } - if ((right instanceof LogicalFilter) + if ((right instanceof Filter) && cm.mapRefRelToCorRef.containsKey(right)) { // rightInputRel has this shape: // - // LogicalFilter (references corvar) + // Filter (references corvar) // FilterInputRel - LogicalFilter filter = (LogicalFilter) right; + Filter filter = (Filter) right; right = filter.getInput(); assert right instanceof HepRelVertex; @@ -2550,7 +2533,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // expressions. These comparisons are AND'ed together. List<RexNode> rightJoinKeys = Lists.newArrayList(); List<RexNode> tmpCorrelatedJoinKeys = Lists.newArrayList(); - RelOptUtil.splitCorrelatedFilterCondition( + HiveRelOptUtil.splitCorrelatedFilterCondition( filter, rightJoinKeys, tmpCorrelatedJoinKeys, @@ -2600,21 +2583,21 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby(0), agg0(),agg1()...) - // LogicalProject-B (may reference coVar) - // LogicalFilter (references corVar) + // Project-A (a RexNode) + // Aggregate (groupby(0), agg0(),agg1()...) + // Project-B (may reference coVar) + // Filter (references corVar) // RightInputRel (no correlated reference) // // to this plan: // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs) + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs) // agg0(rewritten expression), // agg1()...) - // LogicalProject-B' (rewriten original projected exprs) - // LogicalJoin(replace corvar w/ input ref from LeftInputRel) + // Project-B' (rewriten original projected exprs) + // Join(replace corvar w/ input ref from LeftInputRel) // LeftInputRel // RightInputRel // @@ -2626,14 +2609,14 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // projection list from the RHS for simplicity to avoid // searching for non-null fields. // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs), + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs), // count(nullIndicator), other aggs...) - // LogicalProject-B' (all left input refs plus + // Project-B' (all left input refs plus // the rewritten original projected exprs) - // LogicalJoin(replace corvar to input ref from LeftInputRel) + // Join(replace corvar to input ref from LeftInputRel) // LeftInputRel - // LogicalProject (everything from RightInputRel plus + // Project (everything from RightInputRel plus // the nullIndicator "true") // RightInputRel // @@ -2668,20 +2651,20 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby(0), agg0(), agg1()...) - // LogicalProject-B (references coVar) + // Project-A (a RexNode) + // Aggregate (groupby(0), agg0(), agg1()...) + // Project-B (references coVar) // RightInputRel (no correlated reference) // // to this plan: // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs) + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs) // agg0(rewritten expression), // agg1()...) - // LogicalProject-B' (rewriten original projected exprs) - // LogicalJoin (LOJ cond = true) + // Project-B' (rewriten original projected exprs) + // Join (LOJ cond = true) // LeftInputRel // RightInputRel // @@ -2693,14 +2676,14 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // projection list from the RHS for simplicity to avoid // searching for non-null fields. // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs), + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs), // count(nullIndicator), other aggs...) - // LogicalProject-B' (all left input refs plus + // Project-B' (all left input refs plus // the rewritten original projected exprs) - // LogicalJoin(replace corvar to input ref from LeftInputRel) + // Join(replace corvar to input ref from LeftInputRel) // LeftInputRel - // LogicalProject (everything from RightInputRel plus + // Project (everything from RightInputRel plus // the nullIndicator "true") // RightInputRel } else { @@ -2718,9 +2701,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { Pair.<RexNode, String>of(rexBuilder.makeLiteral(true), "nullIndicator"))); - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.<CorrelationId>of(), joinType); + Join join = (Join) relBuilder.push(left).push(right) + .join(joinType, joinCond).build(); // To the consumer of joinOutputProjRel, nullIndicator is located // at the end @@ -2754,11 +2736,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { joinOutputProjects.add( rexBuilder.makeInputRef(join, nullIndicatorPos)); - RelNode joinOutputProject = - RelOptUtil.createProject( - join, - joinOutputProjects, - null); + RelNode joinOutputProject = RelOptUtil.createProject( + join, joinOutputProjects, null, false, relBuilder); // nullIndicator is now at a different location in the output of // the join @@ -2793,12 +2772,9 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { ImmutableBitSet groupSet = ImmutableBitSet.range(groupCount); - LogicalAggregate newAggregate = - LogicalAggregate.create(joinOutputProject, - false, - groupSet, - null, - newAggCalls); + Aggregate newAggregate = + (Aggregate) relBuilder.push(joinOutputProject) + .aggregate(relBuilder.groupKey(groupSet, null), newAggCalls).build(); List<RexNode> newAggOutputProjectList = Lists.newArrayList(); for (int i : groupSet) { @@ -2815,11 +2791,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { true), newAggOutputProjects)); - RelNode newAggOutputProject = - RelOptUtil.createProject( - newAggregate, - newAggOutputProjectList, - null); + RelNode newAggOutputProject = RelOptUtil.createProject( + newAggregate, newAggOutputProjectList, null, false, relBuilder); call.transformTo(newAggOutputProject); @@ -2844,19 +2817,19 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { flavor ? operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalProject.class, - operand(LogicalAggregate.class, any()))) + operand(Project.class, + operand(Aggregate.class, any()))) : operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalAggregate.class, any()))); + operand(Aggregate.class, any()))); this.flavor = flavor; } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalProject aggOutputProject; - final LogicalAggregate aggregate; + final Project aggOutputProject; + final Aggregate aggregate; if (flavor) { aggOutputProject = call.rel(2); aggregate = call.rel(3); @@ -2870,11 +2843,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { for (int i = 0; i < fields.size(); i++) { projects.add(RexInputRef.of2(projects.size(), fields)); } - aggOutputProject = - (LogicalProject) RelOptUtil.createProject( - aggregate, - projects, - false); + aggOutputProject = (Project) RelOptUtil.createProject( + aggregate, Pair.left(projects), Pair.right(projects), false, relBuilder); } onMatch2(call, correlate, left, aggOutputProject, aggregate); } @@ -2883,8 +2853,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { RelOptRuleCall call, LogicalCorrelate correlate, RelNode leftInput, - LogicalProject aggOutputProject, - LogicalAggregate aggregate) { + Project aggOutputProject, + Aggregate aggregate) { if (generatedCorRels.contains(correlate)) { // This correlator was generated by a previous invocation of // this rule. No further work to do. @@ -2899,8 +2869,8 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby (0), agg0(), agg1()...) + // Project-A (a RexNode) + // Aggregate (groupby (0), agg0(), agg1()...) // check aggOutputProj projects only one expression List<RexNode> aggOutputProjExprs = aggOutputProject.getProjects(); @@ -2908,6 +2878,10 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return; } + if(correlate.getJoinType() != SemiJoinType.LEFT) { + return; + } + JoinRelType joinType = correlate.getJoinType().toJoinType(); // corRel.getCondition was here, however Correlate was updated so it // never includes a join condition. The code was not modified for brevity. @@ -2940,7 +2914,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { // replacing references to count() with case statement) // Correlator(left correlation, condition = true) // LeftInputRel - // LogicalAggregate (groupby (0), agg0(), agg1()...) + // Aggregate (groupby (0), agg0(), agg1()...) // LogicalCorrelate newCorrelate = LogicalCorrelate.create(leftInput, aggregate, @@ -3179,24 +3153,10 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { mightRequireValueGen = true; return rel; } - public RelNode visit(LogicalUnion rel) { - mightRequireValueGen = true; - return rel; - } - public RelNode visit(LogicalIntersect rel) { - mightRequireValueGen = true; - return rel; - } - public RelNode visit(HiveIntersect rel) { mightRequireValueGen = true; return rel; } - - @Override public RelNode visit(LogicalJoin rel) { - mightRequireValueGen = true; - return rel; - } @Override public RelNode visit(HiveProject rel) { if(!(hasRexOver(((HiveProject)rel).getProjects()))) { mightRequireValueGen = false; @@ -3206,15 +3166,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return rel; } } - @Override public RelNode visit(LogicalProject rel) { - if(!(hasRexOver(((LogicalProject)rel).getProjects()))) { - mightRequireValueGen = false; - return super.visit(rel); - } else { - mightRequireValueGen = true; - return rel; - } - } @Override public RelNode visit(HiveAggregate rel) { // if there are aggregate functions or grouping sets we will need // value generator @@ -3228,17 +3179,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { return rel; } } - @Override public RelNode visit(LogicalAggregate rel) { - if(rel.getAggCallList().isEmpty() && !rel.indicator) { - this.mightRequireValueGen = false; - return super.visit(rel); - } else { - // need to reset to true in case previous aggregate/project - // has set it to false - this.mightRequireValueGen = true; - return rel; - } - } @Override public RelNode visit(LogicalCorrelate rel) { // this means we are hitting nested subquery so don't // need to go further @@ -3279,16 +3219,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { mapFieldAccessToCorVar); } - @Override public RelNode visit(LogicalJoin join) { - try { - Stacks.push(stack, join); - join.getCondition().accept(rexVisitor(join)); - } finally { - Stacks.pop(stack, join); - } - return visitJoin(join); - } - public RelNode visit(HiveJoin join) { try { Stacks.push(stack, join); @@ -3329,6 +3259,7 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } return super.visit(project); } + public RelNode visit(final HiveFilter filter) { try { Stacks.push(stack, filter); @@ -3338,27 +3269,6 @@ public final class HiveRelDecorrelator implements ReflectiveVisitor { } return super.visit(filter); } - @Override public RelNode visit(final LogicalFilter filter) { - try { - Stacks.push(stack, filter); - filter.getCondition().accept(rexVisitor(filter)); - } finally { - Stacks.pop(stack, filter); - } - return super.visit(filter); - } - - @Override public RelNode visit(LogicalProject project) { - try { - Stacks.push(stack, project); - for (RexNode node : project.getProjects()) { - node.accept(rexVisitor(project)); - } - } finally { - Stacks.pop(stack, project); - } - return super.visit(project); - } private RexVisitorImpl<Void> rexVisitor(final RelNode rel) { return new RexVisitorImpl<Void>(true) { http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/test/results/clientpositive/llap/explainuser_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 0c339e5..8801331 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2287,27 +2287,27 @@ Stage-0 limit:-1 Stage-1 Reducer 3 llap - File Output Operator [FS_26] - Select Operator [SEL_25] (rows=13 width=223) + File Output Operator [FS_24] + Select Operator [SEL_23] (rows=13 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=13 width=231) - predicate:(not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) - Merge Join Operator [MERGEJOIN_32] (rows=26 width=230) - Conds:RS_21._col0, _col1=RS_22._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] + Filter Operator [FIL_22] (rows=13 width=231) + predicate:CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END + Merge Join Operator [MERGEJOIN_30] (rows=26 width=230) + Conds:RS_19._col0, _col1=RS_20._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + SHUFFLE [RS_19] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_31] (rows=26 width=229) - Conds:RS_18._col1=RS_19._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] + Merge Join Operator [MERGEJOIN_29] (rows=26 width=229) + Conds:RS_16._col1=RS_17._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] + SHUFFLE [RS_16] PartitionCols:_col1 Select Operator [SEL_1] (rows=26 width=223) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] <-Reducer 4 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_19] + FORWARD [RS_17] PartitionCols:_col0 Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2318,28 +2318,26 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr Select Operator [SEL_4] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_29] (rows=8 width=223) + Filter Operator [FIL_27] (rows=8 width=223) predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] <-Reducer 5 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_22] + FORWARD [RS_20] PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=4 width=223) + Select Operator [SEL_15] (rows=4 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_16] (rows=4 width=219) - predicate:_col0 is not null - Group By Operator [GBY_14] (rows=4 width=219) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=4 width=219) - Output:["_col0","_col1"],keys:p_name, p_mfgr - Select Operator [SEL_11] (rows=8 width=223) - Output:["p_name","p_mfgr"] - Filter Operator [FIL_30] (rows=8 width=223) - predicate:((p_size < 10) and p_mfgr is not null) - Please refer to the previous TableScan [TS_0] + Group By Operator [GBY_14] (rows=4 width=219) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_13] + PartitionCols:_col0, _col1 + Group By Operator [GBY_12] (rows=4 width=219) + Output:["_col0","_col1"],keys:p_name, p_mfgr + Select Operator [SEL_11] (rows=8 width=223) + Output:["p_name","p_mfgr"] + Filter Operator [FIL_28] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null and p_name is not null) + Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select p_name, p_size from @@ -2451,24 +2449,24 @@ Stage-0 limit:-1 Stage-1 Reducer 5 llap - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=3 width=106) + File Output Operator [FS_36] + Select Operator [SEL_35] (rows=2 width=106) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=3 width=106) + SHUFFLE [RS_34] + Select Operator [SEL_33] (rows=2 width=106) Output:["_col0","_col1"] - Filter Operator [FIL_33] (rows=3 width=119) - predicate:(not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) - Merge Join Operator [MERGEJOIN_44] (rows=5 width=114) - Conds:RS_30._col0, _col1=RS_31._col0, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] + Filter Operator [FIL_32] (rows=2 width=116) + predicate:CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END + Merge Join Operator [MERGEJOIN_43] (rows=5 width=114) + Conds:RS_29._col0, _col1=RS_30._col0, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + SHUFFLE [RS_29] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_43] (rows=5 width=112) - Conds:RS_27._col1=RS_28._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] + Merge Join Operator [MERGEJOIN_42] (rows=5 width=112) + Conds:RS_26._col1=RS_27._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] + SHUFFLE [RS_26] PartitionCols:_col1 Group By Operator [GBY_4] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 @@ -2482,7 +2480,7 @@ Stage-0 TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] <-Reducer 7 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_28] + FORWARD [RS_27] PartitionCols:_col0 Group By Operator [GBY_16] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2493,7 +2491,7 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col1 Select Operator [SEL_12] (rows=1 width=114) Output:["_col0","_col1"] - Filter Operator [FIL_40] (rows=1 width=114) + Filter Operator [FIL_39] (rows=1 width=114) predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) Group By Operator [GBY_10] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 @@ -2504,24 +2502,22 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr Please refer to the previous Select Operator [SEL_1] <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] + SHUFFLE [RS_30] PartitionCols:_col0, _col1 - Select Operator [SEL_26] (rows=1 width=110) + Select Operator [SEL_25] (rows=1 width=110) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=1 width=110) - predicate:_col0 is not null - Select Operator [SEL_24] (rows=1 width=110) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) - Group By Operator [GBY_22] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_20] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Please refer to the previous TableScan [TS_0] + Filter Operator [FIL_40] (rows=1 width=114) + predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) + Group By Operator [GBY_23] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_22] + PartitionCols:_col0 + Group By Operator [GBY_21] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr + Filter Operator [FIL_41] (rows=26 width=106) + predicate:p_mfgr is not null + Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out index 6ba8811..5e18f33 100644 --- a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out @@ -230,9 +230,9 @@ Stage-0 SHUFFLE [RS_139] PartitionCols:_col0 Group By Operator [GBY_138] (rows=1 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col9 + Output:["_col0","_col1"],aggregations:["count()"],keys:_col8 Map Join Operator [MAPJOIN_137] (rows=5185194 width=4) - Conds:MAPJOIN_136._col7=RS_130._col0(Inner),Output:["_col9"] + Conds:MAPJOIN_136._col6=RS_130._col0(Inner),Output:["_col8"] <-Map 7 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_130] PartitionCols:_col0 @@ -271,17 +271,17 @@ Stage-0 Output:["_col0"] Please refer to the previous Map Join Operator [MAPJOIN_129] <-Map Join Operator [MAPJOIN_136] (rows=370371 width=4) - Conds:RS_30._col0=SEL_135._col0(Inner),Output:["_col7"] + Conds:RS_30._col0=SEL_135._col0(Inner),Output:["_col6"] <-Map 1 [BROADCAST_EDGE] llap BROADCAST [RS_30] PartitionCols:_col0 Map Join Operator [MAPJOIN_100] (rows=6 width=228) - Conds:SEL_2._col1=RS_113._col2(Inner),Output:["_col0","_col2","_col3","_col4"],residual filter predicates:{(_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END)} + Conds:SEL_2._col1=RS_113._col1(Inner),Output:["_col0","_col2","_col3"],residual filter predicates:{(_col2 > _col3)} <-Reducer 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_113] - PartitionCols:_col2 - Select Operator [SEL_112] (rows=1 width=201) - Output:["_col0","_col1","_col2"] + PartitionCols:_col1 + Select Operator [SEL_112] (rows=1 width=197) + Output:["_col0","_col1"] Group By Operator [GBY_111] (rows=1 width=197) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 <-Map 2 [SIMPLE_EDGE] vectorized, llap @@ -296,7 +296,7 @@ Stage-0 <-Select Operator [SEL_2] (rows=18 width=201) Output:["_col0","_col1","_col2"] Filter Operator [FIL_57] (rows=18 width=201) - predicate:i_item_sk is not null + predicate:(i_category is not null and i_item_sk is not null) TableScan [TS_0] (rows=18 width=201) default@x1_item,i,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_category","i_current_price"] <-Select Operator [SEL_135] (rows=123457 width=8) http://git-wip-us.apache.org/repos/asf/hive/blob/f567a823/ql/src/test/results/clientpositive/llap/lineage3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 1ac02ec..16c4efa 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -178,7 +178,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"94e9cc0a67801fe1503a3cb0c5029d59","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"94e9cc0a67801fe1503a3cb0c5029d59","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.ctinyint is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1 .value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select key, value from src1 @@ -196,7 +196,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"723e79692e1de404c4ffb702097586da","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value" },{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"723e79692e1de404c4ffb702097586da","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.ctinyint is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN" ,"vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 369 401 val_401 406 val_406
