HIVE-11927: Implement/Enable constant related optimization rules in Calcite: enable HiveReduceExpressionsRule to fold constants (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b340ecb5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b340ecb5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b340ecb5 Branch: refs/heads/master Commit: b340ecb5e163277e86def59b454b8c041ece39d5 Parents: af96223 Author: Pengcheng Xiong <pxi...@apache.org> Authored: Tue Dec 22 21:06:14 2015 -0800 Committer: Pengcheng Xiong <pxi...@apache.org> Committed: Tue Dec 22 21:06:14 2015 -0800 ---------------------------------------------------------------------- .../calcite/CalciteSemanticException.java | 2 +- .../optimizer/calcite/HiveRexExecutorImpl.java | 80 ++ .../rules/HiveReduceExpressionsRule.java | 905 +++++++++++++++++++ .../calcite/translator/ASTBuilder.java | 30 +- .../calcite/translator/RexNodeConverter.java | 48 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 84 +- ql/src/test/queries/clientpositive/cbo_const.q | 52 ++ .../queries/clientpositive/constantfolding.q | 88 ++ .../clientpositive/annotate_stats_select.q.out | 58 +- .../bucketizedhiveinputformat.q.out | 2 + ql/src/test/results/clientpositive/cast1.q.out | 10 +- .../test/results/clientpositive/cbo_const.q.out | 334 +++++++ .../cbo_rp_cross_product_check_2.q.out | 4 +- .../clientpositive/cbo_rp_lineage2.q.out | 2 +- .../clientpositive/constantfolding.q.out | 305 +++++++ .../clientpositive/cross_product_check_1.q.out | 4 +- .../clientpositive/cross_product_check_2.q.out | 4 +- .../clientpositive/dynamic_rdd_cache.q.out | 46 +- .../dynpart_sort_optimization2.q.out | 45 +- .../results/clientpositive/groupby_ppd.q.out | 18 +- .../clientpositive/groupby_sort_1_23.q.out | 12 +- .../clientpositive/groupby_sort_skew_1_23.q.out | 12 +- .../results/clientpositive/input_part1.q.out | 4 +- .../results/clientpositive/input_part5.q.out | 2 +- .../results/clientpositive/input_part6.q.out | 2 +- .../test/results/clientpositive/lineage2.q.out | 2 +- .../test/results/clientpositive/lineage3.q.out | 10 +- .../list_bucket_query_oneskew_2.q.out | 50 +- .../llap/bucket_map_join_tez1.q.out | 4 +- .../llap/vector_join_part_col_char.q.out | 2 +- .../clientpositive/orc_predicate_pushdown.q.out | 22 +- .../parquet_predicate_pushdown.q.out | 22 +- .../clientpositive/partition_multilevels.q.out | 44 +- .../results/clientpositive/perf/query31.q.out | 338 +++---- .../results/clientpositive/perf/query39.q.out | 58 +- .../results/clientpositive/perf/query42.q.out | 171 ++-- .../results/clientpositive/perf/query52.q.out | 163 ++-- .../results/clientpositive/perf/query64.q.out | 362 ++++---- .../results/clientpositive/perf/query66.q.out | 627 ++++++------- .../results/clientpositive/perf/query75.q.out | 643 ++++++------- .../results/clientpositive/pointlookup2.q.out | 6 +- .../results/clientpositive/quotedid_basic.q.out | 62 +- .../clientpositive/quotedid_partition.q.out | 22 +- .../spark/bucket_map_join_tez1.q.out | 4 +- .../spark/bucketizedhiveinputformat.q.out | 2 + .../spark/cross_product_check_1.q.out | 4 +- .../spark/cross_product_check_2.q.out | 4 +- .../spark/dynamic_rdd_cache.q.out | 46 +- .../spark/groupby_sort_1_23.q.out | 12 +- .../spark/groupby_sort_skew_1_23.q.out | 12 +- .../clientpositive/spark/union_remove_25.q.out | 16 +- .../clientpositive/spark/union_view.q.out | 60 +- .../results/clientpositive/subquery_notin.q.out | 18 +- .../subquery_notin_having.q.java1.7.out | 10 +- .../subquery_unqualcolumnrefs.q.out | 2 +- .../tez/bucket_map_join_tez1.q.out | 4 +- .../tez/cross_product_check_1.q.out | 4 +- .../tez/cross_product_check_2.q.out | 4 +- .../tez/dynpart_sort_optimization2.q.out | 34 +- .../clientpositive/tez/explainuser_1.q.out | 260 +++--- .../tez/vector_decimal_round.q.out | 6 +- ql/src/test/results/clientpositive/udf1.q.out | 40 +- .../results/clientpositive/udf_10_trims.q.out | 2 +- .../clientpositive/udf_folder_constants.q.out | 8 +- .../clientpositive/union_remove_25.q.out | 20 +- .../results/clientpositive/union_view.q.out | 60 +- .../clientpositive/vector_decimal_round.q.out | 6 +- 67 files changed, 3660 insertions(+), 1739 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java index 336745b..f6b4124 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java @@ -31,7 +31,7 @@ public class CalciteSemanticException extends SemanticException { public enum UnsupportedFeature { Distinct_without_an_aggreggation, Duplicates_in_RR, Filter_expression_with_non_boolean_return_type, - Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, + Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, Invalid_interval, Less_than_equal_greater_than, Multi_insert, Others, Same_name_in_multiple_expressions, Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery, Table_sample_clauses, UDTF, Union_type, Unique_join http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java new file mode 100644 index 0000000..f7958c6 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import java.util.HashSet; +import java.util.List; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + +public class HiveRexExecutorImpl implements RelOptPlanner.Executor { + + private final RelOptCluster cluster; + + protected final Logger LOG; + + public HiveRexExecutorImpl(RelOptCluster cluster) { + this.cluster = cluster; + LOG = LoggerFactory.getLogger(this.getClass().getName()); + } + + @Override + public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) { + RexNodeConverter rexNodeConverter = new RexNodeConverter(cluster); + for (RexNode rexNode : constExps) { + // initialize the converter + ExprNodeConverter converter = new ExprNodeConverter("", null, null, null, + new HashSet<Integer>(), cluster.getTypeFactory()); + // convert RexNode to ExprNodeGenericFuncDesc + ExprNodeDesc expr = rexNode.accept(converter); + if (expr instanceof ExprNodeGenericFuncDesc) { + // folding the constant + ExprNodeDesc constant = ConstantPropagateProcFactory + .foldExpr((ExprNodeGenericFuncDesc) expr); + if (constant != null) { + try { + // convert constant back to RexNode + reducedValues.add(rexNodeConverter.convert((ExprNodeConstantDesc) constant)); + } catch (Exception e) { + LOG.warn(e.getMessage()); + reducedValues.add(rexNode); + } + } else { + reducedValues.add(rexNode); + } + } else { + reducedValues.add(rexNode); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java new file mode 100644 index 0000000..50e139b --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptPredicateList; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexRangeRef; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlRowOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.Pair; +import org.apache.calcite.util.Stacks; +import org.apache.calcite.util.Util; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Collection of planner rules that apply various simplifying transformations on + * RexNode trees. Currently, there are two transformations: + * + * <ul> + * <li>Constant reduction, which evaluates constant subtrees, replacing them + * with a corresponding RexLiteral + * <li>Removal of redundant casts, which occurs when the argument into the cast + * is the same as the type of the resulting cast expression + * </ul> + */ +public abstract class HiveReduceExpressionsRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** + * Regular expression that matches the description of all instances of this + * rule and {@link ValuesReduceRule} also. Use + * it to prevent the planner from invoking these rules. + */ + public static final Pattern EXCLUSION_PATTERN = + Pattern.compile("Reduce(Expressions|Values)Rule.*"); + + /** + * Singleton rule that reduces constants inside a + * {@link org.apache.calcite.rel.logical.HiveFilter}. + */ + public static final HiveReduceExpressionsRule FILTER_INSTANCE = + new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER); + + /** + * Singleton rule that reduces constants inside a + * {@link org.apache.calcite.rel.logical.HiveProject}. + */ + public static final HiveReduceExpressionsRule PROJECT_INSTANCE = + new ProjectReduceExpressionsRule(HiveProject.class, HiveRelFactories.HIVE_BUILDER); + + /** + * Singleton rule that reduces constants inside a + * {@link org.apache.calcite.rel.core.HiveJoin}. + */ + public static final HiveReduceExpressionsRule JOIN_INSTANCE = + new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER); + + /** + * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}. + * If the condition is a constant, the filter is removed (if TRUE) or replaced with + * an empty {@link org.apache.calcite.rel.core.Values} (if FALSE or NULL). + */ + public static class FilterReduceExpressionsRule extends HiveReduceExpressionsRule { + + public FilterReduceExpressionsRule(Class<? extends Filter> filterClass, + RelBuilderFactory relBuilderFactory) { + super(filterClass, relBuilderFactory, "HiveReduceExpressionsRule(Filter)"); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Filter filter = call.rel(0); + final List<RexNode> expList = + Lists.newArrayList(filter.getCondition()); + RexNode newConditionExp; + boolean reduced; + final RelOptPredicateList predicates = + RelMetadataQuery.getPulledUpPredicates(filter.getInput()); + if (reduceExpressions(filter, expList, predicates)) { + assert expList.size() == 1; + newConditionExp = expList.get(0); + reduced = true; + } else { + // No reduction, but let's still test the original + // predicate to see if it was already a constant, + // in which case we don't need any runtime decision + // about filtering. + newConditionExp = filter.getCondition(); + reduced = false; + } + if (newConditionExp.isAlwaysTrue()) { + call.transformTo( + filter.getInput()); + } + // TODO: support LogicalValues + else if (newConditionExp instanceof RexLiteral + || RexUtil.isNullLiteral(newConditionExp, true)) { + // call.transformTo(call.builder().values(filter.getRowType()).build()); + return; + } + else if (reduced) { + call.transformTo(call.builder(). + push(filter.getInput()).filter(expList.get(0)).build()); + } else { + if (newConditionExp instanceof RexCall) { + RexCall rexCall = (RexCall) newConditionExp; + boolean reverse = + rexCall.getOperator() + == SqlStdOperatorTable.NOT; + if (reverse) { + rexCall = (RexCall) rexCall.getOperands().get(0); + } + reduceNotNullableFilter(call, filter, rexCall, reverse); + } + return; + } + + // New plan is absolutely better than old plan. + call.getPlanner().setImportance(filter, 0.0); + } + + private void reduceNotNullableFilter( + RelOptRuleCall call, + Filter filter, + RexCall rexCall, + boolean reverse) { + // If the expression is a IS [NOT] NULL on a non-nullable + // column, then we can either remove the filter or replace + // it with an Empty. + boolean alwaysTrue; + switch (rexCall.getKind()) { + case IS_NULL: + case IS_UNKNOWN: + alwaysTrue = false; + break; + case IS_NOT_NULL: + alwaysTrue = true; + break; + default: + return; + } + if (reverse) { + alwaysTrue = !alwaysTrue; + } + RexNode operand = rexCall.getOperands().get(0); + if (operand instanceof RexInputRef) { + RexInputRef inputRef = (RexInputRef) operand; + if (!inputRef.getType().isNullable()) { + if (alwaysTrue) { + call.transformTo(filter.getInput()); + } else { + // TODO: support LogicalValues + // call.transformTo(call.builder().values(filter.getRowType()).build()); + return; + } + } + } + } + } + + /** + * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}. + */ + public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule { + + public ProjectReduceExpressionsRule(Class<? extends Project> projectClass, + RelBuilderFactory relBuilderFactory) { + super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)"); + } + + public boolean matches(RelOptRuleCall call) { + Project project = call.rel(0); + HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); + + // If this operator has been visited already by the rule, + // we do not need to apply the optimization + if (registry != null && registry.getVisited(this).contains(project)) { + return false; + } + + return true; + } + + @Override public void onMatch(RelOptRuleCall call) { + Project project = call.rel(0); + // Register that we have visited this operator in this rule + HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); + if (registry != null) { + registry.registerVisited(this, project); + } + final RelOptPredicateList predicates = + RelMetadataQuery.getPulledUpPredicates(project.getInput()); + final List<RexNode> expList = + Lists.newArrayList(project.getProjects()); + if (reduceExpressions(project, expList, predicates)) { + RelNode newProject = call.builder().push(project.getInput()) + .project(expList, project.getRowType().getFieldNames()).build(); + if (registry != null) { + registry.registerVisited(this, newProject); + } + call.transformTo(newProject); + + // New plan is absolutely better than old plan. + call.getPlanner().setImportance(project, 0.0); + } + } + } + + /** + * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}. + */ + public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule { + + public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass, + RelBuilderFactory relBuilderFactory) { + super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)"); + } + + @Override public void onMatch(RelOptRuleCall call) { + final HiveJoin join = call.rel(0); + final List<RexNode> expList = Lists.newArrayList(join.getCondition()); + final int fieldCount = join.getLeft().getRowType().getFieldCount(); + final RelOptPredicateList leftPredicates = + RelMetadataQuery.getPulledUpPredicates(join.getLeft()); + final RelOptPredicateList rightPredicates = + RelMetadataQuery.getPulledUpPredicates(join.getRight()); + final RelOptPredicateList predicates = + leftPredicates.union(rightPredicates.shift(fieldCount)); + if (!reduceExpressions(join, expList, predicates)) { + return; + } + final JoinInfo joinInfo = JoinInfo.of(join.getLeft(), join.getRight(), expList.get(0)); + if (!joinInfo.isEqui()) { + // This kind of join must be an equi-join, and the condition is + // no longer an equi-join. SemiJoin is an example of this. + return; + } + call.transformTo( + join.copy( + join.getTraitSet(), + expList.get(0), + join.getLeft(), + join.getRight(), + join.getJoinType(), + join.isSemiJoinDone())); + + // New plan is absolutely better than old plan. + call.getPlanner().setImportance(join, 0.0); + } + } + + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a HiveReduceExpressionsRule. + * + * @param clazz class of rels to which this rule should apply + */ + protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz, + RelBuilderFactory relBuilderFactory, String desc) { + super(operand(clazz, any()), relBuilderFactory, desc); + } + + //~ Methods ---------------------------------------------------------------- + + /** + * Reduces a list of expressions. + * + * @param rel Relational expression + * @param expList List of expressions, modified in place + * @param predicates Constraints known to hold on input expressions + * @return whether reduction found something to change, and succeeded + */ + protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList, + RelOptPredicateList predicates) { + RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + // Replace predicates on CASE to CASE on predicates. + new CaseShuttle().mutate(expList); + + // Find reducible expressions. + final List<RexNode> constExps = Lists.newArrayList(); + List<Boolean> addCasts = Lists.newArrayList(); + final List<RexNode> removableCasts = Lists.newArrayList(); + final ImmutableMap<RexNode, RexLiteral> constants = + predicateConstants(predicates); + findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants, + constExps, addCasts, removableCasts); + if (constExps.isEmpty() && removableCasts.isEmpty()) { + return false; + } + + // Remove redundant casts before reducing constant expressions. + // If the argument to the redundant cast is a reducible constant, + // reducing that argument to a constant first will result in not being + // able to locate the original cast expression. + if (!removableCasts.isEmpty()) { + final List<RexNode> reducedExprs = Lists.newArrayList(); + for (RexNode exp : removableCasts) { + RexCall call = (RexCall) exp; + reducedExprs.add(call.getOperands().get(0)); + } + RexReplacer replacer = + new RexReplacer( + rexBuilder, + removableCasts, + reducedExprs, + Collections.nCopies(removableCasts.size(), false)); + replacer.mutate(expList); + } + + if (constExps.isEmpty()) { + return true; + } + + final List<RexNode> constExps2 = Lists.newArrayList(constExps); + if (!constants.isEmpty()) { + //noinspection unchecked + final List<Map.Entry<RexNode, RexNode>> pairs = + (List<Map.Entry<RexNode, RexNode>>) (List) + Lists.newArrayList(constants.entrySet()); + RexReplacer replacer = + new RexReplacer( + rexBuilder, + Pair.left(pairs), + Pair.right(pairs), + Collections.nCopies(pairs.size(), false)); + replacer.mutate(constExps2); + } + + // Compute the values they reduce to. + RelOptPlanner.Executor executor = + rel.getCluster().getPlanner().getExecutor(); + if (executor == null) { + // Cannot reduce expressions: caller has not set an executor in their + // environment. Caller should execute something like the following before + // invoking the planner: + // + // final RexExecutorImpl executor = + // new RexExecutorImpl(Schemas.createDataContext(null)); + // rootRel.getCluster().getPlanner().setExecutor(executor); + return false; + } + + final List<RexNode> reducedValues = Lists.newArrayList(); + executor.reduce(rexBuilder, constExps2, reducedValues); + + // For Project, we have to be sure to preserve the result + // types, so always cast regardless of the expression type. + // For other RelNodes like Filter, in general, this isn't necessary, + // and the presence of casts could hinder other rules such as sarg + // analysis, which require bare literals. But there are special cases, + // like when the expression is a UDR argument, that need to be + // handled as special cases. + if (rel instanceof Project) { + addCasts = Collections.nCopies(reducedValues.size(), true); + } + + RexReplacer replacer = + new RexReplacer( + rexBuilder, + constExps, + reducedValues, + addCasts); + replacer.mutate(expList); + return true; + } + + /** + * Locates expressions that can be reduced to literals or converted to + * expressions with redundant casts removed. + * + * @param typeFactory Type factory + * @param exps list of candidate expressions to be examined for + * reduction + * @param constants List of expressions known to be constant + * @param constExps returns the list of expressions that can be constant + * reduced + * @param addCasts indicator for each expression that can be constant + * reduced, whether a cast of the resulting reduced + * expression is potentially necessary + * @param removableCasts returns the list of cast expressions where the cast + */ + protected static void findReducibleExps(RelDataTypeFactory typeFactory, + List<RexNode> exps, ImmutableMap<RexNode, RexLiteral> constants, + List<RexNode> constExps, List<Boolean> addCasts, + List<RexNode> removableCasts) { + ReducibleExprLocator gardener = + new ReducibleExprLocator(typeFactory, constants, constExps, + addCasts, removableCasts); + for (RexNode exp : exps) { + gardener.analyze(exp); + } + assert constExps.size() == addCasts.size(); + } + + protected static ImmutableMap<RexNode, RexLiteral> predicateConstants( + RelOptPredicateList predicates) { + // We cannot use an ImmutableMap.Builder here. If there are multiple entries + // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't + // matter which we take, so the latter will replace the former. + // The basic idea is to find all the pairs of RexNode = RexLiteral + // (1) If 'predicates' contain a non-EQUALS, we bail out. + // (2) It is OK if a RexNode is equal to the same RexLiteral several times, + // (e.g. "WHERE deptno = 1 AND deptno = 1") + // (3) It will return false if there are inconsistent constraints (e.g. + // "WHERE deptno = 1 AND deptno = 2") + final Map<RexNode, RexLiteral> map = new HashMap<>(); + final Set<RexNode> excludeSet = new HashSet<>(); + for (RexNode predicate : predicates.pulledUpPredicates) { + gatherConstraints(map, predicate, excludeSet); + } + final ImmutableMap.Builder<RexNode, RexLiteral> builder = + ImmutableMap.builder(); + for (Map.Entry<RexNode, RexLiteral> entry : map.entrySet()) { + RexNode rexNode = entry.getKey(); + if (!overlap(rexNode, excludeSet)) { + builder.put(rexNode, entry.getValue()); + } + } + return builder.build(); + } + + private static boolean overlap(RexNode rexNode, Set<RexNode> set) { + if (rexNode instanceof RexCall) { + for (RexNode r : ((RexCall) rexNode).getOperands()) { + if (overlap(r, set)) { + return true; + } + } + return false; + } else { + return set.contains(rexNode); + } + } + + /** Tries to decompose the RexNode which is a RexCall into non-literal + * RexNodes. */ + private static void decompose(Set<RexNode> set, RexNode rexNode) { + if (rexNode instanceof RexCall) { + for (RexNode r : ((RexCall) rexNode).getOperands()) { + decompose(set, r); + } + } else if (!(rexNode instanceof RexLiteral)) { + set.add(rexNode); + } + } + + private static void gatherConstraints(Map<RexNode, RexLiteral> map, + RexNode predicate, Set<RexNode> excludeSet) { + if (predicate.getKind() != SqlKind.EQUALS) { + decompose(excludeSet, predicate); + return; + } + final List<RexNode> operands = ((RexCall) predicate).getOperands(); + if (operands.size() != 2) { + decompose(excludeSet, predicate); + return; + } + // if it reaches here, we have rexNode equals rexNode + final RexNode left = operands.get(0); + final RexNode right = operands.get(1); + // note that literals are immutable too and they can only be compared through + // values. + if (right instanceof RexLiteral && !excludeSet.contains(left)) { + RexLiteral existedValue = map.get(left); + if (existedValue == null) { + map.put(left, (RexLiteral) right); + } else { + if (!existedValue.getValue().equals(((RexLiteral) right).getValue())) { + // we found conflict values. + map.remove(left); + excludeSet.add(left); + } + } + } else if (left instanceof RexLiteral && !excludeSet.contains(right)) { + RexLiteral existedValue = map.get(right); + if (existedValue == null) { + map.put(right, (RexLiteral) left); + } else { + if (!existedValue.getValue().equals(((RexLiteral) left).getValue())) { + map.remove(right); + excludeSet.add(right); + } + } + } + } + + /** Pushes predicates into a CASE. + * + * <p>We have a loose definition of 'predicate': any boolean expression will + * do, except CASE. For example '(CASE ...) = 5' or '(CASE ...) IS NULL'. + */ + protected static RexCall pushPredicateIntoCase(RexCall call) { + if (call.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { + return call; + } + switch (call.getKind()) { + case CASE: + case AND: + case OR: + return call; // don't push CASE into CASE! + } + int caseOrdinal = -1; + final List<RexNode> operands = call.getOperands(); + for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + switch (operand.getKind()) { + case CASE: + caseOrdinal = i; + } + } + if (caseOrdinal < 0) { + return call; + } + // Convert + // f(CASE WHEN p1 THEN v1 ... END, arg) + // to + // CASE WHEN p1 THEN f(v1, arg) ... END + final RexCall case_ = (RexCall) operands.get(caseOrdinal); + final List<RexNode> nodes = new ArrayList<>(); + for (int i = 0; i < case_.getOperands().size(); i++) { + RexNode node = case_.getOperands().get(i); + if (!RexUtil.isCasePredicate(case_, i)) { + node = substitute(call, caseOrdinal, node); + } + nodes.add(node); + } + return case_.clone(call.getType(), nodes); + } + + /** Converts op(arg0, ..., argOrdinal, ..., argN) to op(arg0,..., node, ..., argN). */ + protected static RexNode substitute(RexCall call, int ordinal, RexNode node) { + final List<RexNode> newOperands = Lists.newArrayList(call.getOperands()); + newOperands.set(ordinal, node); + return call.clone(call.getType(), newOperands); + } + + //~ Inner Classes ---------------------------------------------------------- + + /** + * Replaces expressions with their reductions. Note that we only have to + * look for RexCall, since nothing else is reducible in the first place. + */ + protected static class RexReplacer extends RexShuttle { + private final RexBuilder rexBuilder; + private final List<RexNode> reducibleExps; + private final List<RexNode> reducedValues; + private final List<Boolean> addCasts; + + RexReplacer( + RexBuilder rexBuilder, + List<RexNode> reducibleExps, + List<RexNode> reducedValues, + List<Boolean> addCasts) { + this.rexBuilder = rexBuilder; + this.reducibleExps = reducibleExps; + this.reducedValues = reducedValues; + this.addCasts = addCasts; + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + RexNode node = visit(inputRef); + if (node == null) { + return super.visitInputRef(inputRef); + } + return node; + } + + @Override public RexNode visitCall(RexCall call) { + RexNode node = visit(call); + if (node != null) { + return node; + } + node = super.visitCall(call); + if (node != call) { + node = RexUtil.simplify(rexBuilder, node); + } + return node; + } + + private RexNode visit(final RexNode call) { + int i = reducibleExps.indexOf(call); + if (i == -1) { + return null; + } + RexNode replacement = reducedValues.get(i); + if (addCasts.get(i) + && (replacement.getType() != call.getType())) { + // Handle change from nullable to NOT NULL by claiming + // that the result is still nullable, even though + // we know it isn't. + // + // Also, we cannot reduce CAST('abc' AS VARCHAR(4)) to 'abc'. + // If we make 'abc' of type VARCHAR(4), we may later encounter + // the same expression in a Project's digest where it has + // type VARCHAR(3), and that's wrong. + replacement = rexBuilder.makeAbstractCast(call.getType(), replacement); + } + return replacement; + } + } + + /** + * Helper class used to locate expressions that either can be reduced to + * literals or contain redundant casts. + */ + protected static class ReducibleExprLocator extends RexVisitorImpl<Void> { + /** Whether an expression is constant, and if so, whether it can be + * reduced to a simpler constant. */ + enum Constancy { + NON_CONSTANT, REDUCIBLE_CONSTANT, IRREDUCIBLE_CONSTANT + } + + private final RelDataTypeFactory typeFactory; + + private final List<Constancy> stack; + + private final ImmutableMap<RexNode, RexLiteral> constants; + + private final List<RexNode> constExprs; + + private final List<Boolean> addCasts; + + private final List<RexNode> removableCasts; + + private final List<SqlOperator> parentCallTypeStack; + + ReducibleExprLocator(RelDataTypeFactory typeFactory, + ImmutableMap<RexNode, RexLiteral> constants, List<RexNode> constExprs, + List<Boolean> addCasts, List<RexNode> removableCasts) { + // go deep + super(true); + this.typeFactory = typeFactory; + this.constants = constants; + this.constExprs = constExprs; + this.addCasts = addCasts; + this.removableCasts = removableCasts; + this.stack = Lists.newArrayList(); + this.parentCallTypeStack = Lists.newArrayList(); + } + + public void analyze(RexNode exp) { + assert stack.isEmpty(); + + exp.accept(this); + + // Deal with top of stack + assert stack.size() == 1; + assert parentCallTypeStack.isEmpty(); + Constancy rootConstancy = stack.get(0); + if (rootConstancy == Constancy.REDUCIBLE_CONSTANT) { + // The entire subtree was constant, so add it to the result. + addResult(exp); + } + stack.clear(); + } + + private Void pushVariable() { + stack.add(Constancy.NON_CONSTANT); + return null; + } + + private void addResult(RexNode exp) { + // Cast of literal can't be reduced, so skip those (otherwise we'd + // go into an infinite loop as we add them back). + if (exp.getKind() == SqlKind.CAST) { + RexCall cast = (RexCall) exp; + RexNode operand = cast.getOperands().get(0); + if (operand instanceof RexLiteral) { + return; + } + } + constExprs.add(exp); + + // In the case where the expression corresponds to a UDR argument, + // we need to preserve casts. Note that this only applies to + // the topmost argument, not expressions nested within the UDR + // call. + // + // REVIEW zfong 6/13/08 - Are there other expressions where we + // also need to preserve casts? + if (parentCallTypeStack.isEmpty()) { + addCasts.add(false); + } else { + addCasts.add(isUdf(Stacks.peek(parentCallTypeStack))); + } + } + + private Boolean isUdf(SqlOperator operator) { + // return operator instanceof UserDefinedRoutine + return false; + } + + public Void visitInputRef(RexInputRef inputRef) { + if (constants.containsKey(inputRef)) { + stack.add(Constancy.REDUCIBLE_CONSTANT); + return null; + } + return pushVariable(); + } + + public Void visitLiteral(RexLiteral literal) { + stack.add(Constancy.IRREDUCIBLE_CONSTANT); + return null; + } + + public Void visitOver(RexOver over) { + // assume non-constant (running SUM(1) looks constant but isn't) + analyzeCall(over, Constancy.NON_CONSTANT); + return null; + } + + public Void visitCorrelVariable(RexCorrelVariable correlVariable) { + return pushVariable(); + } + + public Void visitCall(RexCall call) { + // assume REDUCIBLE_CONSTANT until proven otherwise + analyzeCall(call, Constancy.REDUCIBLE_CONSTANT); + return null; + } + + private void analyzeCall(RexCall call, Constancy callConstancy) { + Stacks.push(parentCallTypeStack, call.getOperator()); + + // visit operands, pushing their states onto stack + super.visitCall(call); + + // look for NON_CONSTANT operands + int operandCount = call.getOperands().size(); + List<Constancy> operandStack = Util.last(stack, operandCount); + for (Constancy operandConstancy : operandStack) { + if (operandConstancy == Constancy.NON_CONSTANT) { + callConstancy = Constancy.NON_CONSTANT; + } + } + + // Even if all operands are constant, the call itself may + // be non-deterministic. + if (!call.getOperator().isDeterministic()) { + callConstancy = Constancy.NON_CONSTANT; + } else if (call.getOperator().isDynamicFunction()) { + // We can reduce the call to a constant, but we can't + // cache the plan if the function is dynamic. + // For now, treat it same as non-deterministic. + callConstancy = Constancy.NON_CONSTANT; + } + + // Row operator itself can't be reduced to a literal, but if + // the operands are constants, we still want to reduce those + if ((callConstancy == Constancy.REDUCIBLE_CONSTANT) + && (call.getOperator() instanceof SqlRowOperator)) { + callConstancy = Constancy.NON_CONSTANT; + } + + if (callConstancy == Constancy.NON_CONSTANT) { + // any REDUCIBLE_CONSTANT children are now known to be maximal + // reducible subtrees, so they can be added to the result + // list + for (int iOperand = 0; iOperand < operandCount; ++iOperand) { + Constancy constancy = operandStack.get(iOperand); + if (constancy == Constancy.REDUCIBLE_CONSTANT) { + addResult(call.getOperands().get(iOperand)); + } + } + + // if this cast expression can't be reduced to a literal, + // then see if we can remove the cast + if (call.getOperator() == SqlStdOperatorTable.CAST) { + reduceCasts(call); + } + } + + // pop operands off of the stack + operandStack.clear(); + + // pop this parent call operator off the stack + Stacks.pop(parentCallTypeStack, call.getOperator()); + + // push constancy result for this call onto stack + stack.add(callConstancy); + } + + private void reduceCasts(RexCall outerCast) { + List<RexNode> operands = outerCast.getOperands(); + if (operands.size() != 1) { + return; + } + RelDataType outerCastType = outerCast.getType(); + RelDataType operandType = operands.get(0).getType(); + if (operandType.equals(outerCastType)) { + removableCasts.add(outerCast); + return; + } + + // See if the reduction + // CAST((CAST x AS type) AS type NOT NULL) + // -> CAST(x AS type NOT NULL) + // applies. TODO jvs 15-Dec-2008: consider + // similar cases for precision changes. + if (!(operands.get(0) instanceof RexCall)) { + return; + } + RexCall innerCast = (RexCall) operands.get(0); + if (innerCast.getOperator() != SqlStdOperatorTable.CAST) { + return; + } + if (innerCast.getOperands().size() != 1) { + return; + } + RelDataType outerTypeNullable = + typeFactory.createTypeWithNullability(outerCastType, true); + RelDataType innerTypeNullable = + typeFactory.createTypeWithNullability(operandType, true); + if (outerTypeNullable != innerTypeNullable) { + return; + } + if (operandType.isNullable()) { + removableCasts.add(innerCast); + } + } + + public Void visitDynamicParam(RexDynamicParam dynamicParam) { + return pushVariable(); + } + + public Void visitRangeRef(RexRangeRef rangeRef) { + return pushVariable(); + } + + public Void visitFieldAccess(RexFieldAccess fieldAccess) { + return pushVariable(); + } + } + + /** Shuttle that pushes predicates into a CASE. */ + protected static class CaseShuttle extends RexShuttle { + @Override public RexNode visitCall(RexCall call) { + for (;;) { + call = (RexCall) super.visitCall(call); + final RexCall old = call; + call = pushPredicateIntoCase(call); + if (call == old) { + return call; + } + } + } + } +} + +// End HiveReduceExpressionsRule.java http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index 425514d..d39744b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -150,11 +150,33 @@ class ASTBuilder { switch (sqlType) { case BINARY: - ByteString bs = (ByteString) literal.getValue(); - val = bs.byteAt(0); - type = HiveParser.BigintLiteral; + case DATE: + case TIME: + case TIMESTAMP: + case INTERVAL_YEAR_MONTH: + case INTERVAL_DAY_TIME: + if (literal.getValue() == null) { + return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node(); + } break; case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case DOUBLE: + case DECIMAL: + case FLOAT: + case REAL: + case VARCHAR: + case CHAR: + case BOOLEAN: + if (literal.getValue3() == null) { + return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node(); + } + } + + switch (sqlType) { + case TINYINT: if (useTypeQualInLiteral) { val = literal.getValue3() + "Y"; } else { @@ -244,6 +266,8 @@ class ASTBuilder { type = HiveParser.TOK_NULL; break; + //binary type should not be seen. + case BINARY: default: throw new RuntimeException("Unsupported Type: " + sqlType); } http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 631a4ca..122546f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -46,12 +46,9 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ConversionUtil; import org.apache.calcite.util.NlsString; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -69,8 +66,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate; @@ -91,8 +88,6 @@ import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; public class RexNodeConverter { - private static final Logger LOG = LoggerFactory.getLogger(RexNodeConverter.class); - private static class InputCtx { private final RelDataType calciteInpDataType; private final ImmutableMap<String, Integer> hiveNameToPosMap; @@ -112,6 +107,11 @@ public class RexNodeConverter { private final ImmutableList<InputCtx> inputCtxs; private final boolean flattenExpr; + //Constructor used by HiveRexExecutorImpl + public RexNodeConverter(RelOptCluster cluster) { + this(cluster, new ArrayList<InputCtx>(), false); + } + public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap<String, Integer> nameToPosMap, int offset, boolean flattenExpr) { this.cluster = cluster; @@ -259,6 +259,9 @@ public class RexNodeConverter { GenericUDF udf = func.getGenericUDF(); if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar) || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate) + // Calcite can not specify the scale for timestamp. As a result, all + // the millisecond part will be lost + || (udf instanceof GenericUDFTimestamp) || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { castExpr = cluster.getRexBuilder().makeAbstractCast( TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()), @@ -321,6 +324,10 @@ public class RexNodeConverter { coi); RexNode calciteLiteral = null; + // If value is null, the type should also be VOID. + if (value == null) { + hiveTypeCategory = PrimitiveCategory.VOID; + } // TODO: Verify if we need to use ConstantObjectInspector to unwrap data switch (hiveTypeCategory) { case BOOLEAN: @@ -378,6 +385,10 @@ public class RexNodeConverter { calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), calciteDataType); break; case DOUBLE: + // TODO: The best solution is to support NaN in expression reduction. + if (Double.isNaN((Double) value)) { + throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal); + } calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), calciteDataType); break; case CHAR: @@ -417,14 +428,22 @@ public class RexNodeConverter { new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1,1))); break; case INTERVAL_DAY_TIME: + // Calcite RexBuilder L525 divides value by the multiplier. + // Need to get CAlCITE-1020 in. + throw new CalciteSemanticException("INTERVAL_DAY_TIME is not well supported", + UnsupportedFeature.Invalid_interval); // Calcite day-time interval is millis value as BigDecimal // Seconds converted to millis - BigDecimal secsValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000); - // Nanos converted to millis - BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6); - calciteLiteral = rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd), - new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, new SqlParserPos(1,1))); - break; + // BigDecimal secsValueBd = BigDecimal + // .valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000); + // // Nanos converted to millis + // BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) + // value).getNanos(), 6); + // calciteLiteral = + // rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd), + // new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new + // SqlParserPos(1, 1))); + // break; case VOID: calciteLiteral = cluster.getRexBuilder().makeLiteral(null, cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true); @@ -438,11 +457,6 @@ public class RexNodeConverter { return calciteLiteral; } - private RexNode createNullLiteral(ExprNodeDesc expr) throws CalciteSemanticException { - return cluster.getRexBuilder().makeNullLiteral( - TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName()); - } - public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode, List<RelNode> inputRels, LinkedHashMap<RelNode, RowResolver> relToHiveRR, Map<RelNode, ImmutableMap<String, Integer>> relToHiveColNameCalcitePosMap, boolean flattenExpr) http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 87b18b7..21423c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -40,6 +40,7 @@ import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptPlanner.Executor; import org.apache.calcite.plan.RelOptQuery; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptSchema; @@ -119,6 +120,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -154,6 +156,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule; @@ -881,9 +884,12 @@ public class CalcitePlanner extends SemanticAnalyzer { // Create MD provider HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf); + // Create executor + Executor executorProvider = new HiveRexExecutorImpl(cluster); + // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, - mdProvider.getMetadataProvider()); + mdProvider.getMetadataProvider(), executorProvider); // 3. Apply join order optimizations: reordering MST algorithm // If join optimizations failed because of missing stats, we continue with @@ -935,7 +941,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // 4. Run other optimizations that do not need stats perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE, new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY), @@ -979,7 +985,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // aggregation columns (HIVE-10627) if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) { perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Window fixing rule"); } @@ -988,7 +994,7 @@ public class CalcitePlanner extends SemanticAnalyzer { if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // 8.1. Merge join into multijoin operators (if possible) - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER, HiveJoinProjectTransposeRule.LEFT_PROJECT_INCLUDE_OUTER, HiveJoinProjectTransposeRule.RIGHT_PROJECT_INCLUDE_OUTER, @@ -998,15 +1004,15 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveRelFactories.HIVE_BUILDER.create(cluster, null)); calciteOptimizedPlan = fieldTrimmer.trim(calciteOptimizedPlan); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY)); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, new HiveFilterProjectTSTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class)); // 8.2. Introduce exchange operators below join/multijoin operators - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree"); @@ -1031,9 +1037,11 @@ public class CalcitePlanner extends SemanticAnalyzer { * original plan * @param mdProvider * meta data provider + * @param executorProvider + * executor * @return */ - private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider) { + private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider, Executor executorProvider) { // TODO: Decorelation of subquery should be done before attempting // Partition Pruning; otherwise Expression evaluation may try to execute // corelated sub query. @@ -1048,7 +1056,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // Its not clear, if this rewrite is always performant on MR, since extra map phase // introduced for 2nd MR job may offset gains of this multi-stage aggregation. // We need a cost model for MR to enable this on MR. - basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE); + basePlan = hepPlan(basePlan, true, mdProvider, null, HiveExpandDistinctAggregatesRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Distinct aggregate rewrite"); } @@ -1059,7 +1067,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // Ex: select * from R1 join R2 where ((R1.x=R2.x) and R1.y<10) or // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1 perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, false, mdProvider, HepMatchOrder.ARBITRARY, + basePlan = hepPlan(basePlan, false, mdProvider, null, HepMatchOrder.ARBITRARY, HivePreFilteringRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, factor out common filter elements and separating deterministic vs non-deterministic UDF"); @@ -1070,7 +1078,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // TODO: Add in ReduceExpressionrules (Constant folding) to below once // HIVE-11927 is fixed. perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC, + basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC, HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN, HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( @@ -1078,6 +1086,7 @@ public class CalcitePlanner extends SemanticAnalyzer { perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, PPD for old join syntax"); + // TODO: Transitive inference, constant prop & Predicate push down has to // do multiple passes till no more inference is left // Currently doing so would result in a spin. Just checking for if inferred @@ -1085,7 +1094,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // could have been mutated by constant folding/prop // 4. Transitive inference for join on clauses perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, new HiveJoinPushTransitivePredicatesRule( + basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveJoinPushTransitivePredicatesRule( Join.class, HiveRelFactories.HIVE_FILTER_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Transitive inference for join on clauses"); @@ -1102,10 +1111,10 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_PERCENTAGE); final long reductionTuples = HiveConf.getLongVar(conf, HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_TUPLES); - basePlan = hepPlan(basePlan, true, mdProvider, HiveSortMergeRule.INSTANCE, + basePlan = hepPlan(basePlan, true, mdProvider, null, HiveSortMergeRule.INSTANCE, HiveSortProjectTransposeRule.INSTANCE, HiveSortJoinReduceRule.INSTANCE, HiveSortUnionReduceRule.INSTANCE); - basePlan = hepPlan(basePlan, true, mdProvider, HepMatchOrder.BOTTOM_UP, + basePlan = hepPlan(basePlan, true, mdProvider, null, HepMatchOrder.BOTTOM_UP, new HiveSortRemoveRule(reductionProportion, reductionTuples), HiveProjectSortTransposeRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, @@ -1114,7 +1123,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // 6. Add not null filters perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE); + basePlan = hepPlan(basePlan, true, mdProvider, null, HiveJoinAddNotNullRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Add not null filters"); @@ -1122,7 +1131,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // TODO: Add in ReduceExpressionrules (Constant folding) to below once // HIVE-11927 is fixed. perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC, + basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC, HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN, HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( @@ -1132,18 +1141,26 @@ public class CalcitePlanner extends SemanticAnalyzer { // 8. Push Down Semi Joins perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE, + basePlan = hepPlan(basePlan, true, mdProvider, null, SemiJoinJoinTransposeRule.INSTANCE, SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Push Down Semi Joins"); - // 9. Apply Partition Pruning + // 9. Constant folding + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, + HiveReduceExpressionsRule.PROJECT_INSTANCE, HiveReduceExpressionsRule.FILTER_INSTANCE, + HiveReduceExpressionsRule.JOIN_INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Constant folding"); + + // 10. Apply Partition Pruning perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, false, mdProvider, new HivePartitionPruneRule(conf)); + basePlan = hepPlan(basePlan, false, mdProvider, null, new HivePartitionPruneRule(conf)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Partition Pruning"); - // 10. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP) + // 11. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP) perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveRelFactories.HIVE_BUILDER.create(cluster, null)); @@ -1151,19 +1168,19 @@ public class CalcitePlanner extends SemanticAnalyzer { perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Projection Pruning"); - // 11. Merge Project-Project if possible + // 12. Merge Project-Project if possible perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, false, mdProvider, new ProjectMergeRule(true, + basePlan = hepPlan(basePlan, false, mdProvider, null, new ProjectMergeRule(true, HiveRelFactories.HIVE_PROJECT_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Merge Project-Project"); - // 12. Rerun PPD through Project as column pruning would have introduced + // 13. Rerun PPD through Project as column pruning would have introduced // DT above scans; By pushing filter just above TS, Hive can push it into // storage (incase there are filters on non partition cols). This only // matches FIL-PROJ-TS perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTSTransposeRule( + basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveFilterProjectTSTransposeRule( Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, @@ -1178,12 +1195,13 @@ public class CalcitePlanner extends SemanticAnalyzer { * @param basePlan * @param followPlanChanges * @param mdProvider + * @param executorProvider * @param rules * @return optimized RelNode */ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, - RelMetadataProvider mdProvider, RelOptRule... rules) { - return hepPlan(basePlan, followPlanChanges, mdProvider, + RelMetadataProvider mdProvider, Executor executorProvider, RelOptRule... rules) { + return hepPlan(basePlan, followPlanChanges, mdProvider, executorProvider, HepMatchOrder.TOP_DOWN, rules); } @@ -1193,12 +1211,14 @@ public class CalcitePlanner extends SemanticAnalyzer { * @param basePlan * @param followPlanChanges * @param mdProvider + * @param executorProvider * @param order * @param rules * @return optimized RelNode */ - private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, - HepMatchOrder order, RelOptRule... rules) { + private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, + RelMetadataProvider mdProvider, Executor executorProvider, HepMatchOrder order, + RelOptRule... rules) { RelNode optimizedRelNode = basePlan; HepProgramBuilder programBuilder = new HepProgramBuilder(); @@ -1214,6 +1234,7 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveRulesRegistry registry = new HiveRulesRegistry(); HiveHepPlannerContext context = new HiveHepPlannerContext(registry); HepPlanner planner = new HepPlanner(programBuilder.build(), context); + List<RelMetadataProvider> list = Lists.newArrayList(); list.add(mdProvider); planner.registerMetadataProviders(list); @@ -1221,10 +1242,9 @@ public class CalcitePlanner extends SemanticAnalyzer { basePlan.getCluster().setMetadataProvider( new CachingRelMetadataProvider(chainedProvider, planner)); - // Executor is required for constant-reduction rules; see [CALCITE-566] - final RexExecutorImpl executor = - new RexExecutorImpl(Schemas.createDataContext(null)); - basePlan.getCluster().getPlanner().setExecutor(executor); + if (executorProvider != null) { + basePlan.getCluster().getPlanner().setExecutor(executorProvider); + } planner.setRoot(basePlan); optimizedRelNode = planner.findBestExp(); http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/cbo_const.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/cbo_const.q b/ql/src/test/queries/clientpositive/cbo_const.q new file mode 100644 index 0000000..e38e2da --- /dev/null +++ b/ql/src/test/queries/clientpositive/cbo_const.q @@ -0,0 +1,52 @@ +set hive.mapred.mode=nonstrict; +set hive.cbo.enable=true; + +select + interval_day_time('2 1:2:3'), + interval_day_time(cast('2 1:2:3' as string)), + interval_day_time(cast('2 1:2:3' as varchar(10))), + interval_day_time(cast('2 1:2:3' as char(10))), + interval_day_time('2 1:2:3') = interval '2 1:2:3' day to second +from src limit 1; + +select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08'; + +drop view t1; + +create table t1_new (key string, value string) partitioned by (ds string); + +insert overwrite table t1_new partition (ds = '2011-10-15') +select 'key1', 'value1' from src tablesample (1 rows); + +insert overwrite table t1_new partition (ds = '2011-10-16') +select 'key2', 'value2' from src tablesample (1 rows); + +create view t1 partitioned on (ds) as +select * from +( +select key, value, ds from t1_new +union all +select key, value, ds from t1_new +)subq; + +select * from t1 where ds = '2011-10-15'; + + +explain select array(1,2,3) from src; + +EXPLAIN +select key from (SELECT key from src where key = 1+3)s; + +select * from (select key from src where key = '1')subq; + +select '1'; + +select * from (select '1')subq; + +select * from (select key from src where false)subq; + +EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3); + http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/constantfolding.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/constantfolding.q b/ql/src/test/queries/clientpositive/constantfolding.q new file mode 100644 index 0000000..4ddb710 --- /dev/null +++ b/ql/src/test/queries/clientpositive/constantfolding.q @@ -0,0 +1,88 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.ppd=true; + +-- SORT_QUERY_RESULTS + +select * from (select 'k2' as key, '1 ' as value from src limit 2)b +union all +select * from (select 'k3' as key, '' as value from src limit 2)b +union all +select * from (select 'k4' as key, ' ' as value from src limit 2)c; + + +drop table if exists union_all_bug_test_1; +drop table if exists union_all_bug_test_2; +create table if not exists union_all_bug_test_1 +( +f1 int, +f2 int +); + +create table if not exists union_all_bug_test_2 +( +f1 int +); + +insert into table union_all_bug_test_1 values (1,1); +insert into table union_all_bug_test_2 values (1); +insert into table union_all_bug_test_1 values (0,0); +insert into table union_all_bug_test_2 values (0); + + + +SELECT f1 +FROM ( + +SELECT +f1 +, if('helloworld' like '%hello%' ,f1,f2) as filter +FROM union_all_bug_test_1 + +union all + +select +f1 +, 0 as filter +from union_all_bug_test_2 +) A +WHERE (filter = 1 and f1 = 1); + + +select percentile(cast(key as bigint), array()) from src where false; + +select unbase64("0xe23") from src limit 1; + +SELECT key,randum123, h4 +FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a +WHERE a.h4 <= 3 limit 1; + +select null from src limit 1; + +-- numRows: 2 rawDataSize: 80 +explain select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from src; + +-- numRows: 2 rawDataSize: 112 +explain select cast("1970-12-31 15:59:58.174" as DATE) from src; + +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; + +FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86; + +EXPLAIN +SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), + LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1), + ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), + LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), + POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1; + +SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), + LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1), + ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), + LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), + POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1; http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/annotate_stats_select.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/annotate_stats_select.q.out b/ql/src/test/results/clientpositive/annotate_stats_select.q.out index c4d59c8..b158d85 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_select.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_select.q.out @@ -925,24 +925,46 @@ POSTHOOK: query: -- inner select - numRows: 2 rawDataSize: 24 explain select x from (select i1,11.0 as x from alltypes_orc limit 10) temp POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 10 - Processor Tree: - TableScan - alias: alltypes_orc - Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypes_orc + Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Reduce Operator Tree: + Limit + Number of rows: 10 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 11.0 (type: double) outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - ListSink + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink PREHOOK: query: -- inner select - numRows: 2 rawDataSize: 104 -- outer select - numRows: 2 rawDataSize: 186 @@ -1024,21 +1046,21 @@ STAGE PLANS: alias: alltypes_orc Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Limit Number of rows: 10 - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -1052,12 +1074,12 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Limit Number of rows: 10 - Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'hello' (type: string), 11.0 (type: double) outputColumnNames: _col0, _col1 http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out index 277b0f7..cfb95be 100644 --- a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out +++ b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out @@ -22,6 +22,8 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@T2 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM ( SELECT tmp1.name as name FROM ( SELECT name, 'MMM' AS n FROM T1) tmp1 http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/cast1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/cast1.q.out b/ql/src/test/results/clientpositive/cast1.q.out index 0bdecba..48a0c14 100644 --- a/ql/src/test/results/clientpositive/cast1.q.out +++ b/ql/src/test/results/clientpositive/cast1.q.out @@ -105,11 +105,11 @@ POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1.c5 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] +POSTHOOK: Lineage: dest1.c2 SIMPLE [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] +POSTHOOK: Lineage: dest1.c4 SIMPLE [] +POSTHOOK: Lineage: dest1.c5 SIMPLE [] POSTHOOK: Lineage: dest1.c6 EXPRESSION [] POSTHOOK: Lineage: dest1.c7 EXPRESSION [] PREHOOK: query: select dest1.* FROM dest1