http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java index f81c21b..49e4bec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptUtil; @@ -32,17 +33,28 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; //TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand public class HiveFilterProjectTSTransposeRule extends RelOptRule { + public final static HiveFilterProjectTSTransposeRule INSTANCE = + new HiveFilterProjectTSTransposeRule( + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class); + + public final static HiveFilterProjectTSTransposeRule INSTANCE_DRUID = + new HiveFilterProjectTSTransposeRule( + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class); + private final RelFactories.FilterFactory filterFactory; private final RelFactories.ProjectFactory projectFactory; - public HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass, + private HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass, FilterFactory filterFactory, Class<? extends Project> projectClass, - ProjectFactory projectFactory, Class<? extends TableScan> tsClass) { + ProjectFactory projectFactory, Class<? extends RelNode> tsClass) { super(operand(filterClass, operand(projectClass, operand(tsClass, none())))); this.filterFactory = filterFactory; this.projectFactory = projectFactory;
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index d43c2c6..91d674d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -27,8 +27,6 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories.FilterFactory; -import org.apache.calcite.rel.core.RelFactories.ProjectFactory; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; @@ -37,6 +35,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; @@ -44,25 +43,25 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule { public static final HiveFilterProjectTransposeRule INSTANCE_DETERMINISTIC_WINDOWING = - new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, - HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, true, true); + new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class, + HiveRelFactories.HIVE_BUILDER, true, true); public static final HiveFilterProjectTransposeRule INSTANCE_DETERMINISTIC = - new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, - HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, true, false); + new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class, + HiveRelFactories.HIVE_BUILDER, true, false); public static final HiveFilterProjectTransposeRule INSTANCE = - new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, - HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, false, false); + new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class, + HiveRelFactories.HIVE_BUILDER, false, false); private final boolean onlyDeterministic; private final boolean pushThroughWindowing; private HiveFilterProjectTransposeRule(Class<? extends Filter> filterClass, - FilterFactory filterFactory, Class<? extends Project> projectClass, - ProjectFactory projectFactory, boolean onlyDeterministic,boolean pushThroughWindowing) { - super(filterClass, filterFactory, projectClass, projectFactory); + Class<? extends Project> projectClass, RelBuilderFactory relBuilderFactory, + boolean onlyDeterministic,boolean pushThroughWindowing) { + super(filterClass, projectClass, false, false, relBuilderFactory); this.onlyDeterministic = onlyDeterministic; this.pushThroughWindowing = pushThroughWindowing; } http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java index 7c2a7e5..dccb6a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java @@ -39,7 +39,6 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,9 +49,7 @@ import com.google.common.collect.Sets; public class HivePreFilteringRule extends RelOptRule { - protected static final Logger LOG = LoggerFactory - .getLogger(HivePreFilteringRule.class - .getName()); + protected static final Logger LOG = LoggerFactory.getLogger(HivePreFilteringRule.class); private static final Set<SqlKind> COMPARISON = EnumSet.of(SqlKind.EQUALS, SqlKind.GREATER_THAN_OR_EQUAL, @@ -209,7 +206,7 @@ public class HivePreFilteringRule extends RelOptRule { for (int i = 0; i < operands.size(); i++) { final RexNode operand = operands.get(i); - final RexNode operandCNF = HiveRexUtil.toCnf(rexBuilder, maxCNFNodeCount, operand); + final RexNode operandCNF = RexUtil.toCnf(rexBuilder, maxCNFNodeCount, operand); final List<RexNode> conjunctions = RelOptUtil.conjunctions(operandCNF); Set<String> refsInCurrentOperand = Sets.newHashSet(); http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 2fc68ae..a1b5aeb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,59 +16,28 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptPredicateList; -import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.ValuesReduceRule; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rel.rules.ReduceExpressionsRule; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexDynamicParam; -import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexOver; -import org.apache.calcite.rex.RexRangeRef; -import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.fun.SqlRowOperator; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.Pair; -import org.apache.calcite.util.Stacks; -import org.apache.calcite.util.Util; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil.ExprSimplifier; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; /** @@ -82,51 +51,55 @@ import com.google.common.collect.Lists; * is the same as the type of the resulting cast expression * </ul> */ -public abstract class HiveReduceExpressionsRule extends RelOptRule { +public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule { protected static final Logger LOG = LoggerFactory.getLogger(HiveReduceExpressionsRule.class); //~ Static fields/initializers --------------------------------------------- /** - * Regular expression that matches the description of all instances of this - * rule and {@link ValuesReduceRule} also. Use - * it to prevent the planner from invoking these rules. - */ - public static final Pattern EXCLUSION_PATTERN = - Pattern.compile("Reduce(Expressions|Values)Rule.*"); - - /** * Singleton rule that reduces constants inside a - * {@link org.apache.calcite.rel.logical.HiveFilter}. + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter}. */ - public static final HiveReduceExpressionsRule FILTER_INSTANCE = - new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER); + public static final ReduceExpressionsRule FILTER_INSTANCE = + new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER); /** * Singleton rule that reduces constants inside a - * {@link org.apache.calcite.rel.logical.HiveProject}. + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject}. */ - public static final HiveReduceExpressionsRule PROJECT_INSTANCE = + public static final ReduceExpressionsRule PROJECT_INSTANCE = new ProjectReduceExpressionsRule(HiveProject.class, HiveRelFactories.HIVE_BUILDER); /** * Singleton rule that reduces constants inside a - * {@link org.apache.calcite.rel.core.HiveJoin}. + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin}. */ - public static final HiveReduceExpressionsRule JOIN_INSTANCE = + public static final ReduceExpressionsRule JOIN_INSTANCE = new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER); + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a HiveReduceExpressionsRule. + * + * @param clazz class of rels to which this rule should apply + */ + protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz, + RelBuilderFactory relBuilderFactory, String desc) { + super(clazz, relBuilderFactory, desc); + } + /** * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}. * If the condition is a constant, the filter is removed (if TRUE) or replaced with * an empty {@link org.apache.calcite.rel.core.Values} (if FALSE or NULL). */ - public static class FilterReduceExpressionsRule extends HiveReduceExpressionsRule { + public static class FilterReduceExpressionsRule extends ReduceExpressionsRule { public FilterReduceExpressionsRule(Class<? extends Filter> filterClass, RelBuilderFactory relBuilderFactory) { - super(filterClass, relBuilderFactory, "HiveReduceExpressionsRule(Filter)"); + super(filterClass, relBuilderFactory, "ReduceExpressionsRule(Filter)"); } @Override public void onMatch(RelOptRuleCall call) { @@ -135,8 +108,9 @@ public abstract class HiveReduceExpressionsRule extends RelOptRule { Lists.newArrayList(filter.getCondition()); RexNode newConditionExp; boolean reduced; + final RelMetadataQuery mq = RelMetadataQuery.instance(); final RelOptPredicateList predicates = - RelMetadataQuery.instance().getPulledUpPredicates(filter.getInput()); + mq.getPulledUpPredicates(filter.getInput()); if (reduceExpressions(filter, expList, predicates, true)) { assert expList.size() == 1; newConditionExp = expList.get(0); @@ -154,795 +128,95 @@ public abstract class HiveReduceExpressionsRule extends RelOptRule { // predicate to see if it was already a constant, // in which case we don't need any runtime decision // about filtering. - // TODO: support LogicalValues if (newConditionExp.isAlwaysTrue()) { call.transformTo( filter.getInput()); } else if (reduced) { + if (RexUtil.isNullabilityCast(filter.getCluster().getTypeFactory(), + newConditionExp)) { + newConditionExp = ((RexCall) newConditionExp).getOperands().get(0); + } call.transformTo(call.builder(). push(filter.getInput()).filter(newConditionExp).build()); } else { + if (newConditionExp instanceof RexCall) { + RexCall rexCall = (RexCall) newConditionExp; + boolean reverse = rexCall.getKind() == SqlKind.NOT; + if (reverse) { + if (!(rexCall.getOperands().get(0) instanceof RexCall)) { + // If child is not a RexCall instance, we can bail out + return; + } + rexCall = (RexCall) rexCall.getOperands().get(0); + } + reduceNotNullableFilter(call, filter, rexCall, reverse); + } return; } // New plan is absolutely better than old plan. call.getPlanner().setImportance(filter, 0.0); } - } - - /** - * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}. - */ - public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule { - - public ProjectReduceExpressionsRule(Class<? extends Project> projectClass, - RelBuilderFactory relBuilderFactory) { - super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)"); - } - - @Override public void onMatch(RelOptRuleCall call) { - Project project = call.rel(0); - final RelOptPredicateList predicates = - RelMetadataQuery.instance().getPulledUpPredicates(project.getInput()); - final List<RexNode> expList = - Lists.newArrayList(project.getProjects()); - if (reduceExpressions(project, expList, predicates)) { - RelNode newProject = call.builder().push(project.getInput()) - .project(expList, project.getRowType().getFieldNames()).build(); - call.transformTo(newProject); - // New plan is absolutely better than old plan. - call.getPlanner().setImportance(project, 0.0); - } - } - } - - /** - * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}. - */ - public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule { - - public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass, - RelBuilderFactory relBuilderFactory) { - super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)"); - } - - @Override public void onMatch(RelOptRuleCall call) { - final HiveJoin join = call.rel(0); - final List<RexNode> expList = Lists.newArrayList(join.getCondition()); - final int fieldCount = join.getLeft().getRowType().getFieldCount(); - RelMetadataQuery mq = RelMetadataQuery.instance(); - final RelOptPredicateList leftPredicates = - mq.getPulledUpPredicates(join.getLeft()); - final RelOptPredicateList rightPredicates = - mq.getPulledUpPredicates(join.getRight()); - final RelOptPredicateList predicates = - leftPredicates.union(rightPredicates.shift(fieldCount)); - if (!reduceExpressions(join, expList, predicates, true)) { - return; - } - call.transformTo( - join.copy( - join.getTraitSet(), - expList.get(0), - join.getLeft(), - join.getRight(), - join.getJoinType(), - join.isSemiJoinDone())); - - // New plan is absolutely better than old plan. - call.getPlanner().setImportance(join, 0.0); - } - } - - //~ Constructors ----------------------------------------------------------- - - /** - * Creates a HiveReduceExpressionsRule. - * - * @param clazz class of rels to which this rule should apply - */ - protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz, - RelBuilderFactory relBuilderFactory, String desc) { - super(operand(clazz, any()), relBuilderFactory, desc); - } - - //~ Methods ---------------------------------------------------------------- - - /** - * Reduces a list of expressions. - * - * @param rel Relational expression - * @param expList List of expressions, modified in place - * @param predicates Constraints known to hold on input expressions - * @return whether reduction found something to change, and succeeded - */ - protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList, - RelOptPredicateList predicates) { - return reduceExpressions(rel, expList, predicates, false); - } - - /** - * Reduces a list of expressions. - * - * @param rel Relational expression - * @param expList List of expressions, modified in place - * @param predicates Constraints known to hold on input expressions - * @param unknownAsFalse Whether UNKNOWN will be treated as FALSE - * - * @return whether reduction found something to change, and succeeded - */ - protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList, - RelOptPredicateList predicates, boolean unknownAsFalse) { - RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); - - boolean reduced = reduceExpressionsInternal(rel, expList, predicates); - - // Simplify preds in place - ExprSimplifier simplifier = new ExprSimplifier(rexBuilder, unknownAsFalse); - List<RexNode> expList2 = Lists.newArrayList(expList); - simplifier.mutate(expList2); - boolean simplified = false; - for (int i = 0; i < expList.size(); i++) { - if (!expList2.get(i).toString().equals(expList.get(i).toString())) { - expList.remove(i); - expList.add(i, expList2.get(i)); - simplified = true; - } - } - - return reduced || simplified; - } - - protected static boolean reduceExpressionsInternal(RelNode rel, List<RexNode> expList, - RelOptPredicateList predicates) { - RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); - - // Replace predicates on CASE to CASE on predicates. - new CaseShuttle().mutate(expList); - - // Find reducible expressions. - final List<RexNode> constExps = Lists.newArrayList(); - List<Boolean> addCasts = Lists.newArrayList(); - final List<RexNode> removableCasts = Lists.newArrayList(); - final ImmutableMap<RexNode, RexNode> constants = - predicateConstants(RexNode.class, rexBuilder, predicates); - findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants, - constExps, addCasts, removableCasts); - if (constExps.isEmpty() && removableCasts.isEmpty()) { - return false; - } - - // Remove redundant casts before reducing constant expressions. - // If the argument to the redundant cast is a reducible constant, - // reducing that argument to a constant first will result in not being - // able to locate the original cast expression. - if (!removableCasts.isEmpty()) { - final List<RexNode> reducedExprs = Lists.newArrayList(); - for (RexNode exp : removableCasts) { - RexCall call = (RexCall) exp; - reducedExprs.add(call.getOperands().get(0)); - } - RexReplacer replacer = - new RexReplacer( - rexBuilder, - removableCasts, - reducedExprs, - Collections.nCopies(removableCasts.size(), false)); - replacer.mutate(expList); - } - - if (constExps.isEmpty()) { - return true; - } - - final List<RexNode> constExps2 = Lists.newArrayList(constExps); - if (!constants.isEmpty()) { - //noinspection unchecked - final List<Map.Entry<RexNode, RexNode>> pairs = - (List<Map.Entry<RexNode, RexNode>>) (List) - Lists.newArrayList(constants.entrySet()); - RexReplacer replacer = - new RexReplacer( - rexBuilder, - Pair.left(pairs), - Pair.right(pairs), - Collections.nCopies(pairs.size(), false)); - replacer.mutate(constExps2); - } - - // Compute the values they reduce to. - RelOptPlanner.Executor executor = - rel.getCluster().getPlanner().getExecutor(); - if (executor == null) { - // Cannot reduce expressions: caller has not set an executor in their - // environment. Caller should execute something like the following before - // invoking the planner: - // - // final RexExecutorImpl executor = - // new RexExecutorImpl(Schemas.createDataContext(null)); - // rootRel.getCluster().getPlanner().setExecutor(executor); - return false; - } - - final List<RexNode> reducedValues = Lists.newArrayList(); - executor.reduce(rexBuilder, constExps2, reducedValues); - - // Use RexNode.digest to judge whether each newly generated RexNode - // is equivalent to the original one. - if (Lists.transform(constExps, HiveCalciteUtil.REX_STR_FN).equals( - Lists.transform(reducedValues, HiveCalciteUtil.REX_STR_FN))) { - return false; - } - - // For Project, we have to be sure to preserve the result - // types, so always cast regardless of the expression type. - // For other RelNodes like Filter, in general, this isn't necessary, - // and the presence of casts could hinder other rules such as sarg - // analysis, which require bare literals. But there are special cases, - // like when the expression is a UDR argument, that need to be - // handled as special cases. - if (rel instanceof Project) { - addCasts = Collections.nCopies(reducedValues.size(), true); - } - - RexReplacer replacer = - new RexReplacer( - rexBuilder, - constExps, - reducedValues, - addCasts); - replacer.mutate(expList); - return true; - } - - /** - * Locates expressions that can be reduced to literals or converted to - * expressions with redundant casts removed. - * - * @param typeFactory Type factory - * @param exps list of candidate expressions to be examined for - * reduction - * @param constants List of expressions known to be constant - * @param constExps returns the list of expressions that can be constant - * reduced - * @param addCasts indicator for each expression that can be constant - * reduced, whether a cast of the resulting reduced - * expression is potentially necessary - * @param removableCasts returns the list of cast expressions where the cast - */ - protected static void findReducibleExps(RelDataTypeFactory typeFactory, - List<RexNode> exps, ImmutableMap<RexNode, RexNode> constants, - List<RexNode> constExps, List<Boolean> addCasts, - List<RexNode> removableCasts) { - ReducibleExprLocator gardener = - new ReducibleExprLocator(typeFactory, constants, constExps, - addCasts, removableCasts); - for (RexNode exp : exps) { - gardener.analyze(exp); - } - assert constExps.size() == addCasts.size(); - } - - /** Creates a map containing each (e, constant) pair that occurs within - * a predicate list. - * - * @param clazz Class of expression that is considered constant - * @param rexBuilder Rex builder - * @param predicates Predicate list - * @param <C> what to consider a constant: {@link RexLiteral} to use a narrow - * definition of constant, or {@link RexNode} to use - * {@link RexUtil#isConstant(RexNode)} - * @return Map from values to constants - */ - public static <C extends RexNode> ImmutableMap<RexNode, C> predicateConstants( - Class<C> clazz, RexBuilder rexBuilder, RelOptPredicateList predicates) { - // We cannot use an ImmutableMap.Builder here. If there are multiple entries - // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't - // matter which we take, so the latter will replace the former. - // The basic idea is to find all the pairs of RexNode = RexLiteral - // (1) If 'predicates' contain a non-EQUALS, we bail out. - // (2) It is OK if a RexNode is equal to the same RexLiteral several times, - // (e.g. "WHERE deptno = 1 AND deptno = 1") - // (3) It will return false if there are inconsistent constraints (e.g. - // "WHERE deptno = 1 AND deptno = 2") - final Map<RexNode, C> map = new HashMap<>(); - final Set<RexNode> excludeSet = new HashSet<>(); - for (RexNode predicate : predicates.pulledUpPredicates) { - gatherConstraints(clazz, predicate, map, excludeSet, rexBuilder); - } - final ImmutableMap.Builder<RexNode, C> builder = - ImmutableMap.builder(); - for (Map.Entry<RexNode, C> entry : map.entrySet()) { - RexNode rexNode = entry.getKey(); - if (!overlap(rexNode, excludeSet)) { - builder.put(rexNode, entry.getValue()); - } - } - return builder.build(); - } - - private static <C extends RexNode> void gatherConstraints(Class<C> clazz, - RexNode predicate, Map<RexNode, C> map, Set<RexNode> excludeSet, - RexBuilder rexBuilder) { - if (predicate.getKind() != SqlKind.EQUALS - && predicate.getKind() != SqlKind.IS_NULL) { - decompose(excludeSet, predicate); - return; - } - final List<RexNode> operands = ((RexCall) predicate).getOperands(); - if (operands.size() != 2 && predicate.getKind() == SqlKind.EQUALS) { - decompose(excludeSet, predicate); - return; - } - // if it reaches here, we have rexNode equals rexNode - final RexNode left; - final RexNode right; - if (predicate.getKind() == SqlKind.EQUALS) { - left = operands.get(0); - right = operands.get(1); - } else { - left = operands.get(0); - right = rexBuilder.makeNullLiteral(left.getType().getSqlTypeName()); - } - // note that literals are immutable too and they can only be compared through - // values. - gatherConstraint(clazz, left, right, map, excludeSet, rexBuilder); - gatherConstraint(clazz, right, left, map, excludeSet, rexBuilder); - } - - /** Returns whether a value of {@code type2} can be assigned to a variable - * of {@code type1}. - * - * <p>For example: - * <ul> - * <li>{@code canAssignFrom(BIGINT, TINYINT)} returns {@code true}</li> - * <li>{@code canAssignFrom(TINYINT, BIGINT)} returns {@code false}</li> - * <li>{@code canAssignFrom(BIGINT, VARCHAR)} returns {@code false}</li> - * </ul> - */ - private static boolean canAssignFrom(RelDataType type1, RelDataType type2) { - final SqlTypeName name1 = type1.getSqlTypeName(); - final SqlTypeName name2 = type2.getSqlTypeName(); - if (name1.getFamily() == name2.getFamily()) { - switch (name1.getFamily()) { - case NUMERIC: - return name1.compareTo(name2) >= 0; + /** + * For static schema systems, a filter that is always false or null can be + * replaced by a values operator that produces no rows, as the schema + * information can just be taken from the input Rel. In dynamic schema + * environments, the filter might have an unknown input type, in these cases + * they must define a system specific alternative to a Values operator, such + * as inserting a limit 0 instead of a filter on top of the original input. + * + * <p>The default implementation of this method is to call + * {@link RelBuilder#empty}, which for the static schema will be optimized + * to an empty + * {@link org.apache.calcite.rel.core.Values}. + * + * @param input rel to replace, assumes caller has already determined + * equivalence to Values operation for 0 records or a + * false filter. + * @return equivalent but less expensive replacement rel + */ + protected RelNode createEmptyRelOrEquivalent(RelOptRuleCall call, Filter input) { + return call.builder().push(input).empty().build(); + } + + private void reduceNotNullableFilter( + RelOptRuleCall call, + Filter filter, + RexCall rexCall, + boolean reverse) { + // If the expression is a IS [NOT] NULL on a non-nullable + // column, then we can either remove the filter or replace + // it with an Empty. + boolean alwaysTrue; + switch (rexCall.getKind()) { + case IS_NULL: + case IS_UNKNOWN: + alwaysTrue = false; + break; + case IS_NOT_NULL: + alwaysTrue = true; + break; default: - return true; - } - } - return false; - } - - private static <C extends RexNode> void gatherConstraint(Class<C> clazz, - RexNode left, RexNode right, Map<RexNode, C> map, Set<RexNode> excludeSet, - RexBuilder rexBuilder) { - if (!clazz.isInstance(right)) { - return; - } - if (!RexUtil.isConstant(right)) { - return; - } - C constant = clazz.cast(right); - if (excludeSet.contains(left)) { - return; - } - final C existedValue = map.get(left); - if (existedValue == null) { - switch (left.getKind()) { - case CAST: - // Convert "CAST(c) = literal" to "c = literal", as long as it is a - // widening cast. - final RexNode operand = ((RexCall) left).getOperands().get(0); - if (canAssignFrom(left.getType(), operand.getType())) { - final RexNode castRight = - rexBuilder.makeCast(operand.getType(), constant); - if (castRight instanceof RexLiteral) { - left = operand; - constant = clazz.cast(castRight); - } - } - } - map.put(left, constant); - } else { - if (existedValue instanceof RexLiteral - && constant instanceof RexLiteral - && !((RexLiteral) existedValue).getValue() - .equals(((RexLiteral) constant).getValue())) { - // we found conflicting values, e.g. left = 10 and left = 20 - map.remove(left); - excludeSet.add(left); - } - } - } - - private static boolean overlap(RexNode rexNode, Set<RexNode> set) { - if (rexNode instanceof RexCall) { - for (RexNode r : ((RexCall) rexNode).getOperands()) { - if (overlap(r, set)) { - return true; - } - } - return false; - } else { - return set.contains(rexNode); - } - } - - /** Tries to decompose the RexNode which is a RexCall into non-literal - * RexNodes. */ - private static void decompose(Set<RexNode> set, RexNode rexNode) { - if (rexNode instanceof RexCall) { - for (RexNode r : ((RexCall) rexNode).getOperands()) { - decompose(set, r); - } - } else if (!(rexNode instanceof RexLiteral)) { - set.add(rexNode); - } - } - - /** Pushes predicates into a CASE. - * - * <p>We have a loose definition of 'predicate': any boolean expression will - * do, except CASE. For example '(CASE ...) = 5' or '(CASE ...) IS NULL'. - */ - protected static RexCall pushPredicateIntoCase(RexCall call) { - if (call.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { - return call; - } - switch (call.getKind()) { - case CASE: - case AND: - case OR: - return call; // don't push CASE into CASE! - } - int caseOrdinal = -1; - final List<RexNode> operands = call.getOperands(); - for (int i = 0; i < operands.size(); i++) { - RexNode operand = operands.get(i); - switch (operand.getKind()) { - case CASE: - caseOrdinal = i; - } - } - if (caseOrdinal < 0) { - return call; - } - // Convert - // f(CASE WHEN p1 THEN v1 ... END, arg) - // to - // CASE WHEN p1 THEN f(v1, arg) ... END - final RexCall case_ = (RexCall) operands.get(caseOrdinal); - final List<RexNode> nodes = new ArrayList<>(); - for (int i = 0; i < case_.getOperands().size(); i++) { - RexNode node = case_.getOperands().get(i); - if (!RexUtil.isCasePredicate(case_, i)) { - node = substitute(call, caseOrdinal, node); - } - nodes.add(node); - } - return case_.clone(call.getType(), nodes); - } - - /** Converts op(arg0, ..., argOrdinal, ..., argN) to op(arg0,..., node, ..., argN). */ - protected static RexNode substitute(RexCall call, int ordinal, RexNode node) { - final List<RexNode> newOperands = Lists.newArrayList(call.getOperands()); - newOperands.set(ordinal, node); - return call.clone(call.getType(), newOperands); - } - - //~ Inner Classes ---------------------------------------------------------- - - /** - * Replaces expressions with their reductions. Note that we only have to - * look for RexCall, since nothing else is reducible in the first place. - */ - protected static class RexReplacer extends RexShuttle { - private final RexBuilder rexBuilder; - private final List<RexNode> reducibleExps; - private final List<RexNode> reducedValues; - private final List<Boolean> addCasts; - - RexReplacer( - RexBuilder rexBuilder, - List<RexNode> reducibleExps, - List<RexNode> reducedValues, - List<Boolean> addCasts) { - this.rexBuilder = rexBuilder; - this.reducibleExps = reducibleExps; - this.reducedValues = reducedValues; - this.addCasts = addCasts; - } - - @Override public RexNode visitInputRef(RexInputRef inputRef) { - RexNode node = visit(inputRef); - if (node == null) { - return super.visitInputRef(inputRef); - } - return node; - } - - @Override public RexNode visitCall(RexCall call) { - RexNode node = visit(call); - if (node != null) { - return node; - } - node = super.visitCall(call); - if (node != call) { - node = HiveRexUtil.simplify(rexBuilder, node); - } - return node; - } - - private RexNode visit(final RexNode call) { - int i = reducibleExps.indexOf(call); - if (i == -1) { - return null; - } - RexNode replacement = reducedValues.get(i); - if (addCasts.get(i) - && (replacement.getType() != call.getType())) { - // Handle change from nullable to NOT NULL by claiming - // that the result is still nullable, even though - // we know it isn't. - // - // Also, we cannot reduce CAST('abc' AS VARCHAR(4)) to 'abc'. - // If we make 'abc' of type VARCHAR(4), we may later encounter - // the same expression in a Project's digest where it has - // type VARCHAR(3), and that's wrong. - replacement = rexBuilder.makeAbstractCast(call.getType(), replacement); - } - return replacement; - } - } - - /** - * Helper class used to locate expressions that either can be reduced to - * literals or contain redundant casts. - */ - protected static class ReducibleExprLocator extends RexVisitorImpl<Void> { - /** Whether an expression is constant, and if so, whether it can be - * reduced to a simpler constant. */ - enum Constancy { - NON_CONSTANT, REDUCIBLE_CONSTANT, IRREDUCIBLE_CONSTANT - } - - private final RelDataTypeFactory typeFactory; - - private final List<Constancy> stack; - - private final ImmutableMap<RexNode, RexNode> constants; - - private final List<RexNode> constExprs; - - private final List<Boolean> addCasts; - - private final List<RexNode> removableCasts; - - private final List<SqlOperator> parentCallTypeStack; - - ReducibleExprLocator(RelDataTypeFactory typeFactory, - ImmutableMap<RexNode, RexNode> constants, List<RexNode> constExprs, - List<Boolean> addCasts, List<RexNode> removableCasts) { - // go deep - super(true); - this.typeFactory = typeFactory; - this.constants = constants; - this.constExprs = constExprs; - this.addCasts = addCasts; - this.removableCasts = removableCasts; - this.stack = Lists.newArrayList(); - this.parentCallTypeStack = Lists.newArrayList(); - } - - public void analyze(RexNode exp) { - assert stack.isEmpty(); - - exp.accept(this); - - // Deal with top of stack - assert stack.size() == 1; - assert parentCallTypeStack.isEmpty(); - Constancy rootConstancy = stack.get(0); - if (rootConstancy == Constancy.REDUCIBLE_CONSTANT) { - // The entire subtree was constant, so add it to the result. - addResult(exp); - } - stack.clear(); - } - - private Void pushVariable() { - stack.add(Constancy.NON_CONSTANT); - return null; - } - - private void addResult(RexNode exp) { - // Cast of literal can't be reduced, so skip those (otherwise we'd - // go into an infinite loop as we add them back). - if (exp.getKind() == SqlKind.CAST) { - RexCall cast = (RexCall) exp; - RexNode operand = cast.getOperands().get(0); - if (operand instanceof RexLiteral) { - return; - } - } - constExprs.add(exp); - - // In the case where the expression corresponds to a UDR argument, - // we need to preserve casts. Note that this only applies to - // the topmost argument, not expressions nested within the UDR - // call. - // - // REVIEW zfong 6/13/08 - Are there other expressions where we - // also need to preserve casts? - if (parentCallTypeStack.isEmpty()) { - addCasts.add(false); - } else { - addCasts.add(isUdf(Stacks.peek(parentCallTypeStack))); - } - } - - private Boolean isUdf(SqlOperator operator) { - // return operator instanceof UserDefinedRoutine - return false; - } - - public Void visitInputRef(RexInputRef inputRef) { - if (constants.containsKey(inputRef)) { - stack.add(Constancy.REDUCIBLE_CONSTANT); - return null; - } - return pushVariable(); - } - - public Void visitLiteral(RexLiteral literal) { - stack.add(Constancy.IRREDUCIBLE_CONSTANT); - return null; - } - - public Void visitOver(RexOver over) { - // assume non-constant (running SUM(1) looks constant but isn't) - analyzeCall(over, Constancy.NON_CONSTANT); - return null; - } - - public Void visitCorrelVariable(RexCorrelVariable correlVariable) { - return pushVariable(); - } - - public Void visitCall(RexCall call) { - // assume REDUCIBLE_CONSTANT until proven otherwise - analyzeCall(call, Constancy.REDUCIBLE_CONSTANT); - return null; - } - - private void analyzeCall(RexCall call, Constancy callConstancy) { - Stacks.push(parentCallTypeStack, call.getOperator()); - - // visit operands, pushing their states onto stack - super.visitCall(call); - - // look for NON_CONSTANT operands - int operandCount = call.getOperands().size(); - List<Constancy> operandStack = Util.last(stack, operandCount); - for (Constancy operandConstancy : operandStack) { - if (operandConstancy == Constancy.NON_CONSTANT) { - callConstancy = Constancy.NON_CONSTANT; - } - } - - // Even if all operands are constant, the call itself may - // be non-deterministic. - if (!call.getOperator().isDeterministic()) { - callConstancy = Constancy.NON_CONSTANT; - } else if (call.getOperator().isDynamicFunction()) { - // We can reduce the call to a constant, but we can't - // cache the plan if the function is dynamic. - // For now, treat it same as non-deterministic. - callConstancy = Constancy.NON_CONSTANT; - } - - // Row operator itself can't be reduced to a literal, but if - // the operands are constants, we still want to reduce those - if ((callConstancy == Constancy.REDUCIBLE_CONSTANT) - && (call.getOperator() instanceof SqlRowOperator)) { - callConstancy = Constancy.NON_CONSTANT; + return; } - - if (callConstancy == Constancy.NON_CONSTANT) { - // any REDUCIBLE_CONSTANT children are now known to be maximal - // reducible subtrees, so they can be added to the result - // list - for (int iOperand = 0; iOperand < operandCount; ++iOperand) { - Constancy constancy = operandStack.get(iOperand); - if (constancy == Constancy.REDUCIBLE_CONSTANT) { - addResult(call.getOperands().get(iOperand)); + if (reverse) { + alwaysTrue = !alwaysTrue; + } + RexNode operand = rexCall.getOperands().get(0); + if (operand instanceof RexInputRef) { + RexInputRef inputRef = (RexInputRef) operand; + if (!inputRef.getType().isNullable()) { + if (alwaysTrue) { + call.transformTo(filter.getInput()); + } else { + call.transformTo(createEmptyRelOrEquivalent(call, filter)); } } - - // if this cast expression can't be reduced to a literal, - // then see if we can remove the cast - if (call.getOperator() == SqlStdOperatorTable.CAST) { - reduceCasts(call); - } } - - // pop operands off of the stack - operandStack.clear(); - - // pop this parent call operator off the stack - Stacks.pop(parentCallTypeStack, call.getOperator()); - - // push constancy result for this call onto stack - stack.add(callConstancy); - } - - private void reduceCasts(RexCall outerCast) { - List<RexNode> operands = outerCast.getOperands(); - if (operands.size() != 1) { - return; - } - RelDataType outerCastType = outerCast.getType(); - RelDataType operandType = operands.get(0).getType(); - if (operandType.equals(outerCastType)) { - removableCasts.add(outerCast); - return; - } - - // See if the reduction - // CAST((CAST x AS type) AS type NOT NULL) - // -> CAST(x AS type NOT NULL) - // applies. TODO jvs 15-Dec-2008: consider - // similar cases for precision changes. - if (!(operands.get(0) instanceof RexCall)) { - return; - } - RexCall innerCast = (RexCall) operands.get(0); - if (innerCast.getOperator() != SqlStdOperatorTable.CAST) { - return; - } - if (innerCast.getOperands().size() != 1) { - return; - } - RelDataType outerTypeNullable = - typeFactory.createTypeWithNullability(outerCastType, true); - RelDataType innerTypeNullable = - typeFactory.createTypeWithNullability(operandType, true); - if (outerTypeNullable != innerTypeNullable) { - return; - } - if (operandType.isNullable()) { - removableCasts.add(innerCast); - } - } - - public Void visitDynamicParam(RexDynamicParam dynamicParam) { - return pushVariable(); - } - - public Void visitRangeRef(RexRangeRef rangeRef) { - return pushVariable(); - } - - public Void visitFieldAccess(RexFieldAccess fieldAccess) { - return pushVariable(); } } - /** Shuttle that pushes predicates into a CASE. */ - protected static class CaseShuttle extends RexShuttle { - @Override public RexNode visitCall(RexCall call) { - for (;;) { - call = (RexCall) super.visitCall(call); - final RexCall old = call; - call = pushPredicateIntoCase(call); - if (call == old) { - return call; - } - } - } - } } // End HiveReduceExpressionsRule.java http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java index ec488fe..0644f0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -38,7 +38,6 @@ import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.util.Pair; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -131,7 +130,7 @@ public class HiveReduceExpressionsWithStatsRule extends RelOptRule { && call.operands.get(0) instanceof RexLiteral) { ref = (RexInputRef) call.operands.get(1); literal = (RexLiteral) call.operands.get(0); - kind = HiveRexUtil.invert(call.getOperator().getKind()); + kind = call.getOperator().getKind().reverse(); } // Found an expression that we can try to reduce @@ -252,7 +251,7 @@ public class HiveReduceExpressionsWithStatsRule extends RelOptRule { // If we did not reduce, check the children nodes RexNode node = super.visitCall(call); if (node != call) { - node = HiveRexUtil.simplify(rexBuilder, node); + node = RexUtil.simplify(rexBuilder, node); } return node; } http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java index b0cb8df..458fee7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; @@ -24,28 +25,22 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.RelCollations; -import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldAccess; -import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexPermuteInputsShuttle; -import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql2rel.CorrelationReferenceFinder; @@ -62,7 +57,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo; import com.google.common.collect.ImmutableList; @@ -72,11 +66,9 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer { protected static final Log LOG = LogFactory.getLog(HiveRelFieldTrimmer.class); - private RelBuilder relBuilder; - private ColumnAccessInfo columnAccessInfo; - private Map<HiveProject, Table> viewProjectToTableSchema; + private final RelBuilder relBuilder; public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) { super(validator, relBuilder); @@ -86,9 +78,9 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer { public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder, ColumnAccessInfo columnAccessInfo, Map<HiveProject, Table> viewToTableSchema) { super(validator, relBuilder); - this.relBuilder = relBuilder; this.columnAccessInfo = columnAccessInfo; this.viewProjectToTableSchema = viewToTableSchema; + this.relBuilder = relBuilder; } /** @@ -193,186 +185,73 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer { /** * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for - * {@link org.apache.calcite.rel.core.Sort}. + * {@link org.apache.calcite.adapter.druid.DruidQuery}. */ - public TrimResult trimFields( - HiveSortLimit sort, - ImmutableBitSet fieldsUsed, + public TrimResult trimFields(DruidQuery dq, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) { - final RelDataType rowType = sort.getRowType(); - final int fieldCount = rowType.getFieldCount(); - final RelCollation collation = sort.getCollation(); - final RelNode input = sort.getInput(); - RelOptCluster cluster = sort.getCluster(); - - // We use the fields used by the consumer, plus any fields used as sort - // keys. - final ImmutableBitSet.Builder inputFieldsUsed = - ImmutableBitSet.builder(fieldsUsed); - for (RelFieldCollation field : collation.getFieldCollations()) { - inputFieldsUsed.set(field.getFieldIndex()); + final int fieldCount = dq.getRowType().getFieldCount(); + if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) + && extraFields.isEmpty()) { + // if there is nothing to project or if we are projecting everything + // then no need to introduce another RelNode + return trimFields( + (RelNode) dq, fieldsUsed, extraFields); } - - // Create input with trimmed columns. - final Set<RelDataTypeField> inputExtraFields = Collections.emptySet(); - TrimResult trimResult = - trimChild(sort, input, inputFieldsUsed.build(), inputExtraFields); - RelNode newInput = trimResult.left; - final Mapping inputMapping = trimResult.right; - - // If the input is unchanged, and we need to project all columns, - // there's nothing we can do. - if (newInput == input - && inputMapping.isIdentity() - && fieldsUsed.cardinality() == fieldCount) { - return result(sort, Mappings.createIdentity(fieldCount)); + final RelNode newTableAccessRel = project(dq, fieldsUsed, extraFields, relBuilder); + + // Some parts of the system can't handle rows with zero fields, so + // pretend that one field is used. + if (fieldsUsed.cardinality() == 0) { + RelNode input = newTableAccessRel; + if (input instanceof Project) { + // The table has implemented the project in the obvious way - by + // creating project with 0 fields. Strip it away, and create our own + // project with one field. + Project project = (Project) input; + if (project.getRowType().getFieldCount() == 0) { + input = project.getInput(); + } + } + return dummyProject(fieldCount, input); } - relBuilder.push(newInput); - final int offset = - sort.offset == null ? 0 : RexLiteral.intValue(sort.offset); - final int fetch = - sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch); - final ImmutableList<RexNode> fields = - relBuilder.fields(RexUtil.apply(inputMapping, collation)); - - // The result has the same mapping as the input gave us. Sometimes we - // return fields that the consumer didn't ask for, because the filter - // needs them for its condition. - // TODO: Calcite will return empty LogicalValues when offset == 0 && fetch == 0. - // However, Hive ASTConverter can not deal with LogicalValues. - sortLimit(cluster, relBuilder, offset, fetch, fields); - return result(relBuilder.build(), inputMapping); + final Mapping mapping = createMapping(fieldsUsed, fieldCount); + return result(newTableAccessRel, mapping); } - - private List<RexNode> projects(RelDataType inputRowType, RelOptCluster cluster) { - final List<RexNode> exprList = new ArrayList<>(); - for (RelDataTypeField field : inputRowType.getFieldList()) { - final RexBuilder rexBuilder = cluster.getRexBuilder(); - exprList.add(rexBuilder.makeInputRef(field.getType(), field.getIndex())); + + private static RelNode project(DruidQuery dq, ImmutableBitSet fieldsUsed, + Set<RelDataTypeField> extraFields, RelBuilder relBuilder) { + final int fieldCount = dq.getRowType().getFieldCount(); + if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) + && extraFields.isEmpty()) { + return dq; } - return exprList; - } - - private static RelFieldCollation collation(RexNode node, - RelFieldCollation.Direction direction, - RelFieldCollation.NullDirection nullDirection, List<RexNode> extraNodes) { - switch (node.getKind()) { - case INPUT_REF: - return new RelFieldCollation(((RexInputRef) node).getIndex(), direction, - Util.first(nullDirection, direction.defaultNullDirection())); - case DESCENDING: - return collation(((RexCall) node).getOperands().get(0), - RelFieldCollation.Direction.DESCENDING, - nullDirection, extraNodes); - case NULLS_FIRST: - return collation(((RexCall) node).getOperands().get(0), direction, - RelFieldCollation.NullDirection.FIRST, extraNodes); - case NULLS_LAST: - return collation(((RexCall) node).getOperands().get(0), direction, - RelFieldCollation.NullDirection.LAST, extraNodes); - default: - final int fieldIndex = extraNodes.size(); - extraNodes.add(node); - return new RelFieldCollation(fieldIndex, direction, - Util.first(nullDirection, direction.defaultNullDirection())); + final List<RexNode> exprList = new ArrayList<>(); + final List<String> nameList = new ArrayList<>(); + final RexBuilder rexBuilder = dq.getCluster().getRexBuilder(); + final List<RelDataTypeField> fields = dq.getRowType().getFieldList(); + + // Project the subset of fields. + for (int i : fieldsUsed) { + RelDataTypeField field = fields.get(i); + exprList.add(rexBuilder.makeInputRef(dq, i)); + nameList.add(field.getName()); } - } - - private void sortLimit(RelOptCluster cluster, RelBuilder relBuilder, int offset, int fetch, - Iterable<? extends RexNode> nodes) { - final List<RelFieldCollation> fieldCollations = new ArrayList<>(); - final RelDataType inputRowType = relBuilder.peek().getRowType(); - final List<RexNode> extraNodes = projects(inputRowType, cluster); - final List<RexNode> originalExtraNodes = ImmutableList.copyOf(extraNodes); - for (RexNode node : nodes) { - fieldCollations.add( - collation(node, RelFieldCollation.Direction.ASCENDING, - RelFieldCollation.NullDirection.FIRST, extraNodes)); - } - final RexNode offsetNode = offset <= 0 ? null : relBuilder.literal(offset); - final RexNode fetchNode = fetch < 0 ? null : relBuilder.literal(fetch); - if (offsetNode == null && fetchNode == null && fieldCollations.isEmpty()) { - return; // sort is trivial - } - final boolean addedFields = extraNodes.size() > originalExtraNodes.size(); - if (fieldCollations.isEmpty()) { - assert !addedFields; - RelNode top = relBuilder.peek(); - if (top instanceof Sort) { - final Sort sort2 = (Sort) top; - if (sort2.offset == null && sort2.fetch == null) { - relBuilder.build(); - relBuilder.push(sort2.getInput()); - final RelNode sort = - HiveSortLimit.create(relBuilder.build(), sort2.collation, - offsetNode, fetchNode); - relBuilder.push(sort); - return; - } - } - if (top instanceof Project) { - final Project project = (Project) top; - if (project.getInput() instanceof Sort) { - final Sort sort2 = (Sort) project.getInput(); - if (sort2.offset == null && sort2.fetch == null) { - relBuilder.build(); - relBuilder.push(sort2.getInput()); - final RelNode sort = - HiveSortLimit.create(relBuilder.build(), sort2.collation, - offsetNode, fetchNode); - relBuilder.push(sort); - relBuilder.project(project.getProjects()); - return; - } - } - } - } - if (addedFields) { - relBuilder.project(extraNodes); - } - final RelNode sort = - HiveSortLimit.create(relBuilder.build(), RelCollations.of(fieldCollations), - offsetNode, fetchNode); - relBuilder.push(sort); - if (addedFields) { - relBuilder.project(originalExtraNodes); - } - return; - } - - private TrimResult result(RelNode r, final Mapping mapping) { - final RexBuilder rexBuilder = relBuilder.getRexBuilder(); - for (final CorrelationId correlation : r.getVariablesSet()) { - r = r.accept( - new CorrelationReferenceFinder() { - @Override - protected RexNode handle(RexFieldAccess fieldAccess) { - final RexCorrelVariable v = - (RexCorrelVariable) fieldAccess.getReferenceExpr(); - if (v.id.equals(correlation) - && v.getType().getFieldCount() == mapping.getSourceCount()) { - final int old = fieldAccess.getField().getIndex(); - final int new_ = mapping.getTarget(old); - final RelDataTypeFactory.FieldInfoBuilder typeBuilder = - relBuilder.getTypeFactory().builder(); - for (int target : Util.range(mapping.getTargetCount())) { - typeBuilder.add( - v.getType().getFieldList().get(mapping.getSource(target))); - } - final RexNode newV = - rexBuilder.makeCorrel(typeBuilder.build(), v.id); - if (old != new_) { - return rexBuilder.makeFieldAccess(newV, new_); - } - } - return fieldAccess; - } - - }); + // Project nulls for the extra fields. (Maybe a sub-class table has + // extra fields, but we don't.) + for (RelDataTypeField extraField : extraFields) { + exprList.add( + rexBuilder.ensureType( + extraField.getType(), + rexBuilder.constantNull(), + true)); + nameList.add(extraField.getName()); } - return new TrimResult(r, mapping); + + HiveProject hp = (HiveProject) relBuilder.push(dq).project(exprList, nameList).build(); + hp.setSynthetic(); + return hp; } /** http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java index 18fe650..b86b947 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java @@ -20,7 +20,10 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdCollation; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -32,7 +35,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import com.google.common.collect.ImmutableList; -public class HiveRelMdCollation { +public class HiveRelMdCollation implements MetadataHandler<BuiltInMetadata.Collation> { public static final RelMetadataProvider SOURCE = ChainedRelMetadataProvider.of( @@ -47,6 +50,11 @@ public class HiveRelMdCollation { //~ Methods ---------------------------------------------------------------- + @Override + public MetadataDef<BuiltInMetadata.Collation> getDef() { + return BuiltInMetadata.Collation.DEF; + } + public ImmutableList<RelCollation> collations(HiveAggregate aggregate, RelMetadataQuery mq) { // Compute collations ImmutableList.Builder<RelFieldCollation> collationListBuilder = http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java index 62d3ead..e574e7a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java @@ -18,9 +18,11 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; -import org.apache.calcite.rel.metadata.RelMdDistribution; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.util.BuiltInMethod; @@ -30,7 +32,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import com.google.common.collect.ImmutableList; -public class HiveRelMdDistribution { +public class HiveRelMdDistribution implements MetadataHandler<BuiltInMetadata.Distribution> { public static final RelMetadataProvider SOURCE = ChainedRelMetadataProvider.of( @@ -44,6 +46,10 @@ public class HiveRelMdDistribution { //~ Methods ---------------------------------------------------------------- + public MetadataDef<BuiltInMetadata.Distribution> getDef() { + return BuiltInMetadata.Distribution.DEF; + } + public RelDistribution distribution(HiveAggregate aggregate, RelMetadataQuery mq) { return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, aggregate.getGroupSet().asList()); http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index e468573..69e157e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -40,6 +40,10 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.metadata.BuiltInMetadata; +import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdPredicates; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -72,14 +76,28 @@ import com.google.common.collect.Maps; //TODO: Move this to calcite -public class HiveRelMdPredicates extends RelMdPredicates { +public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Predicates> { - public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.PREDICATES.method, - new HiveRelMdPredicates()); + public static final RelMetadataProvider SOURCE = + ChainedRelMetadataProvider.of( + ImmutableList.of( + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.PREDICATES.method, new HiveRelMdPredicates()), + RelMdPredicates.SOURCE)); private static final List<RexNode> EMPTY_LIST = ImmutableList.of(); + //~ Constructors ----------------------------------------------------------- + + private HiveRelMdPredicates() {} + + //~ Methods ---------------------------------------------------------------- + + @Override + public MetadataDef<BuiltInMetadata.Predicates> getDef() { + return BuiltInMetadata.Predicates.DEF; + } + /** * Infers predicates for a project. * @@ -99,8 +117,8 @@ public class HiveRelMdPredicates extends RelMdPredicates { * * </ol> */ - @Override public RelOptPredicateList getPredicates(Project project, RelMetadataQuery mq) { + RelNode child = project.getInput(); final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); RelOptPredicateList childInfo = mq.getPulledUpPredicates(child); @@ -151,7 +169,6 @@ public class HiveRelMdPredicates extends RelMdPredicates { } /** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */ - @Override public RelOptPredicateList getPredicates(Join join, RelMetadataQuery mq) { RexBuilder rB = join.getCluster().getRexBuilder(); RelNode left = join.getInput(0); @@ -181,7 +198,6 @@ public class HiveRelMdPredicates extends RelMdPredicates { * pulledUpExprs : { a > 7} * </pre> */ - @Override public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq) { final RelNode input = agg.getInput(); final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input); @@ -209,7 +225,6 @@ public class HiveRelMdPredicates extends RelMdPredicates { /** * Infers predicates for a Union. */ - @Override public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) { RexBuilder rB = union.getCluster().getRexBuilder(); http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index 0d03ebb..651adc0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -41,13 +41,16 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import com.google.common.collect.ImmutableMap; public class HiveRelMdSelectivity extends RelMdSelectivity { - public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource( - BuiltInMethod.SELECTIVITY.method, - new HiveRelMdSelectivity()); - protected HiveRelMdSelectivity() { - super(); - } + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.SELECTIVITY.method, new HiveRelMdSelectivity()); + + //~ Constructors ----------------------------------------------------------- + + private HiveRelMdSelectivity() {} + + //~ Methods ---------------------------------------------------------------- public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) { if (predicate != null) { @@ -58,7 +61,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { return 1.0; } - public Double getSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) throws CalciteSemanticException { + public Double getSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) { if (j.getJoinType().equals(JoinRelType.INNER)) { return computeInnerJoinSelectivity(j, mq, predicate); } else if (j.getJoinType().equals(JoinRelType.LEFT) || @@ -75,7 +78,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { return 1.0; } - private Double computeInnerJoinSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) throws CalciteSemanticException { + private Double computeInnerJoinSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) { double ndvCrossProduct = 1; Pair<Boolean, RexNode> predInfo = getCombinedPredicateForJoin(j, predicate); @@ -86,8 +89,13 @@ public class HiveRelMdSelectivity extends RelMdSelectivity { } RexNode combinedPredicate = predInfo.getValue(); - JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, - combinedPredicate); + JoinPredicateInfo jpi; + try { + jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, + combinedPredicate); + } catch (CalciteSemanticException e) { + throw new RuntimeException(e); + } ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap .builder(); ImmutableMap<Integer, Double> colStatMap; http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java index 31adb41..1039f56 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java @@ -122,7 +122,18 @@ public class HiveRelMdSize extends RelMdSize { case BIGINT: case DOUBLE: case TIMESTAMP: - case INTERVAL_DAY_TIME: + case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: + case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: + case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: + case INTERVAL_MONTH: + case INTERVAL_SECOND: + case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: return 8d; case BINARY: http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java index 0718150..9a5a2ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java @@ -30,9 +30,9 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.BuiltInMetadata; -import org.apache.calcite.rel.metadata.Metadata; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; -import org.apache.calcite.rel.metadata.RelMdUniqueKeys; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexInputRef; @@ -43,13 +43,16 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; -import com.google.common.base.Function; +public class HiveRelMdUniqueKeys implements MetadataHandler<BuiltInMetadata.UniqueKeys> { -public class HiveRelMdUniqueKeys { + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.UNIQUE_KEYS.method, new HiveRelMdUniqueKeys()); - public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider - .reflectiveSource(BuiltInMethod.UNIQUE_KEYS.method, - new HiveRelMdUniqueKeys()); + @Override + public MetadataDef<BuiltInMetadata.UniqueKeys> getDef() { + return BuiltInMetadata.UniqueKeys.DEF; + } /* * Infer Uniquenes if: - rowCount(col) = ndv(col) - TBD for numerics: max(col) @@ -65,7 +68,60 @@ public class HiveRelMdUniqueKeys { HiveTableScan tScan = getTableScan(rel.getInput(), false); if (tScan == null) { - return mq.getUniqueKeys(rel, ignoreNulls); + // If HiveTableScan is not found, e.g., not sequence of Project and + // Filter operators, execute the original getUniqueKeys method + + // LogicalProject maps a set of rows to a different set; + // Without knowledge of the mapping function(whether it + // preserves uniqueness), it is only safe to derive uniqueness + // info from the child of a project when the mapping is f(a) => a. + // + // Further more, the unique bitset coming from the child needs + // to be mapped to match the output of the project. + final Map<Integer, Integer> mapInToOutPos = new HashMap<>(); + final List<RexNode> projExprs = rel.getProjects(); + final Set<ImmutableBitSet> projUniqueKeySet = new HashSet<>(); + + // Build an input to output position map. + for (int i = 0; i < projExprs.size(); i++) { + RexNode projExpr = projExprs.get(i); + if (projExpr instanceof RexInputRef) { + mapInToOutPos.put(((RexInputRef) projExpr).getIndex(), i); + } + } + + if (mapInToOutPos.isEmpty()) { + // if there's no RexInputRef in the projected expressions + // return empty set. + return projUniqueKeySet; + } + + Set<ImmutableBitSet> childUniqueKeySet = + mq.getUniqueKeys(rel.getInput(), ignoreNulls); + + if (childUniqueKeySet != null) { + // Now add to the projUniqueKeySet the child keys that are fully + // projected. + for (ImmutableBitSet colMask : childUniqueKeySet) { + ImmutableBitSet.Builder tmpMask = ImmutableBitSet.builder(); + boolean completeKeyProjected = true; + for (int bit : colMask) { + if (mapInToOutPos.containsKey(bit)) { + tmpMask.set(mapInToOutPos.get(bit)); + } else { + // Skip the child unique key if part of it is not + // projected. + completeKeyProjected = false; + break; + } + } + if (completeKeyProjected) { + projUniqueKeySet.add(tmpMask.build()); + } + } + } + + return projUniqueKeySet; } Map<Integer, Integer> posMap = new HashMap<Integer, Integer>(); http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index 9a5becb..7b9ee84 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -22,15 +22,15 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; +import org.apache.calcite.adapter.druid.DruidQuery; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; -import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidQuery; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; @@ -58,19 +58,20 @@ class ASTBuilder { "TOK_TMP_FILE")).node(); } - static ASTNode table(TableScan scan) { - RelOptHiveTable hTbl = (RelOptHiveTable) scan.getTable(); - ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add( - ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") - .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()) - .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName())); - + static ASTNode table(RelNode scan) { HiveTableScan hts; if (scan instanceof DruidQuery) { hts = (HiveTableScan) ((DruidQuery)scan).getTableScan(); } else { hts = (HiveTableScan) scan; } + + RelOptHiveTable hTbl = (RelOptHiveTable) hts.getTable(); + ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add( + ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME") + .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName()) + .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName())); + ASTBuilder propList = ASTBuilder.construct(HiveParser.TOK_TABLEPROPLIST, "TOK_TABLEPROPLIST"); if (scan instanceof DruidQuery) { // Pass possible query to Druid @@ -181,8 +182,19 @@ class ASTBuilder { case DATE: case TIME: case TIMESTAMP: + case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: + case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: + case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: + case INTERVAL_MONTH: + case INTERVAL_SECOND: + case INTERVAL_YEAR: case INTERVAL_YEAR_MONTH: - case INTERVAL_DAY_TIME: if (literal.getValue() == null) { return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node(); } @@ -273,14 +285,25 @@ class ASTBuilder { val = "'" + val + "'"; } break; + case INTERVAL_YEAR: + case INTERVAL_MONTH: case INTERVAL_YEAR_MONTH: { type = HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL; BigDecimal monthsBd = (BigDecimal) literal.getValue(); HiveIntervalYearMonth intervalYearMonth = new HiveIntervalYearMonth(monthsBd.intValue()); val = "'" + intervalYearMonth.toString() + "'"; - break; } - case INTERVAL_DAY_TIME: { + break; + case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: + case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: + case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: + case INTERVAL_SECOND: { type = HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL; BigDecimal millisBd = (BigDecimal) literal.getValue(); @@ -288,8 +311,8 @@ class ASTBuilder { BigDecimal secsBd = millisBd.divide(BigDecimal.valueOf(1000)); HiveIntervalDayTime intervalDayTime = new HiveIntervalDayTime(secsBd); val = "'" + intervalDayTime.toString() + "'"; - break; } + break; case NULL: type = HiveParser.TOK_NULL; break;