Repository: hive Updated Branches: refs/heads/master 2e95037c1 -> 1907977cf
HIVE-12542: Create HiveRelFactories (Jesus Camacho Rodriguez, reviewed by Pengcheng Xiong) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1907977c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1907977c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1907977c Branch: refs/heads/master Commit: 1907977cf13450b1e19bf9420f9403d80da3dd0a Parents: 2e95037 Author: Jesus Camacho Rodriguez <[email protected]> Authored: Tue Dec 8 21:50:11 2015 +0100 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Fri Dec 18 07:33:18 2015 +0100 ---------------------------------------------------------------------- .../ql/optimizer/calcite/HiveRelFactories.java | 206 +++++++++++++++++++ .../calcite/reloperators/HiveAggregate.java | 20 -- .../calcite/reloperators/HiveFilter.java | 16 -- .../calcite/reloperators/HiveJoin.java | 31 +-- .../calcite/reloperators/HiveProject.java | 22 -- .../calcite/reloperators/HiveSemiJoin.java | 18 -- .../calcite/reloperators/HiveSortLimit.java | 17 -- .../calcite/reloperators/HiveUnion.java | 14 -- .../rules/HiveAggregateJoinTransposeRule.java | 23 ++- .../rules/HiveAggregateProjectMergeRule.java | 3 +- .../rules/HiveExpandDistinctAggregatesRule.java | 18 +- .../calcite/rules/HiveFilterJoinRule.java | 14 +- .../calcite/rules/HiveJoinAddNotNullRule.java | 3 +- .../rules/HiveJoinProjectTransposeRule.java | 36 +--- .../calcite/rules/HiveJoinToMultiJoinRule.java | 8 +- .../calcite/rules/HivePreFilteringRule.java | 4 +- .../calcite/rules/HiveProjectMergeRule.java | 4 +- .../calcite/rules/HiveRelFieldTrimmer.java | 17 +- .../calcite/rules/HiveWindowingFixRule.java | 4 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 50 +++-- 20 files changed, 283 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java new file mode 100644 index 0000000..eeec44e --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.InvalidRelException; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories.AggregateFactory; +import org.apache.calcite.rel.core.RelFactories.FilterFactory; +import org.apache.calcite.rel.core.RelFactories.JoinFactory; +import org.apache.calcite.rel.core.RelFactories.ProjectFactory; +import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory; +import org.apache.calcite.rel.core.RelFactories.SetOpFactory; +import org.apache.calcite.rel.core.RelFactories.SortFactory; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; + +import com.google.common.collect.ImmutableList; + +public class HiveRelFactories { + + public static final ProjectFactory HIVE_PROJECT_FACTORY = + new HiveProjectFactoryImpl(); + + public static final FilterFactory HIVE_FILTER_FACTORY = + new HiveFilterFactoryImpl(); + + public static final JoinFactory HIVE_JOIN_FACTORY = + new HiveJoinFactoryImpl(); + + public static final SemiJoinFactory HIVE_SEMI_JOIN_FACTORY = + new HiveSemiJoinFactoryImpl(); + + public static final SortFactory HIVE_SORT_FACTORY = + new HiveSortFactoryImpl(); + + public static final AggregateFactory HIVE_AGGREGATE_FACTORY = + new HiveAggregateFactoryImpl(); + + public static final SetOpFactory HIVE_SET_OP_FACTORY = + new HiveSetOpFactoryImpl(); + + public static final RelBuilderFactory HIVE_BUILDER = + RelBuilder.proto( + Contexts.of(HIVE_PROJECT_FACTORY, + HIVE_FILTER_FACTORY, + HIVE_JOIN_FACTORY, + HIVE_SEMI_JOIN_FACTORY, + HIVE_SORT_FACTORY, + HIVE_AGGREGATE_FACTORY, + HIVE_SET_OP_FACTORY)); + + private HiveRelFactories() { + } + + /** + * Implementation of {@link ProjectFactory} that returns + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject} + * . + */ + private static class HiveProjectFactoryImpl implements ProjectFactory { + @Override + public RelNode createProject(RelNode child, + List<? extends RexNode> childExprs, List<String> fieldNames) { + RelOptCluster cluster = child.getCluster(); + RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); + RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet()); + RelNode project = HiveProject.create(cluster, child, + childExprs, rowType, trait, Collections.<RelCollation> emptyList()); + + return project; + } + } + + /** + * Implementation of {@link FilterFactory} that returns + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter} + * . + */ + private static class HiveFilterFactoryImpl implements FilterFactory { + @Override + public RelNode createFilter(RelNode child, RexNode condition) { + RelOptCluster cluster = child.getCluster(); + HiveFilter filter = new HiveFilter(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); + return filter; + } + } + + private static class HiveJoinFactoryImpl implements JoinFactory { + /** + * Creates a join. + * + * @param left + * Left input + * @param right + * Right input + * @param condition + * Join condition + * @param joinType + * Join type + * @param variablesStopped + * Set of names of variables which are set by the LHS and used by + * the RHS and are not available to nodes above this JoinRel in the + * tree + * @param semiJoinDone + * Whether this join has been translated to a semi-join + */ + @Override + public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRelType joinType, + Set<String> variablesStopped, boolean semiJoinDone) { + return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType, false); + } + } + + /** + * Implementation of {@link SemiJoinFactory} that returns + * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin} + * . + */ + private static class HiveSemiJoinFactoryImpl implements SemiJoinFactory { + @Override + public RelNode createSemiJoin(RelNode left, RelNode right, + RexNode condition) { + final JoinInfo joinInfo = JoinInfo.of(left, right, condition); + final RelOptCluster cluster = left.getCluster(); + return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition, + joinInfo.leftKeys, joinInfo.rightKeys); + } + } + + private static class HiveSortFactoryImpl implements SortFactory { + @Override + public RelNode createSort(RelTraitSet traits, RelNode input, RelCollation collation, + RexNode offset, RexNode fetch) { + return createSort(input, collation, offset, fetch); + } + + @Override + public RelNode createSort(RelNode input, RelCollation collation, RexNode offset, + RexNode fetch) { + return HiveSortLimit.create(input, collation, offset, fetch); + } + } + + private static class HiveAggregateFactoryImpl implements AggregateFactory { + @Override + public RelNode createAggregate(RelNode child, boolean indicator, + ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets, + List<AggregateCall> aggCalls) { + try { + return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, indicator, + groupSet, groupSets, aggCalls); + } catch (InvalidRelException e) { + throw new RuntimeException(e); + } + } + } + + private static class HiveSetOpFactoryImpl implements SetOpFactory { + @Override + public RelNode createSetOp(SqlKind kind, List<RelNode> inputs, boolean all) { + if (kind != SqlKind.UNION) { + throw new IllegalStateException("Expected to get Set operator of type Union. Found : " + kind); + } + return new HiveUnion(inputs.get(0).getCluster(), inputs.get(0).getTraitSet(), inputs); + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java index 903cc19..aae57e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java @@ -29,7 +29,6 @@ import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.RelFactories.AggregateFactory; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -39,15 +38,10 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.IntList; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; public class HiveAggregate extends Aggregate implements HiveRelNode { - public static final HiveAggRelFactory HIVE_AGGR_REL_FACTORY = new HiveAggRelFactory(); - - - public HiveAggregate(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) throws InvalidRelException { @@ -139,18 +133,4 @@ public class HiveAggregate extends Aggregate implements HiveRelNode { return builder.build(); } - private static class HiveAggRelFactory implements AggregateFactory { - - @Override - public RelNode createAggregate(RelNode child, boolean indicator, - ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets, - List<AggregateCall> aggCalls) { - try { - return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, indicator, - groupSet, groupSets, aggCalls); - } catch (InvalidRelException e) { - throw new RuntimeException(e); - } - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java index eb97bec..2f8cc7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java @@ -23,15 +23,12 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.RelFactories.FilterFactory; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; public class HiveFilter extends Filter implements HiveRelNode { - public static final FilterFactory DEFAULT_FILTER_FACTORY = new HiveFilterFactoryImpl(); - public HiveFilter(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); } @@ -51,17 +48,4 @@ public class HiveFilter extends Filter implements HiveRelNode { return RelMetadataQuery.getNonCumulativeCost(this); } - /** - * Implementation of {@link FilterFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter} - * . - */ - private static class HiveFilterFactoryImpl implements FilterFactory { - @Override - public RelNode createFilter(RelNode child, RexNode condition) { - RelOptCluster cluster = child.getCluster(); - HiveFilter filter = new HiveFilter(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); - return filter; - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java index ffd3196..27b1e76 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java @@ -34,7 +34,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories.JoinFactory; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -43,8 +42,8 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCostModel.JoinAlgorithm; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveDefaultCostModel.DefaultJoinAlgorithm; @@ -54,8 +53,6 @@ import com.google.common.collect.ImmutableList; //TODO: Should we convert MultiJoin to be a child of HiveJoin public class HiveJoin extends Join implements HiveRelNode { - public static final JoinFactory HIVE_JOIN_FACTORY = new HiveJoinFactoryImpl(); - public enum MapJoinStreamingRelation { NONE, LEFT_RELATION, RIGHT_RELATION } @@ -246,30 +243,4 @@ public class HiveJoin extends Join implements HiveRelNode { return super.deriveRowType(); } - private static class HiveJoinFactoryImpl implements JoinFactory { - /** - * Creates a join. - * - * @param left - * Left input - * @param right - * Right input - * @param condition - * Join condition - * @param joinType - * Join type - * @param variablesStopped - * Set of names of variables which are set by the LHS and used by - * the RHS and are not available to nodes above this JoinRel in the - * tree - * @param semiJoinDone - * Whether this join has been translated to a semi-join - */ - @Override - public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRelType joinType, - Set<String> variablesStopped, boolean semiJoinDone) { - return getJoin(left.getCluster(), left, right, condition, joinType, false); - } - } - } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java index db625c1..142812c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java @@ -28,7 +28,6 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories.ProjectFactory; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -47,8 +46,6 @@ import com.google.common.collect.ImmutableList; public class HiveProject extends Project implements HiveRelNode { - public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl(); - private final List<Integer> virtualCols; private boolean isSysnthetic; @@ -199,23 +196,4 @@ public class HiveProject extends Project implements HiveRelNode { return isSysnthetic; } - /** - * Implementation of {@link ProjectFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject} - * . - */ - private static class HiveProjectFactoryImpl implements ProjectFactory { - - @Override - public RelNode createProject(RelNode child, - List<? extends RexNode> childExprs, List<String> fieldNames) { - RelOptCluster cluster = child.getCluster(); - RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); - RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet()); - RelNode project = HiveProject.create(cluster, child, - childExprs, rowType, trait, Collections.<RelCollation> emptyList()); - - return project; - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java index af82822..3558676 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java @@ -28,7 +28,6 @@ import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory; import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataTypeField; @@ -41,8 +40,6 @@ import com.google.common.collect.ImmutableList; public class HiveSemiJoin extends SemiJoin implements HiveRelNode { - public static final SemiJoinFactory HIVE_SEMIJOIN_FACTORY = new HiveSemiJoinFactoryImpl(); - private final RexNode joinFilter; @@ -108,19 +105,4 @@ public class HiveSemiJoin extends SemiJoin implements HiveRelNode { return RelMetadataQuery.getNonCumulativeCost(this); } - /** - * Implementation of {@link SemiJoinFactory} that returns - * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin} - * . - */ - private static class HiveSemiJoinFactoryImpl implements SemiJoinFactory { - @Override - public RelNode createSemiJoin(RelNode left, RelNode right, - RexNode condition) { - final JoinInfo joinInfo = JoinInfo.of(left, right, condition); - final RelOptCluster cluster = left.getCluster(); - return getSemiJoin(cluster, left.getTraitSet(), left, right, condition, - joinInfo.leftKeys, joinInfo.rightKeys); - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java index 6a47443..6ed2914 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortLimit.java @@ -24,7 +24,6 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rex.RexNode; import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil; @@ -33,8 +32,6 @@ import com.google.common.collect.ImmutableMap; public class HiveSortLimit extends Sort implements HiveRelNode { - public static final HiveSortRelFactory HIVE_SORT_REL_FACTORY = new HiveSortRelFactory(); - // NOTE: this is to work around Hive Calcite Limitations w.r.t OB. // 1. Calcite can not accept expressions in OB; instead it needs to be expressed // as VC in input Select. @@ -110,18 +107,4 @@ public class HiveSortLimit extends Sort implements HiveRelNode { this.ruleCreated = ruleCreated; } - private static class HiveSortRelFactory implements RelFactories.SortFactory { - - @Override - public RelNode createSort(RelTraitSet traits, RelNode input, RelCollation collation, - RexNode offset, RexNode fetch) { - return createSort(input, collation, offset, fetch); - } - - @Override - public RelNode createSort(RelNode input, RelCollation collation, RexNode offset, - RexNode fetch) { - return create(input, collation, offset, fetch); - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java index 72226e7..8b57b35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java @@ -22,16 +22,12 @@ import java.util.List; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Union; -import org.apache.calcite.sql.SqlKind; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode.Implementor; public class HiveUnion extends Union { - public static final HiveUnionRelFactory UNION_REL_FACTORY = new HiveUnionRelFactory(); - public HiveUnion(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) { super(cluster, traits, inputs, true); } @@ -44,14 +40,4 @@ public class HiveUnion extends Union { public void implement(Implementor implementor) { } - private static class HiveUnionRelFactory implements RelFactories.SetOpFactory { - - @Override - public RelNode createSetOp(SqlKind kind, List<RelNode> inputs, boolean all) { - if (kind != SqlKind.UNION) { - throw new IllegalStateException("Expected to get Set operator of type Union. Found : " + kind); - } - return new HiveUnion(inputs.get(0).getCluster(), inputs.get(0).getTraitSet(), inputs); - } - } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java index 8cbaed0..070c7ea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java @@ -16,6 +16,14 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.RelOptCost; import org.apache.calcite.plan.RelOptRuleCall; @@ -38,23 +46,15 @@ import org.apache.calcite.sql.SqlSplittableAggFunction; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.Mappings; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - /** * Planner rule that pushes an * {@link org.apache.calcite.rel.core.Aggregate} @@ -64,8 +64,9 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule { /** Extended instance of the rule that can push down aggregate functions. */ public static final HiveAggregateJoinTransposeRule INSTANCE = - new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveAggregate.HIVE_AGGR_REL_FACTORY, - HiveJoin.class, HiveJoin.HIVE_JOIN_FACTORY, HiveProject.DEFAULT_PROJECT_FACTORY, true); + new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveRelFactories.HIVE_AGGREGATE_FACTORY, + HiveJoin.class, HiveRelFactories.HIVE_JOIN_FACTORY, HiveRelFactories.HIVE_PROJECT_FACTORY, + true); private final RelFactories.AggregateFactory aggregateFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java index 0df24a3..8af8a0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; @@ -140,7 +141,7 @@ public class HiveAggregateProjectMergeRule extends RelOptRule { i < newAggregate.getRowType().getFieldCount(); i++) { posList.add(i); } - rel = HiveRelOptUtil.createProject(HiveProject.DEFAULT_PROJECT_FACTORY, + rel = HiveRelOptUtil.createProject(HiveRelFactories.HIVE_PROJECT_FACTORY, rel, posList); } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java index 73c7cac..7d7631b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java @@ -16,6 +16,13 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; @@ -30,20 +37,13 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * Planner rule that expands distinct aggregates * (such as {@code COUNT(DISTINCT x)}) from a @@ -70,7 +70,7 @@ public final class HiveExpandDistinctAggregatesRule extends RelOptRule { /** The default instance of the rule; operates only on logical expressions. */ public static final HiveExpandDistinctAggregatesRule INSTANCE = new HiveExpandDistinctAggregatesRule(HiveAggregate.class, - HiveProject.DEFAULT_PROJECT_FACTORY); + HiveRelFactories.HIVE_PROJECT_FACTORY); private static RelFactories.ProjectFactory projFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java index 86ddaa6..4b8568e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java @@ -29,15 +29,14 @@ import org.apache.calcite.plan.RelOptUtil.InputFinder; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; public abstract class HiveFilterJoinRule extends FilterJoinRule { @@ -49,8 +48,8 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { * Creates a PushFilterPastJoinRule with an explicit root operand. */ protected HiveFilterJoinRule(RelOptRuleOperand operand, String id, boolean smart, - RelFactories.FilterFactory filterFactory, RelFactories.ProjectFactory projectFactory) { - super(operand, id, smart, filterFactory, projectFactory); + RelBuilderFactory relBuilderFactory) { + super(operand, id, smart, relBuilderFactory, TRUE_PREDICATE); } /** @@ -60,8 +59,7 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { public static class HiveFilterJoinMergeRule extends HiveFilterJoinRule { public HiveFilterJoinMergeRule() { super(RelOptRule.operand(Filter.class, RelOptRule.operand(Join.class, RelOptRule.any())), - "HiveFilterJoinRule:filter", true, HiveFilter.DEFAULT_FILTER_FACTORY, - HiveProject.DEFAULT_PROJECT_FACTORY); + "HiveFilterJoinRule:filter", true, HiveRelFactories.HIVE_BUILDER); } @Override @@ -84,7 +82,7 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { public static class HiveFilterJoinTransposeRule extends HiveFilterJoinRule { public HiveFilterJoinTransposeRule() { super(RelOptRule.operand(Join.class, RelOptRule.any()), "HiveFilterJoinRule:no-filter", true, - HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.DEFAULT_PROJECT_FACTORY); + HiveRelFactories.HIVE_BUILDER); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java index 2a415d5..c8de1d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinLeafPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -55,7 +56,7 @@ public final class HiveJoinAddNotNullRule extends RelOptRule { /** The singleton. */ public static final HiveJoinAddNotNullRule INSTANCE = - new HiveJoinAddNotNullRule(HiveFilter.DEFAULT_FILTER_FACTORY); + new HiveJoinAddNotNullRule(HiveRelFactories.HIVE_FILTER_FACTORY); private final FilterFactory filterFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java index ac72ee5..f6b2d84 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinProjectTransposeRule.java @@ -17,27 +17,11 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories.ProjectFactory; import org.apache.calcite.rel.rules.JoinProjectTransposeRule; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexLocalRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexProgram; -import org.apache.calcite.rex.RexProgramBuilder; -import org.apache.calcite.util.Pair; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; @@ -49,14 +33,14 @@ public class HiveJoinProjectTransposeRule extends JoinProjectTransposeRule { operand(HiveProject.class, any()), operand(HiveProject.class, any())), "JoinProjectTransposeRule(Project-Project)", - false, HiveProject.DEFAULT_PROJECT_FACTORY); + false, HiveRelFactories.HIVE_BUILDER); public static final HiveJoinProjectTransposeRule LEFT_PROJECT = new HiveJoinProjectTransposeRule( operand(HiveJoin.class, some(operand(HiveProject.class, any()))), "JoinProjectTransposeRule(Project-Other)", - false, HiveProject.DEFAULT_PROJECT_FACTORY); + false, HiveRelFactories.HIVE_BUILDER); public static final HiveJoinProjectTransposeRule RIGHT_PROJECT = new HiveJoinProjectTransposeRule( @@ -65,7 +49,7 @@ public class HiveJoinProjectTransposeRule extends JoinProjectTransposeRule { operand(RelNode.class, any()), operand(HiveProject.class, any())), "JoinProjectTransposeRule(Other-Project)", - false, HiveProject.DEFAULT_PROJECT_FACTORY); + false, HiveRelFactories.HIVE_BUILDER); public static final HiveJoinProjectTransposeRule BOTH_PROJECT_INCLUDE_OUTER = new HiveJoinProjectTransposeRule( @@ -73,14 +57,14 @@ public class HiveJoinProjectTransposeRule extends JoinProjectTransposeRule { operand(HiveProject.class, any()), operand(HiveProject.class, any())), "Join(IncludingOuter)ProjectTransposeRule(Project-Project)", - true, HiveProject.DEFAULT_PROJECT_FACTORY); + true, HiveRelFactories.HIVE_BUILDER); public static final HiveJoinProjectTransposeRule LEFT_PROJECT_INCLUDE_OUTER = new HiveJoinProjectTransposeRule( operand(HiveJoin.class, some(operand(HiveProject.class, any()))), "Join(IncludingOuter)ProjectTransposeRule(Project-Other)", - true, HiveProject.DEFAULT_PROJECT_FACTORY); + true, HiveRelFactories.HIVE_BUILDER); public static final HiveJoinProjectTransposeRule RIGHT_PROJECT_INCLUDE_OUTER = new HiveJoinProjectTransposeRule( @@ -89,13 +73,13 @@ public class HiveJoinProjectTransposeRule extends JoinProjectTransposeRule { operand(RelNode.class, any()), operand(HiveProject.class, any())), "Join(IncludingOuter)ProjectTransposeRule(Other-Project)", - true, HiveProject.DEFAULT_PROJECT_FACTORY); + true, HiveRelFactories.HIVE_BUILDER); private HiveJoinProjectTransposeRule( RelOptRuleOperand operand, String description, - boolean includeOuter, ProjectFactory projectFactory) { - super(operand, description, includeOuter, projectFactory); + boolean includeOuter, RelBuilderFactory relBuilderFactory) { + super(operand, description, includeOuter, relBuilderFactory); } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java index a8b16cc..5d169a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java @@ -35,15 +35,15 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -55,7 +55,7 @@ import com.google.common.collect.Lists; public class HiveJoinToMultiJoinRule extends RelOptRule { public static final HiveJoinToMultiJoinRule INSTANCE = - new HiveJoinToMultiJoinRule(HiveJoin.class, HiveProject.DEFAULT_PROJECT_FACTORY); + new HiveJoinToMultiJoinRule(HiveJoin.class, HiveRelFactories.HIVE_PROJECT_FACTORY); private final ProjectFactory projectFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java index 79a627c..d37fc0e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java @@ -37,10 +37,10 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import com.google.common.collect.ImmutableList; import com.google.common.collect.LinkedHashMultimap; @@ -65,7 +65,7 @@ public class HivePreFilteringRule extends RelOptRule { private HivePreFilteringRule() { super(operand(Filter.class, operand(RelNode.class, any()))); - this.filterFactory = HiveFilter.DEFAULT_FILTER_FACTORY; + this.filterFactory = HiveRelFactories.HIVE_FILTER_FACTORY; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectMergeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectMergeRule.java index 9199b03..fc48a26 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectMergeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectMergeRule.java @@ -18,12 +18,12 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.rel.rules.ProjectMergeRule; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; public class HiveProjectMergeRule extends ProjectMergeRule { public static final HiveProjectMergeRule INSTANCE = new HiveProjectMergeRule(); public HiveProjectMergeRule() { - super(true, HiveProject.DEFAULT_PROJECT_FACTORY); + super(true, HiveRelFactories.HIVE_BUILDER); } } http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java index b543fbb..f677f68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java @@ -23,10 +23,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; @@ -49,19 +47,8 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer { protected static final Log LOG = LogFactory.getLog(HiveRelFieldTrimmer.class); - public HiveRelFieldTrimmer(SqlValidator validator, - RelOptCluster cluster, - RelFactories.ProjectFactory projectFactory, - RelFactories.FilterFactory filterFactory, - RelFactories.JoinFactory joinFactory, - RelFactories.SemiJoinFactory semiJoinFactory, - RelFactories.SortFactory sortFactory, - RelFactories.AggregateFactory aggregateFactory, - RelFactories.SetOpFactory setOpFactory) { - super(validator, - RelBuilder.proto(projectFactory, filterFactory, joinFactory, - semiJoinFactory, sortFactory, aggregateFactory, setOpFactory) - .create(cluster, null)); + public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) { + super(validator, relBuilder); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java index ff203d3..7c3e6c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java @@ -35,7 +35,7 @@ import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; /** * Rule to fix windowing issue when it is done over @@ -58,7 +58,7 @@ public class HiveWindowingFixRule extends RelOptRule { super( operand(Project.class, operand(Aggregate.class, any()))); - this.projectFactory = HiveProject.DEFAULT_PROJECT_FACTORY; + this.projectFactory = HiveRelFactories.HIVE_PROJECT_FACTORY; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/1907977c/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 36a12bf..7c5a43f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -118,6 +118,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.Unsu import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -890,8 +891,8 @@ public class CalcitePlanner extends SemanticAnalyzer { HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); hepPgmBldr.addRuleInstance(new JoinToMultiJoinRule(HiveJoin.class)); - hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoin.HIVE_JOIN_FACTORY, - HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY)); + hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveRelFactories.HIVE_JOIN_FACTORY, + HiveRelFactories.HIVE_PROJECT_FACTORY, HiveRelFactories.HIVE_FILTER_FACTORY)); HepProgram hepPgm = hepPgmBldr.build(); HepPlanner hepPlanner = new HepPlanner(hepPgm); @@ -930,7 +931,7 @@ public class CalcitePlanner extends SemanticAnalyzer { calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE, - new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY), + new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY), HiveAggregateProjectMergeRule.INSTANCE, HiveJoinCommuteRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Optimizations without stats"); @@ -988,14 +989,11 @@ public class CalcitePlanner extends SemanticAnalyzer { // The previous rules can pull up projections through join operators, // thus we run the field trimmer again to push them back down HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, - cluster, HiveProject.DEFAULT_PROJECT_FACTORY, - HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY, - HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSortLimit.HIVE_SORT_REL_FACTORY, - HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY); + HiveRelFactories.HIVE_BUILDER.create(cluster, null)); calciteOptimizedPlan = fieldTrimmer.trim(calciteOptimizedPlan); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, - new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY)); + new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY)); // 8.2. Introduce exchange operators below join/multijoin operators calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), @@ -1063,12 +1061,12 @@ public class CalcitePlanner extends SemanticAnalyzer { // HIVE-11927 is fixed. perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTransposeRule( - Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class, - HiveProject.DEFAULT_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( - HiveFilter.DEFAULT_FILTER_FACTORY), HiveFilterJoinRule.JOIN, + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( + HiveRelFactories.HIVE_FILTER_FACTORY), HiveFilterJoinRule.JOIN, HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, - HiveFilter.DEFAULT_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( - HiveFilter.DEFAULT_FILTER_FACTORY)); + HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( + HiveRelFactories.HIVE_FILTER_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, PPD for old join syntax"); @@ -1080,7 +1078,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // 4. Transitive inference for join on clauses perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveJoinPushTransitivePredicatesRule( - Join.class, HiveFilter.DEFAULT_FILTER_FACTORY)); + Join.class, HiveRelFactories.HIVE_FILTER_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Transitive inference for join on clauses"); @@ -1116,12 +1114,12 @@ public class CalcitePlanner extends SemanticAnalyzer { // HIVE-11927 is fixed. perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTransposeRule( - Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class, - HiveProject.DEFAULT_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( - HiveFilter.DEFAULT_FILTER_FACTORY), HiveFilterJoinRule.JOIN, + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( + HiveRelFactories.HIVE_FILTER_FACTORY), HiveFilterJoinRule.JOIN, HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, - HiveFilter.DEFAULT_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( - HiveFilter.DEFAULT_FILTER_FACTORY)); + HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( + HiveRelFactories.HIVE_FILTER_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Constant propagation and PPD"); @@ -1141,10 +1139,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // 10. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP) perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, - cluster, HiveProject.DEFAULT_PROJECT_FACTORY, - HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY, - HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSortLimit.HIVE_SORT_REL_FACTORY, - HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY); + HiveRelFactories.HIVE_BUILDER.create(cluster, null)); basePlan = fieldTrimmer.trim(basePlan); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Projection Pruning"); @@ -1152,7 +1147,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // 11. Merge Project-Project if possible perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, new ProjectMergeRule(true, - HiveProject.DEFAULT_PROJECT_FACTORY)); + HiveRelFactories.HIVE_PROJECT_FACTORY)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Merge Project-Project"); @@ -1162,10 +1157,11 @@ public class CalcitePlanner extends SemanticAnalyzer { // matches FIL-PROJ-TS perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTSTransposeRule( - Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class, - HiveProject.DEFAULT_PROJECT_FACTORY, HiveTableScan.class)); + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Rerun PPD"); + return basePlan; } @@ -1425,7 +1421,7 @@ public class CalcitePlanner extends SemanticAnalyzer { final List<Integer> leftKeys = new ArrayList<Integer>(); final List<Integer> rightKeys = new ArrayList<Integer>(); calciteJoinCond = HiveCalciteUtil.projectNonColumnEquiConditions( - HiveProject.DEFAULT_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0, + HiveRelFactories.HIVE_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0, leftKeys, rightKeys); joinRel = HiveSemiJoin.getSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
