DRILL-3993: Use custom RelBuilder implementation in rules After the changes, made in CALCITE-1056 if the filter has a predicate that is always false, RelBuilder.filter() method returns values rel node instead of filter rel node. In order to preserve column types, DrillRelBuilder.empty() method, which is returned by filter method was overridden, and now it returns filter with a false predicate. (advice to override this method was in its javadoc) The goal of all other changes in this commit is to use our custom RelBuilder for all rules that are used in Drill.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/0a525aab Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/0a525aab Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/0a525aab Branch: refs/heads/master Commit: 0a525aab638016a74cae11f8b2430b5a96ffc4af Parents: 3896a58 Author: Volodymyr Vysotskyi <[email protected]> Authored: Wed Nov 15 12:37:50 2017 +0200 Committer: Volodymyr Vysotskyi <[email protected]> Committed: Tue Jan 16 12:10:13 2018 +0200 ---------------------------------------------------------------------- .../exec/store/jdbc/DrillJdbcRuleBase.java | 10 +- .../exec/store/jdbc/JdbcStoragePlugin.java | 18 ++-- .../drill/exec/planner/DrillRelBuilder.java | 69 ++++++++++++ .../apache/drill/exec/planner/PlannerPhase.java | 64 +++++------ .../apache/drill/exec/planner/RuleInstance.java | 105 +++++++++++++++++++ .../drill/exec/planner/StarColumnHelper.java | 5 +- .../exec/planner/common/DrillRelOptUtil.java | 7 +- .../planner/logical/DrillAggregateRule.java | 6 +- .../planner/logical/DrillConstExecutor.java | 7 +- .../DrillFilterAggregateTransposeRule.java | 14 +-- .../planner/logical/DrillFilterJoinRules.java | 52 +++++---- .../exec/planner/logical/DrillFilterRule.java | 16 +-- .../exec/planner/logical/DrillJoinRule.java | 5 +- .../exec/planner/logical/DrillLimitRule.java | 9 +- .../planner/logical/DrillMergeProjectRule.java | 13 ++- .../exec/planner/logical/DrillProjectRule.java | 11 +- .../logical/DrillProjectSetOpTransposeRule.java | 3 +- .../logical/DrillPushFilterPastProjectRule.java | 14 ++- .../logical/DrillPushLimitToScanRule.java | 22 ++-- .../planner/logical/DrillPushProjIntoScan.java | 15 ++- .../logical/DrillPushProjectPastFilterRule.java | 5 +- .../logical/DrillPushProjectPastJoinRule.java | 3 +- .../logical/DrillReduceAggregatesRule.java | 10 +- .../logical/DrillReduceExpressionsRule.java | 7 +- .../exec/planner/logical/DrillRelFactories.java | 38 ++++++- .../exec/planner/logical/DrillScanRule.java | 7 +- .../exec/planner/logical/DrillSortRule.java | 5 +- .../exec/planner/logical/DrillUnionAllRule.java | 3 +- .../exec/planner/logical/DrillValuesRule.java | 7 +- .../exec/planner/logical/DrillWindowRule.java | 5 +- .../drill/exec/planner/physical/Prule.java | 15 ++- .../exec/planner/physical/ValuesPrule.java | 10 +- .../planner/sql/handlers/SqlHandlerUtil.java | 4 +- .../exec/store/StoragePluginOptimizerRule.java | 5 +- .../TestCorruptParquetDateCorrection.java | 70 ++++--------- .../resources/record/test_recorditerator.json | 2 +- 36 files changed, 422 insertions(+), 239 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/DrillJdbcRuleBase.java ---------------------------------------------------------------------- diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/DrillJdbcRuleBase.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/DrillJdbcRuleBase.java index bbb4daf..963e80a 100644 --- a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/DrillJdbcRuleBase.java +++ b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/DrillJdbcRuleBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,6 +20,7 @@ package org.apache.drill.exec.store.jdbc; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import com.google.common.base.Predicates; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.adapter.jdbc.JdbcRules; import org.apache.calcite.plan.Convention; @@ -34,6 +35,7 @@ import org.apache.calcite.rex.RexNode; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import org.apache.drill.exec.planner.logical.DrillRelFactories; abstract class DrillJdbcRuleBase extends ConverterRule { @@ -50,7 +52,7 @@ abstract class DrillJdbcRuleBase extends ConverterRule { protected final JdbcConvention out; private DrillJdbcRuleBase(Class<? extends RelNode> clazz, RelTrait in, JdbcConvention out, String description) { - super(clazz, in, out, description); + super(clazz, Predicates.<RelNode>alwaysTrue(), in, out, DrillRelFactories.LOGICAL_BUILDER, description); this.out = out; } @@ -71,7 +73,7 @@ abstract class DrillJdbcRuleBase extends ConverterRule { public boolean matches(RelOptRuleCall call) { try { - final LogicalProject project = (LogicalProject) call.rel(0); + final LogicalProject project = call.rel(0); for (RexNode node : project.getChildExps()) { if (!checkedExpressions.get(node)) { return false; @@ -102,7 +104,7 @@ abstract class DrillJdbcRuleBase extends ConverterRule { public boolean matches(RelOptRuleCall call) { try { - final LogicalFilter filter = (LogicalFilter) call.rel(0); + final LogicalFilter filter = call.rel(0); for (RexNode node : filter.getChildExps()) { if (!checkedExpressions.get(node)) { return false; http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java ---------------------------------------------------------------------- diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java index 9efb7af..4b41696 100755 --- a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java +++ b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,6 +28,7 @@ import java.util.Set; import javax.sql.DataSource; +import com.google.common.base.Predicates; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.adapter.jdbc.JdbcRules; import org.apache.calcite.adapter.jdbc.JdbcRules.JdbcJoin; @@ -37,8 +38,6 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.rules.FilterSetOpTransposeRule; -import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; @@ -52,7 +51,9 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.planner.RuleInstance; import org.apache.drill.exec.planner.logical.DrillRel; +import org.apache.drill.exec.planner.logical.DrillRelFactories; import org.apache.drill.exec.planner.physical.Prel; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractSchema; @@ -136,8 +137,8 @@ public class JdbcStoragePlugin extends AbstractStoragePlugin { builder.add(rule); } - builder.add(FilterSetOpTransposeRule.INSTANCE); - builder.add(ProjectRemoveRule.INSTANCE); + builder.add(RuleInstance.FILTER_SET_OP_TRANSPOSE_RULE); + builder.add(RuleInstance.PROJECT_REMOVE_RULE); rules = builder.build(); } @@ -199,13 +200,13 @@ public class JdbcStoragePlugin extends AbstractStoragePlugin { } } - private static final JdbcPrule JDBC_PRULE_INSTANCE = new JdbcPrule(); private static class JdbcPrule extends ConverterRule { private JdbcPrule() { - super(JdbcDrel.class, DrillRel.DRILL_LOGICAL, Prel.DRILL_PHYSICAL, "JDBC_PREL_Converter"); + super(JdbcDrel.class, Predicates.<RelNode>alwaysTrue(), DrillRel.DRILL_LOGICAL, + Prel.DRILL_PHYSICAL, DrillRelFactories.LOGICAL_BUILDER, "JDBC_PREL_Converter"); } @Override @@ -222,7 +223,8 @@ public class JdbcStoragePlugin extends AbstractStoragePlugin { private class JdbcDrelConverterRule extends ConverterRule { public JdbcDrelConverterRule(DrillJdbcConvention in) { - super(RelNode.class, in, DrillRel.DRILL_LOGICAL, "JDBC_DREL_Converter" + in.getName()); + super(RelNode.class, Predicates.<RelNode>alwaysTrue(), in, DrillRel.DRILL_LOGICAL, + DrillRelFactories.LOGICAL_BUILDER, "JDBC_DREL_Converter" + in.getName()); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DrillRelBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DrillRelBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DrillRelBuilder.java new file mode 100644 index 0000000..3923ba3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DrillRelBuilder.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner; + +import org.apache.calcite.plan.Context; +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptSchema; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.Util; + +public class DrillRelBuilder extends RelBuilder { + private final RelFactories.FilterFactory filterFactory; + + protected DrillRelBuilder(Context context, RelOptCluster cluster, RelOptSchema relOptSchema) { + super(context, cluster, relOptSchema); + this.filterFactory = + Util.first(context.unwrap(RelFactories.FilterFactory.class), + RelFactories.DEFAULT_FILTER_FACTORY); + } + + /** + * Original method {@link RelBuilder#empty} returns empty values rel. + * In the order to preserve dara row types, filter with false predicate is created. + */ + @Override + public RelBuilder empty() { + // pops the frame from the stack and returns its relational expression + RelNode relNode = build(); + + // creates filter with false in the predicate + final RelNode filter = filterFactory.createFilter(relNode, cluster.getRexBuilder().makeLiteral(false)); + push(filter); + + return this; + } + + /** Creates a {@link RelBuilderFactory}, a partially-created DrillRelBuilder. + * Just add a {@link RelOptCluster} and a {@link RelOptSchema} */ + public static RelBuilderFactory proto(final Context context) { + return new RelBuilderFactory() { + public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) { + return new DrillRelBuilder(context, cluster, schema); + } + }; + } + + /** Creates a {@link RelBuilderFactory} that uses a given set of factories. */ + public static RelBuilderFactory proto(Object... factories) { + return proto(Contexts.of(factories)); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java index 513db9b..28ac51e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java @@ -21,21 +21,9 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import com.google.common.collect.Lists; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.volcano.AbstractConverter.ExpandConversionRule; import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; -import org.apache.calcite.rel.rules.AggregateRemoveRule; -import org.apache.calcite.rel.rules.FilterMergeRule; -import org.apache.calcite.rel.rules.JoinPushExpressionsRule; -import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; import org.apache.calcite.rel.rules.JoinToMultiJoinRule; import org.apache.calcite.rel.rules.LoptOptimizeJoinRule; -import org.apache.calcite.rel.rules.ProjectRemoveRule; -import org.apache.calcite.rel.rules.ProjectToWindowRule; -import org.apache.calcite.rel.rules.ProjectWindowTransposeRule; -import org.apache.calcite.rel.rules.ReduceExpressionsRule; -import org.apache.calcite.rel.rules.SortRemoveRule; -import org.apache.calcite.rel.rules.UnionToDistinctRule; import org.apache.calcite.tools.RuleSet; import org.apache.calcite.tools.RuleSets; import org.apache.drill.exec.ops.OptimizerRulesContext; @@ -92,6 +80,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +/** + * Returns RuleSet for concrete planner phase. + * Only rules which use DrillRelFactories should be used in this enum. + */ public enum PlannerPhase { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class); @@ -109,8 +101,8 @@ public enum PlannerPhase { WINDOW_REWRITE("Window Function rewrites") { public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) { return RuleSets.ofList( - ReduceExpressionsRule.CALC_INSTANCE, - ProjectToWindowRule.PROJECT + RuleInstance.CALC_INSTANCE, + RuleInstance.PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW_RULE ); } }, @@ -132,7 +124,7 @@ public enum PlannerPhase { rules.add(DRILL_JOIN_TO_MULTIJOIN_RULE); rules.add(DRILL_LOPT_OPTIMIZE_JOIN_RULE); } - rules.add(ProjectRemoveRule.INSTANCE); + rules.add(RuleInstance.PROJECT_REMOVE_RULE); return PlannerPhase.mergedRuleSets( RuleSets.ofList(rules), getStorageRules(context, plugins, this) @@ -208,11 +200,13 @@ public enum PlannerPhase { } - static final RelOptRule DRILL_JOIN_TO_MULTIJOIN_RULE = new JoinToMultiJoinRule(DrillJoinRel.class); - static final RelOptRule DRILL_LOPT_OPTIMIZE_JOIN_RULE = new LoptOptimizeJoinRule( - DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY, - DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY, - DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY); + static final RelOptRule DRILL_JOIN_TO_MULTIJOIN_RULE = + new JoinToMultiJoinRule(DrillJoinRel.class, DrillRelFactories.LOGICAL_BUILDER); + static final RelOptRule DRILL_LOPT_OPTIMIZE_JOIN_RULE = + new LoptOptimizeJoinRule(DrillRelBuilder.proto( + DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY, + DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY, + DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY)); /** * Get a list of logical rules that can be turned on or off by session/system options. @@ -236,7 +230,7 @@ public enum PlannerPhase { if (ps.isConstantFoldingEnabled()) { // TODO - DRILL-2218 - userConfigurableRules.add(ReduceExpressionsRule.PROJECT_INSTANCE); + userConfigurableRules.add(RuleInstance.PROJECT_INSTANCE); userConfigurableRules.add(DrillReduceExpressionsRule.FILTER_INSTANCE_DRILL); userConfigurableRules.add(DrillReduceExpressionsRule.CALC_INSTANCE_DRILL); } @@ -250,12 +244,12 @@ public enum PlannerPhase { */ final static ImmutableSet<RelOptRule> staticRuleSet = ImmutableSet.<RelOptRule> builder().add( // Add support for Distinct Union (by using Union-All followed by Distinct) - UnionToDistinctRule.INSTANCE, + RuleInstance.UNION_TO_DISTINCT_RULE, // Add support for WHERE style joins. DrillFilterJoinRules.DRILL_FILTER_ON_JOIN, DrillFilterJoinRules.DRILL_JOIN, - JoinPushExpressionsRule.INSTANCE, + RuleInstance.JOIN_PUSH_EXPRESSIONS_RULE, // End support for WHERE style joins. /* @@ -266,12 +260,12 @@ public enum PlannerPhase { //FilterSetOpTransposeRule.INSTANCE, DrillFilterAggregateTransposeRule.INSTANCE, - FilterMergeRule.INSTANCE, - AggregateRemoveRule.INSTANCE, - ProjectRemoveRule.INSTANCE, - SortRemoveRule.INSTANCE, + RuleInstance.FILTER_MERGE_RULE, + RuleInstance.AGGREGATE_REMOVE_RULE, + RuleInstance.PROJECT_REMOVE_RULE, + RuleInstance.SORT_REMOVE_RULE, - AggregateExpandDistinctAggregatesRule.JOIN, + RuleInstance.AGGREGATE_EXPAND_DISTINCT_AGGREGATES_RULE, DrillReduceAggregatesRule.INSTANCE, /* @@ -281,13 +275,13 @@ public enum PlannerPhase { DrillPushProjectPastJoinRule.INSTANCE, // Due to infinite loop in planning (DRILL-3257), temporarily disable this rule //DrillProjectSetOpTransposeRule.INSTANCE, - ProjectWindowTransposeRule.INSTANCE, + RuleInstance.PROJECT_WINDOW_TRANSPOSE_RULE, DrillPushProjIntoScan.INSTANCE, /* Convert from Calcite Logical to Drill Logical Rules. */ - ExpandConversionRule.INSTANCE, + RuleInstance.EXPAND_CONVERSION_RULE, DrillScanRule.INSTANCE, DrillFilterRule.INSTANCE, DrillProjectRule.INSTANCE, @@ -393,9 +387,9 @@ public enum PlannerPhase { // Ruleset for join permutation, used only in VolcanoPlanner. static RuleSet getJoinPermRules(OptimizerRulesContext optimizerRulesContext) { - return RuleSets.ofList(ImmutableSet.<RelOptRule> builder().add( // - JoinPushThroughJoinRule.RIGHT, - JoinPushThroughJoinRule.LEFT + return RuleSets.ofList(ImmutableSet.<RelOptRule> builder().add( + RuleInstance.JOIN_PUSH_THROUGH_JOIN_RULE_RIGHT, + RuleInstance.JOIN_PUSH_THROUGH_JOIN_RULE_LEFT ).build()); } @@ -404,7 +398,7 @@ public enum PlannerPhase { )); static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) { - final List<RelOptRule> ruleList = new ArrayList<RelOptRule>(); + final List<RelOptRule> ruleList = new ArrayList<>(); final PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); ruleList.add(ConvertCountToDirectScan.AGG_ON_PROJ_ON_SCAN); @@ -414,7 +408,7 @@ public enum PlannerPhase { ruleList.add(ProjectPrule.INSTANCE); ruleList.add(ScanPrule.INSTANCE); ruleList.add(ScreenPrule.INSTANCE); - ruleList.add(ExpandConversionRule.INSTANCE); + ruleList.add(RuleInstance.EXPAND_CONVERSION_RULE); ruleList.add(FilterPrule.INSTANCE); ruleList.add(LimitPrule.INSTANCE); ruleList.add(WriterPrule.INSTANCE); http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java new file mode 100644 index 0000000..a390ff4 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.volcano.AbstractConverter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalCalc; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; +import org.apache.calcite.rel.rules.AggregateRemoveRule; +import org.apache.calcite.rel.rules.FilterMergeRule; +import org.apache.calcite.rel.rules.FilterSetOpTransposeRule; +import org.apache.calcite.rel.rules.JoinPushExpressionsRule; +import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; +import org.apache.calcite.rel.rules.ProjectRemoveRule; +import org.apache.calcite.rel.rules.ProjectToWindowRule; +import org.apache.calcite.rel.rules.ProjectWindowTransposeRule; +import org.apache.calcite.rel.rules.ReduceExpressionsRule; +import org.apache.calcite.rel.rules.SortRemoveRule; +import org.apache.calcite.rel.rules.UnionToDistinctRule; +import org.apache.drill.exec.planner.logical.DrillRelFactories; + +/** + * Contains rule instances which use custom RelBuilder. + */ +public interface RuleInstance { + + ReduceExpressionsRule PROJECT_INSTANCE = + new ReduceExpressionsRule.ProjectReduceExpressionsRule(LogicalProject.class, + DrillRelFactories.LOGICAL_BUILDER); + + UnionToDistinctRule UNION_TO_DISTINCT_RULE = + new UnionToDistinctRule(LogicalUnion.class, + DrillRelFactories.LOGICAL_BUILDER); + + JoinPushExpressionsRule JOIN_PUSH_EXPRESSIONS_RULE = + new JoinPushExpressionsRule(Join.class, + DrillRelFactories.LOGICAL_BUILDER); + + FilterMergeRule FILTER_MERGE_RULE = + new FilterMergeRule(DrillRelFactories.LOGICAL_BUILDER); + + AggregateRemoveRule AGGREGATE_REMOVE_RULE = + new AggregateRemoveRule(LogicalAggregate.class, DrillRelFactories.LOGICAL_BUILDER); + + AggregateExpandDistinctAggregatesRule AGGREGATE_EXPAND_DISTINCT_AGGREGATES_RULE = + new AggregateExpandDistinctAggregatesRule(LogicalAggregate.class, false, + DrillRelFactories.LOGICAL_BUILDER); + + /** + * Instance of the rule that works on logical joins only, and pushes to the + * right. + */ + RelOptRule JOIN_PUSH_THROUGH_JOIN_RULE_RIGHT = + new JoinPushThroughJoinRule("JoinPushThroughJoinRule:right", true, + LogicalJoin.class, DrillRelFactories.LOGICAL_BUILDER); + + /** + * Instance of the rule that works on logical joins only, and pushes to the + * left. + */ + RelOptRule JOIN_PUSH_THROUGH_JOIN_RULE_LEFT = + new JoinPushThroughJoinRule("JoinPushThroughJoinRule:left", false, + LogicalJoin.class, DrillRelFactories.LOGICAL_BUILDER); + + ReduceExpressionsRule CALC_INSTANCE = + new ReduceExpressionsRule.CalcReduceExpressionsRule(LogicalCalc.class, + DrillRelFactories.LOGICAL_BUILDER); + + FilterSetOpTransposeRule FILTER_SET_OP_TRANSPOSE_RULE = + new FilterSetOpTransposeRule(DrillRelFactories.LOGICAL_BUILDER); + + ProjectRemoveRule PROJECT_REMOVE_RULE = + new ProjectRemoveRule(DrillRelFactories.LOGICAL_BUILDER); + + ProjectToWindowRule PROJECT_TO_LOGICAL_PROJECT_AND_WINDOW_RULE = + new ProjectToWindowRule.ProjectToLogicalProjectAndWindowRule(DrillRelFactories.LOGICAL_BUILDER); + + SortRemoveRule SORT_REMOVE_RULE = + new SortRemoveRule(DrillRelFactories.LOGICAL_BUILDER); + + ProjectWindowTransposeRule PROJECT_WINDOW_TRANSPOSE_RULE = + new ProjectWindowTransposeRule(DrillRelFactories.LOGICAL_BUILDER); + + AbstractConverter.ExpandConversionRule EXPAND_CONVERSION_RULE = + new AbstractConverter.ExpandConversionRule(DrillRelFactories.LOGICAL_BUILDER); +} http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java index 9034f01..672af42 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,7 +20,6 @@ package org.apache.drill.exec.planner; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexInputRef; @@ -30,7 +29,7 @@ public class StarColumnHelper { public final static String PREFIX_DELIMITER = "\u00a6\u00a6"; - public final static String STAR_COLUMN = "*"; + public final static String STAR_COLUMN = "**"; public final static String PREFIXED_STAR_COLUMN = PREFIX_DELIMITER + STAR_COLUMN; http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java index b3e261c..91c33bd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +24,6 @@ import com.google.common.collect.Lists; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -40,6 +39,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; +import org.apache.drill.exec.planner.logical.DrillRelFactories; import org.apache.drill.exec.resolver.TypeCastRules; /** @@ -121,7 +121,8 @@ public abstract class DrillRelOptUtil { } }; - return RelOptUtil.createProject(rel, refs, fieldNames, false); + return RelOptUtil.createProject(rel, refs, fieldNames, false, + DrillRelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null)); } public static boolean isTrivialProject(Project project, boolean useNamesInIdentityProjCalc) { http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java index cbb5aee..70f2215 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java @@ -36,7 +36,11 @@ public class DrillAggregateRule extends RelOptRule { protected static final Logger tracer = CalciteTrace.getPlannerTracer(); private DrillAggregateRule() { - super(RelOptHelper.some(LogicalAggregate.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillAggregateRule"); + super( + RelOptHelper.some(LogicalAggregate.class, + Convention.NONE, RelOptHelper.any(RelNode.class)), + DrillRelFactories.LOGICAL_BUILDER, + "DrillAggregateRule"); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java index cdba8b8..7b52eda 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java @@ -68,7 +68,6 @@ import org.apache.drill.exec.expr.holders.TimeStampHolder; import org.apache.drill.exec.expr.holders.ValueHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; import org.apache.drill.exec.ops.UdfUtilities; -import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; @@ -316,9 +315,9 @@ public class DrillConstExecutor implements RexExecutor { milliseconds = intervalDayOut.milliseconds; } return rexBuilder.makeLiteral( - new BigDecimal(days * DateUtility.daysToStandardMillis + milliseconds), - TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY, newCall.getType().isNullable()), - false); + new BigDecimal(days * (long) DateUtility.daysToStandardMillis + milliseconds), + TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY, + newCall.getType().isNullable()), false); } // The list of known unsupported types is used to trigger this behavior of re-using the input expression // before the expression is even attempted to be evaluated, this is just here as a last precaution a http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterAggregateTransposeRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterAggregateTransposeRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterAggregateTransposeRule.java index 9d5e444..8a6c0aa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterAggregateTransposeRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterAggregateTransposeRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,7 +25,7 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; -import org.apache.calcite.tools.RelBuilder; +import org.apache.drill.exec.planner.DrillRelBuilder; public class DrillFilterAggregateTransposeRule extends FilterAggregateTransposeRule{ @@ -35,14 +35,16 @@ public class DrillFilterAggregateTransposeRule extends FilterAggregateTransposeR public static final FilterAggregateTransposeRule INSTANCE = new DrillFilterAggregateTransposeRule(); private DrillFilterAggregateTransposeRule() { - super(Filter.class, RelBuilder.proto(Contexts.of(RelFactories.DEFAULT_FILTER_FACTORY)), Aggregate.class); + super(Filter.class, DrillRelBuilder.proto(Contexts.of(RelFactories.DEFAULT_FILTER_FACTORY)), + Aggregate.class); } @Override public boolean matches(RelOptRuleCall call) { - final Filter filter = (Filter) call.rel(0); - final Aggregate aggregate = (Aggregate) call.rel(1); - return filter.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == aggregate.getTraitSet().getTrait(ConventionTraitDef.INSTANCE); + final Filter filter = call.rel(0); + final Aggregate aggregate = call.rel(1); + return filter.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) + == aggregate.getTraitSet().getTrait(ConventionTraitDef.INSTANCE); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterJoinRules.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterJoinRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterJoinRules.java index 0c1fdb3..6d186e1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterJoinRules.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterJoinRules.java @@ -1,20 +1,4 @@ -package org.apache.drill.exec.planner.logical; - -import com.google.common.collect.Lists; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.rules.FilterJoinRule; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlKind; - -import java.util.List; - -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -31,6 +15,19 @@ import java.util.List; * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.drill.exec.planner.logical; + +import com.google.common.collect.Lists; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.rules.FilterJoinRule; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.apache.drill.exec.planner.DrillRelBuilder; + +import java.util.List; public class DrillFilterJoinRules { /** Predicate that always returns true for any filter in OUTER join, and only true @@ -55,26 +52,27 @@ public class DrillFilterJoinRules { List<RelDataTypeField> sysFields = Lists.newArrayList(); List<Integer> filterNulls = Lists.newArrayList(); - RexNode remaining = RelOptUtil.splitJoinCondition(sysFields, join.getLeft(), join.getRight(), exp, tmpLeftKeys, tmpRightKeys, filterNulls, null); - - if (remaining.isAlwaysTrue()) { - return true; - } + RexNode remaining = RelOptUtil.splitJoinCondition(sysFields, join.getLeft(), join.getRight(), + exp, tmpLeftKeys, tmpRightKeys, filterNulls, null); - return false; + return remaining.isAlwaysTrue(); } }; /** Rule that pushes predicates from a Filter into the Join below them. */ public static final FilterJoinRule DRILL_FILTER_ON_JOIN = - new FilterJoinRule.FilterIntoJoinRule(true, RelFactories.DEFAULT_FILTER_FACTORY, - RelFactories.DEFAULT_PROJECT_FACTORY, EQUAL_IS_DISTINCT_FROM); + new FilterJoinRule.FilterIntoJoinRule(true, + DrillRelBuilder.proto(RelFactories.DEFAULT_FILTER_FACTORY, + RelFactories.DEFAULT_PROJECT_FACTORY), + EQUAL_IS_DISTINCT_FROM); /** Rule that pushes predicates in a Join into the inputs to the join. */ public static final FilterJoinRule DRILL_JOIN = - new FilterJoinRule.JoinConditionPushRule(RelFactories.DEFAULT_FILTER_FACTORY, - RelFactories.DEFAULT_PROJECT_FACTORY, EQUAL_IS_DISTINCT_FROM); + new FilterJoinRule.JoinConditionPushRule( + DrillRelBuilder.proto(RelFactories.DEFAULT_FILTER_FACTORY, + RelFactories.DEFAULT_PROJECT_FACTORY), + EQUAL_IS_DISTINCT_FROM); } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java index dd7da48..f3769f0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,26 +21,26 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.logical.LogicalFilter; /** - * Rule that converts a {@link org.apache.calcite.rel.FilterRel} to a Drill "filter" operation. + * Rule that converts a {@link org.apache.calcite.rel.logical.LogicalFilter} to a Drill "filter" operation. */ public class DrillFilterRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillFilterRule(); private DrillFilterRule() { - super(RelOptHelper.any(LogicalFilter.class, Convention.NONE), "DrillFilterRule"); + super(RelOptHelper.any(LogicalFilter.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillFilterRule"); } @Override public void onMatch(RelOptRuleCall call) { - final LogicalFilter filter = (LogicalFilter) call.rel(0); + final LogicalFilter filter = call.rel(0); final RelNode input = filter.getInput(); - //final RelTraitSet traits = filter.getTraitSet().plus(DrillRel.DRILL_LOGICAL); final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL)); - call.transformTo(new DrillFilterRel(filter.getCluster(), convertedInput.getTraitSet(), convertedInput, filter.getCondition())); + call.transformTo(new DrillFilterRel( + filter.getCluster(), convertedInput.getTraitSet(), + convertedInput, filter.getCondition())); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java index deb8ace..766509e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java @@ -46,8 +46,8 @@ public class DrillJoinRule extends RelOptRule { protected static final Logger tracer = CalciteTrace.getPlannerTracer(); private DrillJoinRule() { - super( - RelOptHelper.any(LogicalJoin.class, Convention.NONE), + super(RelOptHelper.any(LogicalJoin.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillJoinRule"); } @@ -64,7 +64,6 @@ public class DrillJoinRule extends RelOptRule { List<Integer> leftKeys = Lists.newArrayList(); List<Integer> rightKeys = Lists.newArrayList(); List<Boolean> filterNulls = Lists.newArrayList(); - int numLeftFields = convertedLeft.getRowType().getFieldCount(); boolean addFilter = false; RexNode origJoinCondition = join.getCondition(); http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java index 03ef1e8..6d2da9d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLimitRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,7 +32,8 @@ public class DrillLimitRule extends RelOptRule { public static DrillLimitRule INSTANCE = new DrillLimitRule(); private DrillLimitRule() { - super(RelOptHelper.any(LogicalSort.class, Convention.NONE), "DrillLimitRule"); + super(RelOptHelper.any(LogicalSort.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillLimitRule"); } @Override @@ -54,7 +55,9 @@ public class DrillLimitRule extends RelOptRule { } RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL)); - call.transformTo(new DrillLimitRel(incomingSort.getCluster(), convertedInput.getTraitSet().plus(DrillRel.DRILL_LOGICAL), convertedInput, incomingSort.offset, incomingSort.fetch)); + call.transformTo(new DrillLimitRel( + incomingSort.getCluster(), convertedInput.getTraitSet().plus(DrillRel.DRILL_LOGICAL), + convertedInput, incomingSort.offset, incomingSort.fetch)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillMergeProjectRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillMergeProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillMergeProjectRule.java index 6c69e28..94964ef 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillMergeProjectRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillMergeProjectRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,22 +27,25 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.calcite.rel.core.RelFactories.ProjectFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; +import org.apache.drill.exec.planner.DrillRelBuilder; public class DrillMergeProjectRule extends ProjectMergeRule { private FunctionImplementationRegistry functionRegistry; private static DrillMergeProjectRule INSTANCE = null; - public static DrillMergeProjectRule getInstance(boolean force, ProjectFactory pFactory, FunctionImplementationRegistry functionRegistry) { + public static DrillMergeProjectRule getInstance(boolean force, ProjectFactory pFactory, + FunctionImplementationRegistry functionRegistry) { if (INSTANCE == null) { INSTANCE = new DrillMergeProjectRule(force, pFactory, functionRegistry); } return INSTANCE; } - private DrillMergeProjectRule(boolean force, ProjectFactory pFactory, FunctionImplementationRegistry functionRegistry) { - super(force, pFactory); - this.functionRegistry = functionRegistry; + private DrillMergeProjectRule(boolean force, ProjectFactory pFactory, + FunctionImplementationRegistry functionRegistry) { + super(force, DrillRelBuilder.proto(pFactory)); + this.functionRegistry = functionRegistry; } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java index e44df33..14c6533 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,16 +32,17 @@ public class DrillProjectRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillProjectRule(); private DrillProjectRule() { - super(RelOptHelper.any(LogicalProject.class, Convention.NONE), "DrillProjectRule"); + super(RelOptHelper.any(LogicalProject.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillProjectRule"); } @Override public void onMatch(RelOptRuleCall call) { - final Project project = (Project) call.rel(0); + final Project project = call.rel(0); final RelNode input = project.getInput(); final RelTraitSet traits = project.getTraitSet().plus(DrillRel.DRILL_LOGICAL); final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL)); - call.transformTo(new DrillProjectRel(project.getCluster(), traits, convertedInput, project.getProjects(), project - .getRowType())); + call.transformTo(new DrillProjectRel( + project.getCluster(), traits, convertedInput, project.getProjects(), project.getRowType())); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectSetOpTransposeRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectSetOpTransposeRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectSetOpTransposeRule.java index baff3b7..86a39dd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectSetOpTransposeRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectSetOpTransposeRule.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.planner.logical; import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.rules.ProjectSetOpTransposeRule; import org.apache.calcite.rel.rules.PushProjector; @@ -27,6 +26,6 @@ public class DrillProjectSetOpTransposeRule extends ProjectSetOpTransposeRule { public final static RelOptRule INSTANCE = new DrillProjectSetOpTransposeRule(DrillConditions.PRESERVE_ITEM); protected DrillProjectSetOpTransposeRule(PushProjector.ExprCondition preserveExprCondition) { - super(preserveExprCondition, RelFactories.LOGICAL_BUILDER); + super(preserveExprCondition, DrillRelFactories.LOGICAL_BUILDER); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java index c2dbfb9..d24abcc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,6 +27,7 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.util.Pair; import org.apache.drill.exec.planner.common.DrillRelOptUtil; import java.util.List; @@ -39,7 +40,8 @@ public class DrillPushFilterPastProjectRule extends RelOptRule { super( operand( LogicalFilter.class, - operand(LogicalProject.class, any()))); + operand(LogicalProject.class, any())), + DrillRelFactories.LOGICAL_BUILDER, null); } //~ Methods ---------------------------------------------------------------- @@ -73,15 +75,17 @@ public class DrillPushFilterPastProjectRule extends RelOptRule { // convert the filter to one that references the child of the project RexNode newCondition = - RelOptUtil.pushFilterPastProject(qualifedPred, projRel); + RelOptUtil.pushPastProject(qualifedPred, projRel); Filter newFilterRel = LogicalFilter.create(projRel.getInput(), newCondition); Project newProjRel = (Project) RelOptUtil.createProject( newFilterRel, - projRel.getNamedProjects(), - false); + Pair.left(projRel.getNamedProjects()), + Pair.right(projRel.getNamedProjects()), + false, + relBuilderFactory.create(projRel.getCluster(), null)); final RexNode unqualifiedPred = RexUtil.composeConjunction(filterRel.getCluster().getRexBuilder(), unqualifiedPredList, true); http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java index 8ce26c8..9c06897 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -23,23 +23,18 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.util.Pair; import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.logical.partition.PruneScanRule; -import org.apache.drill.exec.store.parquet.ParquetGroupScan; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; public abstract class DrillPushLimitToScanRule extends RelOptRule { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillPushLimitToScanRule.class); private DrillPushLimitToScanRule(RelOptRuleOperand operand, String description) { - super(operand, description); + super(operand, DrillRelFactories.LOGICAL_BUILDER, description); } - public static DrillPushLimitToScanRule LIMIT_ON_SCAN = new DrillPushLimitToScanRule( - RelOptHelper.some(DrillLimitRel.class, RelOptHelper.any(DrillScanRel.class)), "DrillPushLimitToScanRule_LimitOnScan") { + public static DrillPushLimitToScanRule LIMIT_ON_SCAN = + new DrillPushLimitToScanRule(RelOptHelper.some(DrillLimitRel.class, RelOptHelper.any(DrillScanRel.class)), + "DrillPushLimitToScanRule_LimitOnScan") { @Override public boolean matches(RelOptRuleCall call) { DrillLimitRel limitRel = call.rel(0); @@ -60,8 +55,11 @@ public abstract class DrillPushLimitToScanRule extends RelOptRule { } }; - public static DrillPushLimitToScanRule LIMIT_ON_PROJECT = new DrillPushLimitToScanRule( - RelOptHelper.some(DrillLimitRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), "DrillPushLimitToScanRule_LimitOnProject") { + public static DrillPushLimitToScanRule LIMIT_ON_PROJECT = + new DrillPushLimitToScanRule( + RelOptHelper.some(DrillLimitRel.class, RelOptHelper.some( + DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), + "DrillPushLimitToScanRule_LimitOnProject") { @Override public boolean matches(RelOptRuleCall call) { DrillLimitRel limitRel = call.rel(0); http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java index b15a843..f6bc6b7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -38,17 +38,16 @@ import com.google.common.collect.Lists; public class DrillPushProjIntoScan extends RelOptRule { public static final RelOptRule INSTANCE = new DrillPushProjIntoScan(LogicalProject.class, EnumerableTableScan.class); - public static final RelOptRule DRILL_LOGICAL_INSTANCE = new DrillPushProjIntoScan(DrillProjectRel.class, DrillScanRel.class); - - private DrillPushProjIntoScan(Class<? extends Project> projectClass, Class<? extends TableScan> scanClass) { - super(RelOptHelper.some(projectClass, RelOptHelper.any(scanClass)), "DrillPushProjIntoScan"); + private DrillPushProjIntoScan(Class<? extends Project> projectClass, + Class<? extends TableScan> scanClass) { + super(RelOptHelper.some(projectClass, RelOptHelper.any(scanClass)), + DrillRelFactories.LOGICAL_BUILDER, "DrillPushProjIntoScan"); } - @Override public void onMatch(RelOptRuleCall call) { - final Project proj = (Project) call.rel(0); - final TableScan scan = (TableScan) call.rel(1); + final Project proj = call.rel(0); + final TableScan scan = call.rel(1); try { ProjectPushInfo columnInfo = PrelUtil.getColumns(scan.getRowType(), proj.getProjects()); http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastFilterRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastFilterRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastFilterRule.java index e5c40b9..6ea2c92 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastFilterRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastFilterRule.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.planner.logical; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; @@ -29,7 +28,7 @@ public class DrillPushProjectPastFilterRule extends ProjectFilterTransposeRule { public final static RelOptRule INSTANCE = new DrillPushProjectPastFilterRule(DrillConditions.PRESERVE_ITEM); protected DrillPushProjectPastFilterRule(PushProjector.ExprCondition preserveExprCondition) { - super(LogicalProject.class, LogicalFilter.class, RelFactories.LOGICAL_BUILDER, preserveExprCondition); + super(LogicalProject.class, LogicalFilter.class, + DrillRelFactories.LOGICAL_BUILDER, preserveExprCondition); } - } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastJoinRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastJoinRule.java index 269f6d5..972a141 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastJoinRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectPastJoinRule.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.planner.logical; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.rules.ProjectJoinTransposeRule; import org.apache.calcite.rel.rules.PushProjector; import org.apache.calcite.plan.RelOptRule; @@ -27,7 +26,7 @@ public class DrillPushProjectPastJoinRule extends ProjectJoinTransposeRule { public static final RelOptRule INSTANCE = new DrillPushProjectPastJoinRule(DrillConditions.PRESERVE_ITEM); protected DrillPushProjectPastJoinRule(PushProjector.ExprCondition preserveExprCondition) { - super(preserveExprCondition, RelFactories.LOGICAL_BUILDER); + super(preserveExprCondition, DrillRelFactories.LOGICAL_BUILDER); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java index bb972d0..21ff9a9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java @@ -99,7 +99,7 @@ public class DrillReduceAggregatesRule extends RelOptRule { //~ Constructors ----------------------------------------------------------- protected DrillReduceAggregatesRule(RelOptRuleOperand operand) { - super(operand); + super(operand, DrillRelFactories.LOGICAL_BUILDER, null); } //~ Methods ---------------------------------------------------------------- @@ -204,7 +204,9 @@ public class DrillReduceAggregatesRule extends RelOptRule { input.getRowType().getFieldNames(), Collections.<String>nCopies( extraArgCount, - null))); + null)), + false, + relBuilderFactory.create(input.getCluster(), null)); } Aggregate newAggRel = newAggregateRel( @@ -661,7 +663,7 @@ public class DrillReduceAggregatesRule extends RelOptRule { protected static final Logger tracer = CalciteTrace.getPlannerTracer(); public DrillConvertSumToSumZero(RelOptRuleOperand operand) { - super(operand); + super(operand, DrillRelFactories.LOGICAL_BUILDER, null); } @Override @@ -725,7 +727,7 @@ public class DrillReduceAggregatesRule extends RelOptRule { private static class DrillConvertWindowSumToSumZero extends RelOptRule { public DrillConvertWindowSumToSumZero(RelOptRuleOperand operand) { - super(operand); + super(operand, DrillRelFactories.LOGICAL_BUILDER, null); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceExpressionsRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceExpressionsRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceExpressionsRule.java index 96bcf02..6214912 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceExpressionsRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceExpressionsRule.java @@ -20,12 +20,9 @@ package org.apache.drill.exec.planner.logical; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.core.Calc; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.SingleRel; -import org.apache.calcite.rel.logical.LogicalCalc; -import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.rules.ReduceExpressionsRule; @@ -42,7 +39,7 @@ public class DrillReduceExpressionsRule { private static class DrillReduceFilterRule extends ReduceExpressionsRule.FilterReduceExpressionsRule { DrillReduceFilterRule() { - super(DrillFilterRel.class, RelFactories.LOGICAL_BUILDER); + super(Filter.class, DrillRelFactories.LOGICAL_BUILDER); } /** @@ -61,7 +58,7 @@ public class DrillReduceExpressionsRule { private static class DrillReduceCalcRule extends ReduceExpressionsRule.CalcReduceExpressionsRule { DrillReduceCalcRule() { - super(Calc.class, RelFactories.LOGICAL_BUILDER); + super(Calc.class, DrillRelFactories.LOGICAL_BUILDER); } /** http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java index 487da4d..ce35612 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.logical; +import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.CorrelationId; @@ -27,10 +28,23 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.drill.exec.planner.DrillRelBuilder; import java.util.List; import java.util.Set; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_AGGREGATE_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_FILTER_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_JOIN_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_MATCH_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_PROJECT_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_SEMI_JOIN_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_SET_OP_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_SORT_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_TABLE_SCAN_FACTORY; +import static org.apache.calcite.rel.core.RelFactories.DEFAULT_VALUES_FACTORY; + /** * Contains factory implementation for creating various Drill Logical Rel nodes. */ @@ -45,6 +59,22 @@ public class DrillRelFactories { public static final RelFactories.JoinFactory DRILL_LOGICAL_JOIN_FACTORY = new DrillJoinFactoryImpl(); + /** + * A {@link RelBuilderFactory} that creates a {@link DrillRelBuilder} that will + * create logical relational expressions for everything. + */ + public static final RelBuilderFactory LOGICAL_BUILDER = + DrillRelBuilder.proto( + Contexts.of(DEFAULT_PROJECT_FACTORY, + DEFAULT_FILTER_FACTORY, + DEFAULT_JOIN_FACTORY, + DEFAULT_SEMI_JOIN_FACTORY, + DEFAULT_SORT_FACTORY, + DEFAULT_AGGREGATE_FACTORY, + DEFAULT_MATCH_FACTORY, + DEFAULT_SET_OP_FACTORY, + DEFAULT_VALUES_FACTORY, + DEFAULT_TABLE_SCAN_FACTORY)); /** * Implementation of {@link RelFactories.ProjectFactory} that returns a vanilla @@ -55,14 +85,13 @@ public class DrillRelFactories { public RelNode createProject(RelNode child, List<? extends RexNode> childExprs, List<String> fieldNames) { final RelOptCluster cluster = child.getCluster(); - final RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); - final RelNode project = DrillProjectRel.create(cluster, child.getTraitSet(), child, childExprs, rowType); + final RelDataType rowType = + RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames, null); - return project; + return DrillProjectRel.create(cluster, child.getTraitSet(), child, childExprs, rowType); } } - /** * Implementation of {@link RelFactories.FilterFactory} that * returns a vanilla {@link LogicalFilter}. @@ -74,7 +103,6 @@ public class DrillRelFactories { } } - /** * Implementation of {@link RelFactories.JoinFactory} that returns a vanilla * {@link org.apache.calcite.rel.logical.LogicalJoin}. http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java index 9a63cb2..722ea85 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,12 +27,13 @@ public class DrillScanRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillScanRule(); private DrillScanRule() { - super(RelOptHelper.any(EnumerableTableScan.class), "DrillTableRule"); + super(RelOptHelper.any(EnumerableTableScan.class), + DrillRelFactories.LOGICAL_BUILDER, "DrillTableRule"); } @Override public void onMatch(RelOptRuleCall call) { - final EnumerableTableScan access = (EnumerableTableScan) call.rel(0); + final EnumerableTableScan access = call.rel(0); final RelTraitSet traits = access.getTraitSet().plus(DrillRel.DRILL_LOGICAL); call.transformTo(new DrillScanRel(access.getCluster(), traits, access.getTable())); } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java index 5ae2b39..1ab061b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSortRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -31,7 +31,8 @@ public class DrillSortRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillSortRule(); private DrillSortRule() { - super(RelOptHelper.any(Sort.class, Convention.NONE), "DrillSortRule"); + super(RelOptHelper.any(Sort.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillSortRule"); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionAllRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionAllRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionAllRule.java index a074952..91697a2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionAllRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionAllRule.java @@ -39,7 +39,8 @@ public class DrillUnionAllRule extends RelOptRule { protected static final Logger tracer = CalciteTrace.getPlannerTracer(); private DrillUnionAllRule() { - super(RelOptHelper.any(LogicalUnion.class, Convention.NONE), "DrillUnionRule"); + super(RelOptHelper.any(LogicalUnion.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillUnionRule"); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java index 066e57b..692afbb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,12 +30,13 @@ public class DrillValuesRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillValuesRule(); private DrillValuesRule() { - super(RelOptHelper.any(LogicalValues.class, Convention.NONE), "DrillValuesRule"); + super(RelOptHelper.any(LogicalValues.class, Convention.NONE), + DrillRelFactories.LOGICAL_BUILDER, "DrillValuesRule"); } @Override public void onMatch(RelOptRuleCall call) { - final LogicalValues values = (LogicalValues) call.rel(0); + final LogicalValues values = call.rel(0); final RelTraitSet traits = values.getTraitSet().plus(DrillRel.DRILL_LOGICAL); call.transformTo(new DrillValuesRel(values.getCluster(), values.getRowType(), values.getTuples(), traits)); } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillWindowRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillWindowRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillWindowRule.java index 1cefcd9..4461796 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillWindowRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillWindowRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -29,7 +29,8 @@ public class DrillWindowRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillWindowRule(); private DrillWindowRule() { - super(RelOptHelper.some(Window.class, Convention.NONE, RelOptHelper.any(RelNode.class)), "DrillWindowRule"); + super(RelOptHelper.some(Window.class, Convention.NONE, RelOptHelper.any(RelNode.class)), + DrillRelFactories.LOGICAL_BUILDER, "DrillWindowRule"); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prule.java index e0a2839..148e74b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,18 +22,18 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelTraitSet; +import org.apache.drill.exec.planner.logical.DrillRelFactories; -public abstract class Prule extends RelOptRule{ +public abstract class Prule extends RelOptRule { public Prule(RelOptRuleOperand operand, String description) { - super(operand, description); + super(operand, DrillRelFactories.LOGICAL_BUILDER, description); } public Prule(RelOptRuleOperand operand) { - super(operand); + this(operand, null); } - - public static RelNode convert(RelNode rel, RelTraitSet toTraits){ + public static RelNode convert(RelNode rel, RelTraitSet toTraits) { toTraits = toTraits.simplify(); PlannerSettings settings = PrelUtil.getSettings(rel.getCluster()); @@ -44,8 +44,7 @@ public abstract class Prule extends RelOptRule{ return RelOptRule.convert(rel, toTraits); } - public static boolean isSingleMode(RelOptRuleCall call){ + public static boolean isSingleMode(RelOptRuleCall call) { return PrelUtil.getPlannerSettings(call.getPlanner()).isSingleMode(); } - } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java index d1f2e28..1251ddf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.physical; import org.apache.calcite.plan.RelOptRule; +import org.apache.drill.exec.planner.logical.DrillRelFactories; import org.apache.drill.exec.planner.logical.DrillValuesRel; import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.calcite.plan.RelOptRuleCall; @@ -26,14 +27,15 @@ public class ValuesPrule extends RelOptRule { public static final ValuesPrule INSTANCE = new ValuesPrule(); - private ValuesPrule(){ - super(RelOptHelper.any(DrillValuesRel.class), "Prel.ValuesPrule"); + private ValuesPrule() { + super(RelOptHelper.any(DrillValuesRel.class), DrillRelFactories.LOGICAL_BUILDER, "Prel.ValuesPrule"); } @Override public void onMatch(final RelOptRuleCall call) { - final DrillValuesRel rel = (DrillValuesRel) call.rel(0); - call.transformTo(new ValuesPrel(rel.getCluster(), rel.getRowType(), rel.getTuples(), rel.getTraitSet().plus(Prel.DRILL_PHYSICAL), rel.getContent())); + final DrillValuesRel rel = call.rel(0); + call.transformTo(new ValuesPrel(rel.getCluster(), rel.getRowType(), rel.getTuples(), + rel.getTraitSet().plus(Prel.DRILL_PHYSICAL), rel.getContent())); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java index 72d2699..69458d4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java @@ -32,6 +32,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.planner.StarColumnHelper; import org.apache.drill.exec.planner.common.DrillRelOptUtil; +import org.apache.drill.exec.planner.logical.DrillRelFactories; import org.apache.drill.exec.store.AbstractSchema; import org.apache.calcite.tools.ValidationException; @@ -203,7 +204,8 @@ public class SqlHandlerUtil { } }; - return RelOptUtil.createProject(input, refs, names, false); + return RelOptUtil.createProject(input, refs, names, false, + DrillRelFactories.LOGICAL_BUILDER.create(input.getCluster(), null)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginOptimizerRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginOptimizerRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginOptimizerRule.java index 8b4f0e6..f81c783 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginOptimizerRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginOptimizerRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,11 +19,12 @@ package org.apache.drill.exec.store; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.drill.exec.planner.logical.DrillRelFactories; public abstract class StoragePluginOptimizerRule extends RelOptRule { public StoragePluginOptimizerRule(RelOptRuleOperand operand, String description) { - super(operand, description); + super(operand, DrillRelFactories.LOGICAL_BUILDER, description); } } http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java index 1013511..507e102 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java @@ -23,9 +23,11 @@ import org.apache.drill.categories.ParquetTest; import org.apache.drill.PlanTestBase; import org.apache.drill.test.TestBuilder; import org.apache.drill.categories.UnlikelyTest; +import org.apache.drill.common.exceptions.UserRemoteException; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.store.parquet.Metadata; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -188,34 +190,26 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase { .go(); } - @Test - public void testReadPartitionedOnCorruptedDates_UserDisabledCorrection() throws Exception { + // according to SQL spec. '4.4.3.5 Datetime types' year should be less than 9999 + @Test(expected = UserRemoteException.class) + public void testQueryWithCorruptedDates() throws Exception { try { - for (String selection : new String[]{"*", "date_col"}) { - for (Path table : new Path[]{CORRUPTED_PARTITIONED_DATES_1_2_PATH, CORRUPTED_PARTITIONED_DATES_1_4_0_PATH}) { - // for sanity, try reading all partitions without a filter - TestBuilder builder = testBuilder() - .sqlQuery("select %s from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))", - selection, table) - .unOrdered() - .baselineColumns("date_col"); - addCorruptedDateBaselineValues(builder); - builder.go(); + TestBuilder builder = testBuilder() + .sqlQuery("select * from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))", + CORRUPTED_PARTITIONED_DATES_1_2_PATH) + .unOrdered() + .baselineColumns("date_col"); + addCorruptedDateBaselineValues(builder); + builder.go(); - String query = format("select %s from table(dfs.`%s` (type => 'parquet', " + - "autoCorrectCorruptDates => false)) where date_col = cast('15334-03-17' as date)", selection, table); - // verify that pruning is actually taking place - testPlanMatchingPatterns(query, new String[]{"numFiles=1"}, null); + String query = "select * from table(dfs.`%s` (type => 'parquet', " + + "autoCorrectCorruptDates => false)) where date_col = cast('15334-03-17' as date)"; - // read with a filter on the partition column - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("date_col") - .baselineValues(new DateTime(15334, 3, 17, 0, 0)) - .go(); - } - } + test(query, CORRUPTED_PARTITIONED_DATES_1_2_PATH); + } catch (UserRemoteException e) { + Assert.assertTrue(e.getMessage() + .contains("Year out of range")); + throw e; } finally { test("alter session reset all"); } @@ -347,32 +341,6 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase { } @Test - public void testReadOldMetadataCacheFileOverrideCorrection() throws Exception { - // for sanity, try reading all partitions without a filter - TestBuilder builder = testBuilder() - .sqlQuery("select date_col from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))", - PARTITIONED_1_2_FOLDER) - .unOrdered() - .baselineColumns("date_col"); - addCorruptedDateBaselineValues(builder); - builder.go(); - - String query = format("select date_col from table(dfs.`%s` (type => 'parquet', " + - "autoCorrectCorruptDates => false)) where date_col = cast('15334-03-17' as date)", - PARTITIONED_1_2_FOLDER); - // verify that pruning is actually taking place - testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null); - - // read with a filter on the partition column - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("date_col") - .baselineValues(new DateTime(15334, 3, 17, 0, 0)) - .go(); - } - - @Test public void testReadNewMetadataCacheFileOverOldAndNewFiles() throws Exception { File meta = dirTestWatcher.copyResourceToRoot( PARQUET_4203.resolve("mixed_version_partitioned_metadata.requires_replace.txt"), http://git-wip-us.apache.org/repos/asf/drill/blob/0a525aab/exec/java-exec/src/test/resources/record/test_recorditerator.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/record/test_recorditerator.json b/exec/java-exec/src/test/resources/record/test_recorditerator.json index 057466b..c822803 100644 --- a/exec/java-exec/src/test/resources/record/test_recorditerator.json +++ b/exec/java-exec/src/test/resources/record/test_recorditerator.json @@ -60,7 +60,7 @@ @id:2, child:1, pop:"project", - exprs:[ { ref : "`*`", expr : "`*`"} ] + exprs:[ { ref : "`**`", expr : "`**`"} ] }, { @id:3,
