HIVE-16888: Upgrade Calcite to 1.13 and Avatica to 1.10 (Remus Rusanu and Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a567f86 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a567f86 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a567f86 Branch: refs/heads/master Commit: 4a567f86504b81e74cd0f7e5fb5375712ce4a62f Parents: 65b351b Author: Jesus Camacho Rodriguez <jcama...@apache.org> Authored: Tue Jun 27 11:05:48 2017 +0100 Committer: Jesus Camacho Rodriguez <jcama...@apache.org> Committed: Thu Jul 13 20:20:35 2017 +0200 ---------------------------------------------------------------------- .../positive/accumulo_predicate_pushdown.q.out | 3 +- .../results/positive/accumulo_queries.q.out | 2 +- data/conf/hive-log4j2.properties | 5 +- .../results/positive/hbase_ppd_key_range.q.out | 3 +- .../test/results/positive/hbase_queries.q.out | 2 +- .../test/results/positive/ppd_key_ranges.q.out | 34 +- pom.xml | 6 +- .../ql/optimizer/calcite/HiveCalciteUtil.java | 6 + .../optimizer/calcite/HiveTypeSystemImpl.java | 17 +- .../HiveDruidProjectFilterTransposeRule.java | 48 - .../HiveMaterializedViewFilterScanRule.java | 92 - .../rules/views/HiveMaterializedViewRule.java | 38 + .../MaterializedViewSubstitutionVisitor.java | 292 --- .../rules/views/SubstitutionVisitor.java | 2458 ------------------ .../calcite/translator/ASTBuilder.java | 31 +- .../calcite/translator/ExprNodeConverter.java | 22 +- .../calcite/translator/RexNodeConverter.java | 62 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 10 +- .../clientpositive/materialized_view_create.q | 6 + .../materialized_view_create_rewrite.q | 2 + .../results/clientpositive/auto_join12.q.out | 22 +- .../results/clientpositive/auto_join16.q.out | 16 +- .../results/clientpositive/auto_join4.q.out | 6 +- .../results/clientpositive/auto_join5.q.out | 6 +- .../results/clientpositive/auto_join8.q.out | 6 +- .../materialized_view_create_rewrite.q.out | 8 + .../clientpositive/cbo_rp_auto_join1.q.out | 18 +- .../clientpositive/cbo_rp_outer_join_ppr.q.out | 44 +- .../results/clientpositive/druid_basic2.q.out | 103 +- .../clientpositive/druid_intervals.q.out | 130 +- .../clientpositive/druid_timeseries.q.out | 6 +- .../results/clientpositive/druid_topn.q.out | 268 +- .../results/clientpositive/filter_union.q.out | 356 +-- .../clientpositive/fouter_join_ppr.q.out | 84 +- .../clientpositive/groupby_position.q.out | 32 +- .../clientpositive/index_auto_mult_tables.q.out | 52 +- .../index_auto_mult_tables_compact.q.out | 52 +- ql/src/test/results/clientpositive/join12.q.out | 30 +- ql/src/test/results/clientpositive/join16.q.out | 22 +- ql/src/test/results/clientpositive/join4.q.out | 8 +- ql/src/test/results/clientpositive/join5.q.out | 8 +- ql/src/test/results/clientpositive/join8.q.out | 8 +- .../clientpositive/join_filters_overlap.q.out | 4 +- .../llap/auto_smb_mapjoin_14.q.out | 42 +- .../llap/auto_sortmerge_join_9.q.out | 96 +- .../llap/bucket_map_join_tez2.q.out | 34 +- .../clientpositive/llap/filter_union.q.out | 236 +- .../clientpositive/llap/smb_mapjoin_14.q.out | 54 +- .../clientpositive/llap/subquery_scalar.q.out | 236 +- .../llap/vector_auto_smb_mapjoin_14.q.out | 42 +- .../clientpositive/llap/vector_date_1.q.out | 2 +- .../clientpositive/louter_join_ppr.q.out | 84 +- .../test/results/clientpositive/masking_4.q.out | 16 +- .../materialized_view_create.q.out | 40 + .../materialized_view_create_rewrite.q.out | 8 + .../results/clientpositive/outer_join_ppr.q.out | 40 +- .../results/clientpositive/perf/query1.q.out | 34 +- .../results/clientpositive/perf/query30.q.out | 94 +- .../results/clientpositive/perf/query6.q.out | 304 ++- .../results/clientpositive/perf/query81.q.out | 96 +- .../test/results/clientpositive/ppd_gby.q.out | 32 +- .../test/results/clientpositive/ppd_gby2.q.out | 44 +- .../results/clientpositive/ppd_gby_join.q.out | 60 +- .../test/results/clientpositive/ppd_join.q.out | 48 +- .../clientpositive/ppd_outer_join1.q.out | 40 +- .../clientpositive/ppd_outer_join2.q.out | 40 +- .../clientpositive/ppd_outer_join3.q.out | 40 +- .../clientpositive/ppd_outer_join4.q.out | 60 +- .../test/results/clientpositive/ppd_union.q.out | 82 +- ql/src/test/results/clientpositive/ppd_vc.q.out | 18 +- .../clientpositive/rcfile_null_value.q.out | 8 +- .../clientpositive/router_join_ppr.q.out | 84 +- .../clientpositive/spark/auto_join12.q.out | 22 +- .../clientpositive/spark/auto_join16.q.out | 16 +- .../clientpositive/spark/auto_join4.q.out | 6 +- .../clientpositive/spark/auto_join5.q.out | 6 +- .../clientpositive/spark/auto_join8.q.out | 6 +- .../spark/auto_smb_mapjoin_14.q.out | 24 +- .../spark/auto_sortmerge_join_9.q.out | 66 +- .../spark/bucket_map_join_tez2.q.out | 28 +- .../clientpositive/spark/groupby_position.q.out | 38 +- .../results/clientpositive/spark/join12.q.out | 30 +- .../results/clientpositive/spark/join16.q.out | 22 +- .../results/clientpositive/spark/join4.q.out | 8 +- .../results/clientpositive/spark/join5.q.out | 8 +- .../results/clientpositive/spark/join8.q.out | 8 +- .../spark/join_filters_overlap.q.out | 4 +- .../clientpositive/spark/louter_join_ppr.q.out | 84 +- .../clientpositive/spark/outer_join_ppr.q.out | 40 +- .../clientpositive/spark/ppd_gby_join.q.out | 60 +- .../results/clientpositive/spark/ppd_join.q.out | 48 +- .../clientpositive/spark/ppd_outer_join1.q.out | 40 +- .../clientpositive/spark/ppd_outer_join2.q.out | 40 +- .../clientpositive/spark/ppd_outer_join3.q.out | 40 +- .../clientpositive/spark/ppd_outer_join4.q.out | 60 +- .../clientpositive/spark/router_join_ppr.q.out | 84 +- .../clientpositive/spark/smb_mapjoin_14.q.out | 54 +- .../results/clientpositive/vector_date_1.q.out | 2 +- 98 files changed, 1810 insertions(+), 5476 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out ---------------------------------------------------------------------- diff --git a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out index 93b10a8..dffd5a2 100644 --- a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out +++ b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out @@ -312,10 +312,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: accumulo_pushdown - filterExpr: ((key < '80') and (key > '90')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (value like '%90%') (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), value (type: string) http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/accumulo-handler/src/test/results/positive/accumulo_queries.q.out ---------------------------------------------------------------------- diff --git a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out index de82857..b3adf4e 100644 --- a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out +++ b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out @@ -289,7 +289,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: accumulo_table_1 - filterExpr: ((100 < key) and (key < 120)) (type: boolean) + filterExpr: (key < 120) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: int) http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/data/conf/hive-log4j2.properties ---------------------------------------------------------------------- diff --git a/data/conf/hive-log4j2.properties b/data/conf/hive-log4j2.properties index 1124d57..acaf2b4 100644 --- a/data/conf/hive-log4j2.properties +++ b/data/conf/hive-log4j2.properties @@ -50,7 +50,7 @@ appender.DRFA.strategy.type = DefaultRolloverStrategy appender.DRFA.strategy.max = 30 # list of all loggers -loggers = HadoopIPC, HadoopSecurity, Hdfs, HdfsServer, HadoopMetrics2, Mortbay, Yarn, YarnServer, Tez, HadoopConf, Zookeeper, ServerCnxn, NIOServerCnxn, ClientCnxn, ClientCnxnSocket, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, Operator, Serde2Lazy, ObjectStore +loggers = HadoopIPC, HadoopSecurity, Hdfs, HdfsServer, HadoopMetrics2, Mortbay, Yarn, YarnServer, Tez, HadoopConf, Zookeeper, ServerCnxn, NIOServerCnxn, ClientCnxn, ClientCnxnSocket, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, Operator, Serde2Lazy, ObjectStore, CalcitePlanner logger.HadoopIPC.name = org.apache.hadoop.ipc logger.HadoopIPC.level = WARN @@ -118,6 +118,9 @@ logger.Serde2Lazy.level = INFO logger.ObjectStore.name = org.apache.hadoop.hive.metastore.ObjectStore logger.ObjectStore.level = INFO +logger.CalcitePlanner.name = org.apache.calcite.plan.RelOptPlanner +logger.CalcitePlanner.level = INFO + # root logger rootLogger.level = ${sys:hive.log.level} rootLogger.appenderRefs = root, console http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out index 0d72bdc..d3c4a99 100644 --- a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out +++ b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out @@ -312,10 +312,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_pushdown - filterExpr: ((key < '80') and (key > '90')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (value like '%90%') (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), value (type: string) http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/hbase-handler/src/test/results/positive/hbase_queries.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out index 276b6b8..43a5a25 100644 --- a/hbase-handler/src/test/results/positive/hbase_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -291,7 +291,7 @@ STAGE PLANS: alias: hbase_table_1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((100 < key) and (key < 120)) (type: boolean) + predicate: (key < 120) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: int) http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out b/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out index 1897777..633e435 100644 --- a/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out +++ b/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out @@ -148,36 +148,22 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from hbase_ppd_keyrange where key >= 9 and key < 17 and key = 11 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: hbase_ppd_keyrange - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((key >= 9) and (key < 17)) and (key = 11)) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: 11 (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: hbase_ppd_keyrange + filterExpr: (key = 11) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: 11 (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + ListSink PREHOOK: query: select * from hbase_ppd_keyrange where key >=9 and key < 17 and key = 11 PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1b89632..32f5fd1 100644 --- a/pom.xml +++ b/pom.xml @@ -115,10 +115,10 @@ <antlr.version>3.5.2</antlr.version> <apache-directory-server.version>1.5.6</apache-directory-server.version> <apache-directory-clientapi.version>0.1</apache-directory-clientapi.version> - <avatica.version>1.9.0</avatica.version> + <avatica.version>1.10.0</avatica.version> <avro.version>1.7.7</avro.version> <bonecp.version>0.8.0.RELEASE</bonecp.version> - <calcite.version>1.12.0</calcite.version> + <calcite.version>1.13.0</calcite.version> <datanucleus-api-jdo.version>4.2.4</datanucleus-api-jdo.version> <datanucleus-core.version>4.1.17</datanucleus-core.version> <datanucleus-rdbms.version>4.1.19</datanucleus-rdbms.version> @@ -131,7 +131,7 @@ <commons-httpclient.version>3.0.1</commons-httpclient.version> <commons-io.version>2.4</commons-io.version> <commons-lang.version>2.6</commons-lang.version> - <commons-lang3.version>3.1</commons-lang3.version> + <commons-lang3.version>3.2</commons-lang3.version> <commons-pool.version>1.5.4</commons-pool.version> <commons-dbcp.version>1.4</commons-dbcp.version> <derby.version>10.10.2.0</derby.version> http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index e339d0a..5f34e8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -89,6 +89,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.calcite.rex.RexTableInputRef; /** * Generic utility functions needed for Calcite based Hive CBO. @@ -1080,6 +1081,11 @@ public class HiveCalciteUtil { public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) { return false; } + + @Override + public Boolean visitTableInputRef(RexTableInputRef fieldRef) { + return false; + } } public static Set<Integer> getInputRefs(RexNode expr) { http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java index 279d101..dad783d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java @@ -26,8 +26,14 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl { private static final int MAX_DECIMAL_PRECISION = 38; private static final int MAX_DECIMAL_SCALE = 38; private static final int DEFAULT_DECIMAL_PRECISION = 10; - private static final int MAX_VARCHAR_PRECISION = 65535; - private static final int MAX_CHAR_PRECISION = 255; + // STRING type in Hive is represented as VARCHAR with precision Integer.MAX_VALUE. + // In turn, the max VARCHAR precision should be 65535. However, the value is not + // used for validation, but rather only internally by the optimizer to know the max + // precision supported by the system. Thus, no VARCHAR precision should fall between + // 65535 and Integer.MAX_VALUE; the check for VARCHAR precision is done in Hive. + private static final int MAX_CHAR_PRECISION = Integer.MAX_VALUE; + private static final int DEFAULT_VARCHAR_PRECISION = 65535; + private static final int DEFAULT_CHAR_PRECISION = 255; private static final int MAX_BINARY_PRECISION = Integer.MAX_VALUE; private static final int MAX_TIMESTAMP_PRECISION = 9; @@ -60,13 +66,15 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl { switch (typeName) { // Hive will always require user to specify exact sizes for char, varchar; // Binary doesn't need any sizes; Decimal has the default of 10. - case CHAR: - case VARCHAR: case BINARY: case VARBINARY: case TIME: case TIMESTAMP: return getMaxPrecision(typeName); + case CHAR: + return DEFAULT_CHAR_PRECISION; + case VARCHAR: + return DEFAULT_VARCHAR_PRECISION; case DECIMAL: return DEFAULT_DECIMAL_PRECISION; case INTERVAL_YEAR: @@ -94,7 +102,6 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl { case DECIMAL: return getMaxNumericPrecision(); case VARCHAR: - return MAX_VARCHAR_PRECISION; case CHAR: return MAX_CHAR_PRECISION; case VARBINARY: http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java deleted file mode 100644 index dd39056..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite.rules; - -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; -import org.apache.calcite.rel.rules.PushProjector; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; - -/** - * Rule to push a {@link org.apache.calcite.rel.core.Project} - * past a {@link org.apache.calcite.rel.core.Filter} - * when {@code Filter} is on top of a {@link DruidQuery}. - * TODO: Replace this class with calcite DruidRules.DruidFilterProjectTransposeRule - * once we upgrade to calcite 1.13 - */ -public class HiveDruidProjectFilterTransposeRule - extends ProjectFilterTransposeRule { - - public static final HiveDruidProjectFilterTransposeRule INSTANCE = - new HiveDruidProjectFilterTransposeRule(); - - private HiveDruidProjectFilterTransposeRule() { - super( - operand(Project.class, - operand(Filter.class, operand(DruidQuery.class, none()))), - PushProjector.ExprCondition.FALSE, - HiveRelFactories.HIVE_BUILDER); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java deleted file mode 100644 index 81de33f..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; - -import java.util.Collections; -import java.util.List; - -import org.apache.calcite.plan.RelOptMaterialization; -import org.apache.calcite.plan.RelOptMaterializations; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.volcano.VolcanoPlanner; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; - -import com.google.common.collect.ImmutableList; - -/** - * Planner rule that replaces (if possible) - * a {@link org.apache.calcite.rel.core.Project} - * on a {@link org.apache.calcite.rel.core.Filter} - * on a {@link org.apache.calcite.rel.core.TableScan} - * to use a Materialized View. - */ -public class HiveMaterializedViewFilterScanRule extends RelOptRule { - - public static final HiveMaterializedViewFilterScanRule INSTANCE = - new HiveMaterializedViewFilterScanRule(HiveRelFactories.HIVE_BUILDER); - - - //~ Constructors ----------------------------------------------------------- - - /** Creates a HiveMaterializedViewFilterScanRule. */ - protected HiveMaterializedViewFilterScanRule(RelBuilderFactory relBuilderFactory) { - super(operand(Project.class, operand(Filter.class, operand(TableScan.class, null, none()))), - relBuilderFactory, "MaterializedViewFilterScanRule"); - } - - //~ Methods ---------------------------------------------------------------- - - public void onMatch(RelOptRuleCall call) { - final Project project = call.rel(0); - final Filter filter = call.rel(1); - final TableScan scan = call.rel(2); - apply(call, project, filter, scan); - } - - protected void apply(RelOptRuleCall call, Project project, Filter filter, TableScan scan) { - RelOptPlanner planner = call.getPlanner(); - List<RelOptMaterialization> materializations = - (planner instanceof VolcanoPlanner) - ? ((VolcanoPlanner) planner).getMaterializations() - : ImmutableList.<RelOptMaterialization>of(); - if (!materializations.isEmpty()) { - RelNode root = project.copy(project.getTraitSet(), Collections.singletonList( - filter.copy(filter.getTraitSet(), Collections.singletonList( - (RelNode) scan)))); - // Costing is done in transformTo(), so we call it repeatedly with all applicable - // materialized views and cheapest one will be picked - List<RelOptMaterialization> applicableMaterializations = - RelOptMaterializations.getApplicableMaterializations(root, materializations); - for (RelOptMaterialization materialization : applicableMaterializations) { - List<RelNode> subs = new MaterializedViewSubstitutionVisitor( - materialization.queryRel, root, relBuilderFactory).go(materialization.tableRel); - for (RelNode s : subs) { - call.transformTo(s); - } - } - } - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java new file mode 100644 index 0000000..2fa1138 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewRule.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewOnlyFilterRule; +import org.apache.calcite.rel.rules.AbstractMaterializedViewRule.MaterializedViewProjectFilterRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; + +/** + * Planner rule that replaces (if possible) a {@link org.apache.calcite.rel.core.Project} + * on a {@link org.apache.calcite.rel.core.Filter} and {@link org.apache.calcite.rel.core.Filter} + * or a {@link org.apache.calcite.rel.core.Filter} on a {@link org.apache.calcite.rel.core.TableScan} + * to use a Materialized View. + */ +public class HiveMaterializedViewRule { + + public static final MaterializedViewProjectFilterRule INSTANCE_PROJECT_FILTER = + new MaterializedViewProjectFilterRule(HiveRelFactories.HIVE_BUILDER, false); + + public static final MaterializedViewOnlyFilterRule INSTANCE_FILTER = + new MaterializedViewOnlyFilterRule(HiveRelFactories.HIVE_BUILDER, false); + +} http://git-wip-us.apache.org/repos/asf/hive/blob/4a567f86/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewSubstitutionVisitor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewSubstitutionVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewSubstitutionVisitor.java deleted file mode 100644 index e32f1a6..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewSubstitutionVisitor.java +++ /dev/null @@ -1,292 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; - -import java.util.List; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; -import org.apache.calcite.tools.RelBuilderFactory; - -import com.google.common.collect.ImmutableList; - -/** - * Extension to {@link SubstitutionVisitor}. - * - * TODO: Remove when we upgrade to Calcite version using builders. - */ -public class MaterializedViewSubstitutionVisitor extends SubstitutionVisitor { - private static final ImmutableList<UnifyRule> EXTENDED_RULES = - ImmutableList.<UnifyRule>builder() - .addAll(DEFAULT_RULES) - .add(ProjectToProjectUnifyRule1.INSTANCE) - .add(FilterToFilterUnifyRule1.INSTANCE) - .add(FilterToProjectUnifyRule1.INSTANCE) - .build(); - - public MaterializedViewSubstitutionVisitor(RelNode target_, RelNode query_) { - super(target_, query_, EXTENDED_RULES); - } - - public MaterializedViewSubstitutionVisitor(RelNode target_, RelNode query_, - RelBuilderFactory relBuilderFactory) { - super(target_, query_, EXTENDED_RULES, relBuilderFactory); - } - - public List<RelNode> go(RelNode replacement_) { - return super.go(replacement_); - } - - /** - * Implementation of {@link SubstitutionVisitor.UnifyRule} that matches a - * {@link SubstitutionVisitor.MutableProject} to a - * {@link SubstitutionVisitor.MutableProject} where the condition of the target - * relation is weaker. - * - * <p>Example: target has a weaker condition and contains all columns selected - * by query</p> - * <ul> - * <li>query: Project(projects: [$2, $0]) - * Filter(condition: >($1, 20)) - * Scan(table: [hr, emps])</li> - * <li>target: Project(projects: [$0, $1, $2]) - * Filter(condition: >($1, 10)) - * Scan(table: [hr, emps])</li> - * </ul> - */ - private static class ProjectToProjectUnifyRule1 extends AbstractUnifyRule { - public static final ProjectToProjectUnifyRule1 INSTANCE = - new ProjectToProjectUnifyRule1(); - - private ProjectToProjectUnifyRule1() { - super(operand(MutableProject.class, query(0)), - operand(MutableProject.class, target(0)), 1); - } - - @Override protected UnifyResult apply(UnifyRuleCall call) { - final MutableProject query = (MutableProject) call.query; - - final List<RelDataTypeField> oldFieldList = - query.getInput().getRowType().getFieldList(); - final List<RelDataTypeField> newFieldList = - call.target.getRowType().getFieldList(); - List<RexNode> newProjects; - try { - newProjects = transformRex(query.getProjects(), oldFieldList, newFieldList); - } catch (MatchFailed e) { - return null; - } - - final MutableProject newProject = - MutableProject.of( - query.getRowType(), call.target, newProjects); - - final MutableRel newProject2 = MutableRels.strip(newProject); - return call.result(newProject2); - } - - @Override protected UnifyRuleCall match(SubstitutionVisitor visitor, - MutableRel query, MutableRel target) { - assert query instanceof MutableProject && target instanceof MutableProject; - - if (queryOperand.matches(visitor, query)) { - if (targetOperand.matches(visitor, target)) { - return null; - } else if (targetOperand.isWeaker(visitor, target)) { - - final MutableProject queryProject = (MutableProject) query; - if (queryProject.getInput() instanceof MutableFilter) { - final MutableFilter innerFilter = - (MutableFilter) queryProject.getInput(); - RexNode newCondition; - try { - newCondition = transformRex(innerFilter.getCondition(), - innerFilter.getInput().getRowType().getFieldList(), - target.getRowType().getFieldList()); - } catch (MatchFailed e) { - return null; - } - final MutableFilter newFilter = MutableFilter.of(target, - newCondition); - - return visitor.new UnifyRuleCall(this, query, newFilter, - copy(visitor.slots, slotCount)); - } - } - } - return null; - } - } - - /** - * Implementation of {@link SubstitutionVisitor.UnifyRule} that matches a - * {@link SubstitutionVisitor.MutableFilter} to a - * {@link SubstitutionVisitor.MutableFilter} where the condition of the target - * relation is weaker. - * - * <p>Example: target has a weaker condition</p> - * <ul> - * <li>query: Filter(condition: >($1, 20)) - * Scan(table: [hr, emps])</li> - * <li>target: Filter(condition: >($1, 10)) - * Scan(table: [hr, emps])</li> - * </ul> - */ - private static class FilterToFilterUnifyRule1 extends AbstractUnifyRule { - public static final FilterToFilterUnifyRule1 INSTANCE = - new FilterToFilterUnifyRule1(); - - private FilterToFilterUnifyRule1() { - super(operand(MutableFilter.class, query(0)), - operand(MutableFilter.class, target(0)), 1); - } - - public UnifyResult apply(UnifyRuleCall call) { - final MutableFilter query = (MutableFilter) call.query; - final MutableFilter target = (MutableFilter) call.target; - final MutableFilter newFilter = MutableFilter.of(target, query.getCondition()); - return call.result(newFilter); - } - - @Override protected UnifyRuleCall match(SubstitutionVisitor visitor, - MutableRel query, MutableRel target) { - if (queryOperand.matches(visitor, query)) { - if (targetOperand.matches(visitor, target)) { - if (visitor.isWeaker(query, target)) { - return visitor.new UnifyRuleCall(this, query, target, - copy(visitor.slots, slotCount)); - } - } - } - return null; - } - } - - /** - * Implementation of {@link SubstitutionVisitor.UnifyRule} that matches a - * {@link SubstitutionVisitor.MutableFilter} to a - * {@link SubstitutionVisitor.MutableProject} on top of a - * {@link SubstitutionVisitor.MutableFilter} where the condition of the target - * relation is weaker. - * - * <p>Example: target has a weaker condition and is a permutation projection of - * its child relation</p> - * <ul> - * <li>query: Filter(condition: >($1, 20)) - * Scan(table: [hr, emps])</li> - * <li>target: Project(projects: [$1, $0, $2, $3, $4]) - * Filter(condition: >($1, 10)) - * Scan(table: [hr, emps])</li> - * </ul> - */ - private static class FilterToProjectUnifyRule1 extends AbstractUnifyRule { - public static final FilterToProjectUnifyRule1 INSTANCE = - new FilterToProjectUnifyRule1(); - - private FilterToProjectUnifyRule1() { - super( - operand(MutableFilter.class, query(0)), - operand(MutableProject.class, - operand(MutableFilter.class, target(0))), 1); - } - - public UnifyResult apply(UnifyRuleCall call) { - final MutableRel query = call.query; - - final List<RelDataTypeField> oldFieldList = - query.getRowType().getFieldList(); - final List<RelDataTypeField> newFieldList = - call.target.getRowType().getFieldList(); - List<RexNode> newProjects; - try { - newProjects = transformRex( - (List<RexNode>) call.getCluster().getRexBuilder().identityProjects( - query.getRowType()), - oldFieldList, newFieldList); - } catch (MatchFailed e) { - return null; - } - - final MutableProject newProject = - MutableProject.of( - query.getRowType(), call.target, newProjects); - - final MutableRel newProject2 = MutableRels.strip(newProject); - return call.result(newProject2); - } - - @Override protected UnifyRuleCall match(SubstitutionVisitor visitor, - MutableRel query, MutableRel target) { - assert query instanceof MutableFilter && target instanceof MutableProject; - - if (queryOperand.matches(visitor, query)) { - if (targetOperand.matches(visitor, target)) { - if (visitor.isWeaker(query, ((MutableProject) target).getInput())) { - final MutableFilter filter = (MutableFilter) query; - RexNode newCondition; - try { - newCondition = transformRex(filter.getCondition(), - filter.getInput().getRowType().getFieldList(), - target.getRowType().getFieldList()); - } catch (MatchFailed e) { - return null; - } - final MutableFilter newFilter = MutableFilter.of(target, - newCondition); - return visitor.new UnifyRuleCall(this, query, newFilter, - copy(visitor.slots, slotCount)); - } - } - } - return null; - } - } - - private static RexNode transformRex(RexNode node, - final List<RelDataTypeField> oldFields, - final List<RelDataTypeField> newFields) { - List<RexNode> nodes = - transformRex(ImmutableList.of(node), oldFields, newFields); - return nodes.get(0); - } - - private static List<RexNode> transformRex( - List<RexNode> nodes, - final List<RelDataTypeField> oldFields, - final List<RelDataTypeField> newFields) { - RexShuttle shuttle = new RexShuttle() { - @Override public RexNode visitInputRef(RexInputRef ref) { - RelDataTypeField f = oldFields.get(ref.getIndex()); - for (int index = 0; index < newFields.size(); index++) { - RelDataTypeField newf = newFields.get(index); - if (f.getKey().equals(newf.getKey()) - && f.getValue() == newf.getValue()) { - return new RexInputRef(index, f.getValue()); - } - } - throw MatchFailed.INSTANCE; - } - }; - return shuttle.apply(nodes); - } -} - -// End MaterializedViewSubstitutionVisitor.java
