HIVE-18386 : Create dummy materialized views registry and make it configurable (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/80e6f7b0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/80e6f7b0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/80e6f7b0 Branch: refs/heads/master Commit: 80e6f7b0f13c134642763b0a692c3abbf3ffe106 Parents: a45becb Author: Jesus Camacho Rodriguez <jcama...@apache.org> Authored: Wed Jan 17 22:22:07 2018 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Wed Jan 17 22:22:07 2018 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/cli/CliDriver.java | 4 + .../org/apache/hadoop/hive/conf/HiveConf.java | 5 + data/conf/hive-site.xml | 2 +- .../test/resources/testconfiguration.properties | 2 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 4 + .../MaterializedViewRegistryPropertiesHook.java | 41 + .../MaterializedViewRegistryUpdateHook.java | 5 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 3 +- .../metadata/HiveMaterializedViewsRegistry.java | 88 +- .../ql/metadata/SessionHiveMetaStoreClient.java | 2 +- .../materialized_view_create_rewrite_dummy.q | 87 ++ ...rialized_view_create_rewrite_rebuild_dummy.q | 148 +++ ...materialized_view_create_rewrite_dummy.q.out | 517 +++++++++ ...ized_view_create_rewrite_rebuild_dummy.q.out | 1027 ++++++++++++++++++ .../apache/hive/service/server/HiveServer2.java | 2 +- 15 files changed, 1903 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java ---------------------------------------------------------------------- diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 2cd3c95..54d17df 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; @@ -760,6 +761,9 @@ public class CliDriver { ss.updateThreadName(); + // Create views registry + HiveMaterializedViewsRegistry.get().init(); + // execute cli driver work try { return executeDriver(ss, conf, oproc); http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index f2e927f..d0e2ad2 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2430,6 +2430,11 @@ public class HiveConf extends Configuration { "Setting it to 0s disables the timeout."), HIVE_SERVER2_PARALLEL_OPS_IN_SESSION("hive.server2.parallel.ops.in.session", true, "Whether to allow several parallel operations (such as SQL statements) in one session."), + HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL("hive.server2.materializedviews.registry.impl", "DEFAULT", + new StringSet("DEFAULT", "DUMMY"), + "The implementation that we should use for the materialized views registry. \n" + + " DEFAULT: Default cache for materialized views\n" + + " DUMMY: Do not cache materialized views and hence forward requests to metastore"), // HiveServer2 WebUI HIVE_SERVER2_WEBUI_BIND_HOST("hive.server2.webui.host", "0.0.0.0", "The host address the HiveServer2 WebUI will listen on"), http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/data/conf/hive-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 9f6aec1..01f83d1 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -141,7 +141,7 @@ <property> <name>hive.exec.pre.hooks</name> - <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value> + <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables, org.apache.hadoop.hive.ql.hooks.MaterializedViewRegistryPropertiesHook</value> <description>Pre Execute Hook for Tests</description> </property> http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 25636c8..53da72b 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -572,6 +572,8 @@ minillaplocal.query.files=\ llap_vector_nohybridgrace.q,\ load_dyn_part5.q,\ lvj_mapjoin.q,\ + materialized_view_create_rewrite_dummy.q,\ + materialized_view_create_rewrite_rebuild_dummy.q,\ mapjoin_decimal.q,\ mapjoin_hint.q,\ mapjoin_emit_interval.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 9b102f2..c05c604 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -112,6 +112,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -1123,6 +1124,9 @@ public class QTestUtil { createRemoteDirs(); } + // Create views registry + HiveMaterializedViewsRegistry.get().init(); + testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryPropertiesHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryPropertiesHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryPropertiesHook.java new file mode 100644 index 0000000..4a02a57 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryPropertiesHook.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; +import org.apache.hadoop.hive.ql.session.SessionState; + +/** + * Implementation of a pre execute hook that reloads the materialized view registry + * if needed by the test framework + */ +public class MaterializedViewRegistryPropertiesHook implements ExecuteWithHookContext { + + @Override + public void run(HookContext hookContext) throws Exception { + SessionState ss = SessionState.get(); + if (ss != null && ss.getConf().get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname) + .equals("DUMMY")) { + HiveMaterializedViewsRegistry.get().init(Hive.get(ss.getConf())); + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java index a917c07..f7a4c86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java @@ -79,8 +79,9 @@ public class MaterializedViewRegistryUpdateHook implements QueryLifeTimeHook { } if (isRewriteEnabled) { - Table mvTable = Hive.get().getTable(tableName); - HiveMaterializedViewsRegistry.get().createMaterializedView(mvTable); + Hive db = Hive.get(); + Table mvTable = db.getTable(tableName); + HiveMaterializedViewsRegistry.get().createMaterializedView(db.getConf(), mvTable); } else if (work.getAlterMaterializedViewDesc() != null) { // Disabling rewriting, removing from cache String[] names = tableName.split("\\."); http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 1a2b3c1..aa9bf9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1614,7 +1614,8 @@ public class Hive { LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " was not in the cache"); } - materialization = HiveMaterializedViewsRegistry.get().createMaterializedView(materializedViewTable); + materialization = HiveMaterializedViewsRegistry.get().createMaterializedView( + conf, materializedViewTable); if (materialization != null) { result.add(materialization); } http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 7e5c81b..fe1866b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.calcite.adapter.druid.DruidQuery; @@ -46,6 +47,7 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryState; @@ -62,7 +64,6 @@ import org.apache.hadoop.hive.ql.parse.ColumnStatsList; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -91,8 +92,11 @@ public final class HiveMaterializedViewsRegistry { private final ConcurrentMap<String, ConcurrentMap<String, RelOptMaterialization>> materializedViews = new ConcurrentHashMap<String, ConcurrentMap<String, RelOptMaterialization>>(); + /* If this boolean is true, we bypass the cache. */ + private boolean dummy; + /* Whether the cache has been initialized or not. */ - private boolean initialized; + private AtomicBoolean initialized = new AtomicBoolean(false); private HiveMaterializedViewsRegistry() { } @@ -115,10 +119,32 @@ public final class HiveMaterializedViewsRegistry { * runnable task is created, thus the views will still not be loaded in the cache when * it returns. */ - public void init(final Hive db) { - ExecutorService pool = Executors.newCachedThreadPool(); - pool.submit(new Loader(db)); - pool.shutdown(); + public void init() { + try { + // Create a new conf object to bypass metastore authorization, as we need to + // retrieve all materialized views from all databases + HiveConf conf = new HiveConf(); + conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname, + DefaultMetaStoreFilterHookImpl.class.getName()); + init(Hive.get(conf)); + } catch (HiveException e) { + LOG.error("Problem connecting to the metastore when initializing the view registry", e); + } + } + + public void init(Hive db) { + dummy = db.getConf().get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname) + .equals("DUMMY"); + if (dummy) { + // Dummy registry does not cache information and forwards all requests to metastore + initialized.set(true); + LOG.info("Using dummy materialized views registry"); + } else { + // We initialize the cache + ExecutorService pool = Executors.newCachedThreadPool(); + pool.submit(new Loader(db)); + pool.shutdown(); + } } private class Loader implements Runnable { @@ -133,28 +159,28 @@ public final class HiveMaterializedViewsRegistry { try { for (String dbName : db.getAllDatabases()) { for (Table mv : db.getAllMaterializedViewObjects(dbName)) { - addMaterializedView(mv, OpType.LOAD); + addMaterializedView(db.getConf(), mv, OpType.LOAD); } } - initialized = true; + initialized.set(true); + LOG.info("Materialized views registry has been initialized"); } catch (HiveException e) { - LOG.error("Problem connecting to the metastore when initializing the view registry"); + LOG.error("Problem connecting to the metastore when initializing the view registry", e); } } } public boolean isInitialized() { - return initialized; + return initialized.get(); } /** * Adds a newly created materialized view to the cache. * * @param materializedViewTable the materialized view - * @param tablesUsed tables used by the materialized view */ - public RelOptMaterialization createMaterializedView(Table materializedViewTable) { - return addMaterializedView(materializedViewTable, OpType.CREATE); + public RelOptMaterialization createMaterializedView(HiveConf conf, Table materializedViewTable) { + return addMaterializedView(conf, materializedViewTable, OpType.CREATE); } /** @@ -162,7 +188,7 @@ public final class HiveMaterializedViewsRegistry { * * @param materializedViewTable the materialized view */ - private RelOptMaterialization addMaterializedView(Table materializedViewTable, OpType opType) { + private RelOptMaterialization addMaterializedView(HiveConf conf, Table materializedViewTable, OpType opType) { // Bail out if it is not enabled for rewriting if (!materializedViewTable.isRewriteEnabled()) { return null; @@ -171,22 +197,25 @@ public final class HiveMaterializedViewsRegistry { // We are going to create the map for each view in the given database ConcurrentMap<String, RelOptMaterialization> cq = new ConcurrentHashMap<String, RelOptMaterialization>(); - final ConcurrentMap<String, RelOptMaterialization> prevCq = materializedViews.putIfAbsent( - materializedViewTable.getDbName(), cq); - if (prevCq != null) { - cq = prevCq; + if (!dummy) { + // If we are caching the MV, we include it in the cache + final ConcurrentMap<String, RelOptMaterialization> prevCq = materializedViews.putIfAbsent( + materializedViewTable.getDbName(), cq); + if (prevCq != null) { + cq = prevCq; + } } // Start the process to add MV to the cache // First we parse the view query and create the materialization object final String viewQuery = materializedViewTable.getViewExpandedText(); - final RelNode viewScan = createMaterializedViewScan(materializedViewTable); + final RelNode viewScan = createMaterializedViewScan(conf, materializedViewTable); if (viewScan == null) { LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + " ignored; error creating view replacement"); return null; } - final RelNode queryRel = parseQuery(viewQuery); + final RelNode queryRel = parseQuery(conf, viewQuery); if (queryRel == null) { LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + " ignored; error parsing original query"); @@ -204,7 +233,7 @@ public final class HiveMaterializedViewsRegistry { } if (LOG.isDebugEnabled()) { - LOG.debug("Cached materialized view for rewriting: " + viewScan.getTable().getQualifiedName()); + LOG.debug("Created materialized view for rewriting: " + viewScan.getTable().getQualifiedName()); } return materialization; } @@ -225,6 +254,10 @@ public final class HiveMaterializedViewsRegistry { * @param tableName the name for the materialized view to remove */ public void dropMaterializedView(String dbName, String tableName) { + if (dummy) { + // Nothing to do + return; + } ConcurrentMap<String, RelOptMaterialization> dbMap = materializedViews.get(dbName); if (dbMap != null) { dbMap.remove(tableName); @@ -244,9 +277,8 @@ public final class HiveMaterializedViewsRegistry { return null; } - private static RelNode createMaterializedViewScan(Table viewTable) { + private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable) { // 0. Recreate cluster - final HiveConf conf = SessionState.get().getConf(); final RelOptPlanner planner = CalcitePlanner.createPlanner(conf); final RexBuilder rexBuilder = new RexBuilder( new JavaTypeFactoryImpl( @@ -309,14 +341,14 @@ public final class HiveMaterializedViewsRegistry { } RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<VirtualColumn>(), - SessionState.get().getConf(), new HashMap<String, PrunedPartitionList>(), + conf, new HashMap<String, PrunedPartitionList>(), new HashMap<String, ColumnStatsList>(), new AtomicInteger()); RelNode rel; // 3. Build operator if (obtainTableType(viewTable) == TableType.DRUID) { // Build Druid query - String address = HiveConf.getVar(SessionState.get().getConf(), + String address = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS); String dataSource = viewTable.getParameters().get(Constants.DRUID_DATA_SOURCE); Set<String> metrics = new HashSet<>(); @@ -354,13 +386,13 @@ public final class HiveMaterializedViewsRegistry { return rel; } - private static RelNode parseQuery(String viewQuery) { + private static RelNode parseQuery(HiveConf conf, String viewQuery) { try { final ASTNode node = ParseUtils.parse(viewQuery); final QueryState qs = - new QueryState.Builder().withHiveConf(SessionState.get().getConf()).build(); + new QueryState.Builder().withHiveConf(conf).build(); CalcitePlanner analyzer = new CalcitePlanner(qs); - Context ctx = new Context(SessionState.get().getConf()); + Context ctx = new Context(conf); ctx.setIsLoadingMaterializedView(true); analyzer.initCtx(ctx); analyzer.init(false); http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 80c7804..7933b43 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -253,7 +253,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I throws MetaException, InvalidOperationException, UnknownDBException, TException { dbName = dbName.toLowerCase(); - if (SessionState.get().getTempTables().size() == 0) { + if (SessionState.get() == null || SessionState.get().getTempTables().size() == 0) { // No temp tables, just call underlying client return super.getTableObjectsByName(dbName, tableNames); } http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q new file mode 100644 index 0000000..ab6f46b --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q @@ -0,0 +1,87 @@ +-- SORT_QUERY_RESULTS + +set hive.server2.materializedviews.registry.impl=DUMMY; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1); + +analyze table cmv_basetable compute statistics for columns; + +create materialized view cmv_mat_view enable rewrite +as select a, b, c from cmv_basetable where a = 2; + +select * from cmv_mat_view; + +show tblproperties cmv_mat_view; + +create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from cmv_basetable where a = 3; + +select * from cmv_mat_view2; + +show tblproperties cmv_mat_view2; + +explain +select a, c from cmv_basetable where a = 3; + +select a, c from cmv_basetable where a = 3; + +alter materialized view cmv_mat_view2 disable rewrite; + +explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +explain +alter materialized view cmv_mat_view2 enable rewrite; + +alter materialized view cmv_mat_view2 enable rewrite; + +explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +drop materialized view cmv_mat_view2; + +explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a); + +drop materialized view cmv_mat_view; http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_rebuild_dummy.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_rebuild_dummy.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_rebuild_dummy.q new file mode 100644 index 0000000..2851718 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_rebuild_dummy.q @@ -0,0 +1,148 @@ +set hive.server2.materializedviews.registry.impl=DUMMY; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1); + +analyze table cmv_basetable compute statistics for columns; + +create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3); + +analyze table cmv_basetable_2 compute statistics for columns; + +EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- USE THE VIEW +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1); + +analyze table cmv_basetable_2 compute statistics for columns; + +-- CANNOT USE THE VIEW, IT IS OUTDATED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- REBUILD +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- NOW IT CAN BE USED AGAIN +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +DELETE FROM cmv_basetable_2 WHERE a = 3; + +-- CANNOT USE THE VIEW, IT IS OUTDATED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- REBUILD +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- NOW IT CAN BE USED AGAIN +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- IRRELEVANT OPERATIONS +create table cmv_irrelevant_table (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_irrelevant_table values + (1, 'alfred', 10.30, 2), + (3, 'charlie', 9.8, 1); + +analyze table cmv_irrelevant_table compute statistics for columns; + +-- IT CAN STILL BE USED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +drop materialized view cmv_mat_view; + +-- NOT USED ANYMORE +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; http://git-wip-us.apache.org/repos/asf/hive/blob/80e6f7b0/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out new file mode 100644 index 0000000..e691e02 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out @@ -0,0 +1,517 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +PREHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from cmv_basetable where a = 2 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from cmv_basetable where a = 2 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: select * from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +2 bob 3.14 +2 bonnie 172342.20 +PREHOOK: query: show tblproperties cmv_mat_view +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +numFiles 1 +numRows 2 +rawDataSize 408 +totalSize 453 +#### A masked pattern was here #### +PREHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from cmv_basetable where a = 3 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from cmv_basetable where a = 3 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: select * from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 +3 978.76 +PREHOOK: query: show tblproperties cmv_mat_view2 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view2 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +numFiles 1 +numRows 2 +rawDataSize 232 +totalSize 322 +#### A masked pattern was here #### +PREHOOK: query: explain +select a, c from cmv_basetable where a = 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a, c from cmv_basetable where a = 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view2 + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select a, c from cmv_basetable where a = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select a, c from cmv_basetable where a = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 +3 978.76 +PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: alter materialized view cmv_mat_view2 disable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (a = 3) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: explain +alter materialized view cmv_mat_view2 enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: query: explain +alter materialized view cmv_mat_view2 enable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Alter Materialized View Operator: + Alter Materialized View + name: default.cmv_mat_view2 + operation: UPDATE_REWRITE_FLAG + +PREHOOK: query: alter materialized view cmv_mat_view2 enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view2 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: drop materialized view cmv_mat_view2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: drop materialized view cmv_mat_view2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (a = 3) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view