This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e7fae8f HIVE-24064:Disable Materialized View Replication (Arko
Sharma, reviewed by Aasha Medhi)
e7fae8f is described below
commit e7fae8f12d7ee70f9909088cfd50496d64b1c7d9
Author: Anishek Agarwal <[email protected]>
AuthorDate: Tue Sep 1 15:48:07 2020 +0530
HIVE-24064:Disable Materialized View Replication (Arko Sharma, reviewed by
Aasha Medhi)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hive/ql/parse/TestReplicationScenarios.java | 110 ++++++++++++++++++++-
.../parse/TestReplicationScenariosAcidTables.java | 52 ++++++++++
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 35 ++++++-
4 files changed, 193 insertions(+), 6 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bdf3ae4..dec253f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -540,6 +540,8 @@ public class HiveConf extends Configuration {
REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false,
"Indicates whether replication dump only metadata information or data
+ metadata. \n"
+ "This config makes hive.repl.include.external.tables config
ineffective."),
+ REPL_INCLUDE_MATERIALIZED_VIEWS("hive.repl.include.materialized.views",
false,
+ "Indicates whether replication of materialized views is enabled."),
REPL_DUMP_SKIP_IMMUTABLE_DATA_COPY("hive.repl.dump.skip.immutable.data.copy",
false,
"Indicates whether replication dump can skip copyTask and refer to \n"
+ " original path instead. This would retain all table and
partition meta"),
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 15adac0..81e610e 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -136,7 +136,7 @@ public class TestReplicationScenarios {
protected static final Logger LOG =
LoggerFactory.getLogger(TestReplicationScenarios.class);
private ArrayList<String> lastResults;
- private final boolean VERIFY_SETUP_STEPS = false;
+ private boolean verifySetupSteps = false;
// if verifySetup is set to true, all the test setup we do will perform
additional
// verifications as well, which is useful to verify that our setup occurred
// correctly when developing and debugging tests. These verifications,
however
@@ -1349,7 +1349,7 @@ public class TestReplicationScenarios {
String testKey = "blah";
String testVal = "foo";
run("ALTER TABLE " + dbName + ".unptned2 SET TBLPROPERTIES ('" + testKey +
"' = '" + testVal + "')", driver);
- if (VERIFY_SETUP_STEPS){
+ if (verifySetupSteps){
try {
Table unptn2 = metaStoreClient.getTable(dbName,"unptned2");
assertTrue(unptn2.getParameters().containsKey(testKey));
@@ -1366,7 +1366,7 @@ public class TestReplicationScenarios {
// alter partitioned table set table property
run("ALTER TABLE " + dbName + ".ptned SET TBLPROPERTIES ('" + testKey + "'
= '" + testVal + "')", driver);
- if (VERIFY_SETUP_STEPS){
+ if (verifySetupSteps){
try {
Table ptned = metaStoreClient.getTable(dbName,"ptned");
assertTrue(ptned.getParameters().containsKey(testKey));
@@ -2610,6 +2610,108 @@ public class TestReplicationScenarios {
}
@Test
+ public void testMaterializedViewsReplication() throws Exception {
+ boolean verifySetupOriginal = verifySetupSteps;
+ verifySetupSteps = true;
+ String testName = "materializedviewsreplication";
+ String testName2 = testName + "2";
+ String dbName = createDB(testName, driver);
+ String dbName2 = createDB(testName2, driver); //for creating multi-db
materialized view
+ String replDbName = dbName + "_dupe";
+
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE",
driver);
+ run("CREATE TABLE " + dbName2 + ".unptned(a string) STORED AS TEXTFILE",
driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int)
STORED AS TEXTFILE", driver);
+
+ String[] unptn_data = new String[]{ "eleven", "twelve" };
+ String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
+ String[] ptn_data_2 = new String[]{ "fifteen", "sixteen", "seventeen"};
+ String[] empty = new String[]{};
+
+ String unptn_locn = new Path(TEST_PATH, testName +
"_unptn").toUri().getPath();
+ String ptn_locn_1 = new Path(TEST_PATH, testName +
"_ptn1").toUri().getPath();
+ String ptn_locn_2 = new Path(TEST_PATH, testName +
"_ptn2").toUri().getPath();
+
+ createTestDataFile(unptn_locn, unptn_data);
+ createTestDataFile(ptn_locn_1, ptn_data_1);
+ createTestDataFile(ptn_locn_2, ptn_data_2);
+
+ verifySetup("SELECT a from " + dbName + ".ptned", empty, driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", empty, driver);
+ verifySetup("SELECT * from " + dbName2 + ".unptned", empty, driver);
+
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " +
dbName + ".unptned", driver);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " +
dbName2 + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ verifySetup("SELECT * from " + dbName2 + ".unptned", unptn_data, driver);
+
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " +
dbName + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1,
driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " +
dbName + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2,
driver);
+
+
+ run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_boot disable rewrite
stored as textfile AS SELECT a FROM " + dbName + ".ptned where b=1", driver);
+ verifySetup("SELECT a from " + dbName + ".mat_view_boot", ptn_data_1,
driver);
+
+ run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_boot2 disable
rewrite stored as textfile AS SELECT t1.a FROM " + dbName + ".unptned as t1
join " + dbName2 + ".unptned as t2 on t1.a = t2.a", driver);
+ verifySetup("SELECT a from " + dbName + ".mat_view_boot2", unptn_data,
driver);
+
+ Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
+
+ verifyRun("SELECT * from " + replDbName + ".unptned", unptn_data,
driverMirror);
+ verifyRun("SELECT a from " + replDbName + ".ptned where b=1", ptn_data_1,
driverMirror);
+
+ //verify source MVs are not on replica
+ verifyIfTableNotExist(replDbName, "mat_view_boot", metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror);
+
+ //test alter materialized view with rename
+ run("ALTER TABLE " + dbName + ".mat_view_boot RENAME TO " + dbName +
".mat_view_rename", driver);
+
+ //verify rename, i.e. new MV exists and old MV does not exist
+ verifyIfTableNotExist(dbName, "mat_view_boot", metaStoreClient);
+ verifyIfTableExist(dbName, "mat_view_rename", metaStoreClient);
+
+ // Perform REPL-DUMP/LOAD
+ Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName);
+
+ //verify source MVs are not on replica
+ verifyIfTableNotExist(replDbName, "mat_view_rename",
metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror);
+
+ //test alter materialized view rebuild
+ run("ALTER MATERIALIZED VIEW " + dbName + ".mat_view_boot2 REBUILD" ,
driver);
+ verifyRun("SELECT a from " + dbName + ".mat_view_boot2", unptn_data,
driver);
+
+ // Perform REPL-DUMP/LOAD
+ incrementalDump = incrementalLoadAndVerify(dbName, replDbName);
+
+ //verify source MVs are not on replica
+ verifyIfTableNotExist(replDbName, "mat_view_rename",
metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror);
+
+ run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_inc disable rewrite
stored as textfile AS SELECT a FROM " + dbName + ".ptned where b=2", driver);
+ verifySetup("SELECT a from " + dbName + ".mat_view_inc", ptn_data_2,
driver);
+
+ run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_inc2 disable rewrite
stored as textfile AS SELECT t1.a FROM " + dbName + ".unptned as t1 join " +
dbName2 + ".unptned as t2 on t1.a = t2.a", driver);
+ verifySetup("SELECT a from " + dbName + ".mat_view_inc2", unptn_data,
driver);
+
+ // Perform REPL-DUMP/LOAD
+ incrementalDump = incrementalLoadAndVerify(dbName, replDbName);
+
+ //verify source MVs are not on replica
+ verifyIfTableNotExist(replDbName, "mat_view_rename",
metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_inc", metaStoreClientMirror);
+ verifyIfTableNotExist(replDbName, "mat_view_inc2", metaStoreClientMirror);
+
+ verifySetupSteps = verifySetupOriginal;
+ }
+
+
+
+ @Test
public void testExchangePartition() throws IOException {
String testName = "exchangePartition";
String dbName = createDB(testName, driver);
@@ -3921,7 +4023,7 @@ public class TestReplicationScenarios {
}
private void verifySetup(String cmd, String[] data, IDriver myDriver) throws
IOException {
- if (VERIFY_SETUP_STEPS){
+ if (verifySetupSteps){
run(cmd, myDriver);
verifyResults(data, myDriver);
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
index 2c176a2..4be9144 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
@@ -1971,4 +1971,56 @@ public class TestReplicationScenariosAcidTables extends
BaseReplicationScenarios
private void verifyPathExist(FileSystem fs, Path filePath) throws
IOException {
assertTrue("Path not found:" + filePath, fs.exists(filePath));
}
+
+ @Test
+ public void testMaterializedViewOnAcidTableReplication() throws Throwable {
+ WarehouseInstance.Tuple bootstrapDump = primary.run("use " + primaryDbName)
+ .run("CREATE TABLE t1(a string)" +
+ " stored as orc TBLPROPERTIES ('transactional'='true')")
+ .run("insert into t1 values (1)")
+ .run("insert into t1 values (2)")
+ .run("create materialized view mat_view as select * from t1")
+ .run("show tables")
+ .verifyResults(new String[]{"t1", "mat_view"})
+ .run("select * from mat_view")
+ .verifyResults(new String[]{"1", "2"})
+ .dump(primaryDbName);
+
+ //confirm materialized-view not replicated
+ replica.load(replicatedDbName, primaryDbName)
+ .run("use " + replicatedDbName)
+ .run("select * from t1")
+ .verifyResults(new String[]{"1", "2"})
+ .run("show tables")
+ .verifyResults(new String[]{"t1"});
+
+ //test alter materialized-view rebuild
+ primary.run("use " + primaryDbName)
+ .run("insert into t1 values (3)")
+ .run("alter materialized view mat_view rebuild")
+ .run("select * from mat_view")
+ .verifyResults(new String[]{"1", "2", "3"})
+ .dump(primaryDbName);
+
+ //confirm materialized-view not replicated
+ replica.load(replicatedDbName, primaryDbName)
+ .run("use " + replicatedDbName)
+ .run("select * from t1")
+ .verifyResults(new String[]{"1", "2", "3"})
+ .run("show tables")
+ .verifyResults(new String[]{"t1"});
+
+ //test alter materialized-view disable rewrite
+ primary.run("use " + primaryDbName)
+ .run("alter materialized view mat_view disable rewrite")
+ .run("select * from mat_view")
+ .verifyResults(new String[]{"1", "2", "3"})
+ .dump(primaryDbName);
+
+ //confirm materialized-view not replicated
+ replica.load(replicatedDbName, primaryDbName)
+ .run("use " + replicatedDbName)
+ .run("show tables")
+ .verifyResults(new String[]{"t1"});
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 17aeee6..656c298 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -420,6 +420,13 @@ public class ReplDumpTask extends Task<ReplDumpWork>
implements Serializable {
}
/**
+ * Decide whether to dump materialized views.
+ */
+ private boolean isMaterializedViewsReplEnabled() {
+ return conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_MATERIALIZED_VIEWS);
+ }
+
+ /**
* Decide whether to dump ACID tables.
* @param tableName - Name of ACID table to be replicated
* @return true if need to bootstrap dump ACID table and false if not.
@@ -542,6 +549,23 @@ public class ReplDumpTask extends Task<ReplDumpWork>
implements Serializable {
if (ev.getEventId() <= resumeFrom) {
continue;
}
+
+ //disable materialized-view replication if not configured
+ if(!isMaterializedViewsReplEnabled()){
+ String tblName = ev.getTableName();
+ if(tblName != null) {
+ try {
+ Table table = hiveDb.getTable(dbName, tblName);
+ if (table != null &&
TableType.MATERIALIZED_VIEW.equals(table.getTableType())){
+ LOG.info("Attempt to dump materialized view : " + tblName);
+ continue;
+ }
+ } catch (InvalidTableException te) {
+ LOG.debug(te.getMessage());
+ }
+ }
+ }
+
Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
dumpEvent(ev, evRoot, dumpRoot, cmRoot, hiveDb);
Utils.writeOutput(String.valueOf(lastReplId), ackFile, conf);
@@ -828,12 +852,19 @@ public class ReplDumpTask extends Task<ReplDumpWork>
implements Serializable {
Exception caught = null;
try (Writer writer = new Writer(dbRoot, conf)) {
for (String tblName : Utils.matchesTbl(hiveDb, dbName,
work.replScope)) {
- LOG.debug("Dumping table: " + tblName + " to db root " +
dbRoot.toUri());
Table table = null;
-
try {
HiveWrapper.Tuple<Table> tableTuple = new HiveWrapper(hiveDb,
dbName).table(tblName, conf);
table = tableTuple != null ? tableTuple.object : null;
+
+ //disable materialized-view replication if not configured
+ if(tableTuple != null && !isMaterializedViewsReplEnabled()
+ &&
TableType.MATERIALIZED_VIEW.equals(tableTuple.object.getTableType())){
+ LOG.info("Attempt to dump materialized view : " + tblName);
+ continue;
+ }
+
+ LOG.debug("Dumping table: " + tblName + " to db root " +
dbRoot.toUri());
if (shouldDumpExternalTableLocation()
&&
TableType.EXTERNAL_TABLE.equals(tableTuple.object.getTableType())) {
LOG.debug("Adding table {} to external tables list", tblName);
