Repository: hive Updated Branches: refs/heads/master 8cdee6297 -> b74b3e01e
HIVE-17558 : Skip non-native/temporary tables for constraint related scenarios (Tao Li, reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b74b3e01 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b74b3e01 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b74b3e01 Branch: refs/heads/master Commit: b74b3e01e2385ac541f440b822cbaa398b19fe94 Parents: 8cdee62 Author: Tao LI <[email protected]> Authored: Thu Sep 21 17:27:56 2017 -0700 Committer: Thejas M Nair <[email protected]> Committed: Thu Sep 21 17:28:39 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/parse/TestReplicationScenarios.java | 30 ++++++++++++++++---- .../apache/hadoop/hive/ql/parse/EximUtil.java | 15 ++++++++++ .../repl/dump/events/AddForeignKeyHandler.java | 8 ++++++ .../events/AddNotNullConstraintHandler.java | 8 ++++++ .../repl/dump/events/AddPrimaryKeyHandler.java | 8 ++++++ .../dump/events/AddUniqueConstraintHandler.java | 8 ++++++ 6 files changed, 72 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index c67192b..896ec1a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -2975,8 +2975,8 @@ public class TestReplicationScenarios { // Perform REPL-DUMP/LOAD advanceDumpDir(); run("REPL DUMP " + dbName + " FROM " + replDumpId, driver); - String incrementalDumpLocn = getResult(0,0,driver); - String incrementalDumpId = getResult(0,1,true,driver); + String incrementalDumpLocn = getResult(0, 0, driver); + String incrementalDumpId = getResult(0, 1, true, driver); LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror); verifyIfTableNotExist(dbName + "_dupe", "acid_table_rename", metaStoreClientMirror); @@ -2988,9 +2988,29 @@ public class TestReplicationScenarios { // Perform REPL-DUMP/LOAD advanceDumpDir(); - run("REPL DUMP " + dbName + " FROM " + replDumpId, driver); - incrementalDumpLocn = getResult(0,0,driver); - incrementalDumpId = getResult(0,1,true,driver); + run("REPL DUMP " + dbName + " FROM " + incrementalDumpId, driver); + incrementalDumpLocn = getResult(0, 0, driver); + incrementalDumpId = getResult(0, 1, true, driver); + LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); + run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); + printOutput(driverMirror); + run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror); + verifyIfTableNotExist(dbName + "_dupe", "acid_table_incremental", metaStoreClientMirror); + + // Test adding a constraint + run("ALTER TABLE " + dbName + ".acid_table_incremental ADD CONSTRAINT key_pk PRIMARY KEY (key) DISABLE NOVALIDATE", driver); + try { + List<SQLPrimaryKey> pks = metaStoreClient.getPrimaryKeys(new PrimaryKeysRequest(dbName, "acid_table_incremental")); + assertEquals(pks.size(), 1); + } catch (TException te) { + assertNull(te); + } + + // Perform REPL-DUMP/LOAD + advanceDumpDir(); + run("REPL DUMP " + dbName + " FROM " + incrementalDumpId, driver); + incrementalDumpLocn = getResult(0, 0, driver); + incrementalDumpId = getResult(0, 1, true, driver); LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); printOutput(driverMirror); http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index 95d394e..715a4d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -430,4 +430,19 @@ public class EximUtil { return true; } + + /** + * Verify if a table should be exported or not by talking to metastore to fetch table info. + * Return true when running into errors with metastore call. + */ + public static Boolean tryValidateShouldExportTable(Hive db, String dbName, String tableName, ReplicationSpec replicationSpec) { + try { + Table table = db.getTable(dbName, tableName); + return EximUtil.shouldExportTable(replicationSpec, table); + } catch (Exception e) { + // Swallow the exception + LOG.error("Failed to validate if the table should be exported or not", e); + } + return true; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java index ccd95fe..d0cbd4a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ public class AddForeignKeyHandler extends AbstractEventHandler { @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java index 7973eb3..aa7f4ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ public class AddNotNullConstraintHandler extends AbstractEventHandler { @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java index 3fb3e8b..344fac9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ public class AddPrimaryKeyHandler extends AbstractEventHandler { @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); http://git-wip-us.apache.org/repos/asf/hive/blob/b74b3e01/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java index e4f5415..4cc75a7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ public class AddUniqueConstraintHandler extends AbstractEventHandler { @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write();
