HIVE-17385: incremental repl error for non-native tables
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45f3fac5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45f3fac5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45f3fac5 Branch: refs/heads/hive-14535 Commit: 45f3fac554ada3cd4b15a7bf3c9f0047c65fa616 Parents: 601c764 Author: Daniel Dai <da...@hortonworks.com> Authored: Thu Aug 31 21:08:02 2017 -0700 Committer: Daniel Dai <da...@hortonworks.com> Committed: Thu Aug 31 21:08:02 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/parse/EximUtil.java | 30 ++++++++++++++++++++ .../hive/ql/parse/ImportSemanticAnalyzer.java | 5 ++++ .../hive/ql/parse/repl/dump/TableExport.java | 7 +---- .../repl/dump/events/CreateTableHandler.java | 8 ++++-- .../ql/parse/repl/dump/io/TableSerializer.java | 7 ++--- 5 files changed, 44 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index 22094c0..40c34bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer; import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter; import org.apache.hadoop.hive.ql.parse.repl.dump.io.ReplicationSpecSerializer; @@ -385,4 +386,33 @@ public class EximUtil { }; } + /** + * Verify if a table should be exported or not + */ + public static Boolean shouldExportTable(ReplicationSpec replicationSpec, Table tableHandle) throws SemanticException { + if (replicationSpec == null) + { + replicationSpec = new ReplicationSpec(); + } + + if (replicationSpec.isNoop()) + { + return false; + } + + if (tableHandle == null) + { + return false; + } + + if (replicationSpec.isInReplicationScope()) { + return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative()); + } + + if (tableHandle.isNonNative()) { + throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg()); + } + + return true; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index f8d5c8d..7f3460f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -197,6 +197,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); } + if (rv.getTable() == null) { + // nothing to do here, silently return. + return false; + } + ReplicationSpec replicationSpec = rv.getReplicationSpec(); if (replicationSpec.isNoop()){ // nothing to do here, silently return. http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java index 7533a39..e490633 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java @@ -156,12 +156,7 @@ public class TableExport { } private boolean shouldExport() throws SemanticException { - if (replicationSpec.isInReplicationScope()) { - return !(tableSpec.tableHandle.isTemporary() || tableSpec.tableHandle.isNonNative()); - } else if (tableSpec.tableHandle.isNonNative()) { - throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg()); - } - return true; + return EximUtil.shouldExportTable(replicationSpec, tableSpec.tableHandle); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java index 8737d50..ef6f340 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java @@ -23,13 +23,12 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.EximUtil; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; -import org.apache.hadoop.hive.ql.parse.repl.DumpType; - class CreateTableHandler extends AbstractEventHandler { CreateTableHandler(NotificationEvent event) { @@ -48,6 +47,11 @@ class CreateTableHandler extends AbstractEventHandler { } Table qlMdTable = new Table(tobj); + + if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) { + return; + } + if (qlMdTable.isView()) { withinContext.replicationSpec.setIsMetadataOnly(true); } http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java index 92408a4..c3a70cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.thrift.TException; @@ -44,7 +45,7 @@ public class TableSerializer implements JsonWriter.Serializer { @Override public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider) throws SemanticException, IOException { - if (cannotReplicateTable(additionalPropertiesProvider)) { + if (!EximUtil.shouldExportTable(additionalPropertiesProvider, tableHandle)) { return; } @@ -61,10 +62,6 @@ public class TableSerializer implements JsonWriter.Serializer { } } - private boolean cannotReplicateTable(ReplicationSpec additionalPropertiesProvider) { - return tableHandle == null || additionalPropertiesProvider.isNoop(); - } - private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider) throws SemanticException, IOException { if (additionalPropertiesProvider.isInReplicationScope()) {
