Repository: hive
Updated Branches:
  refs/heads/branch-3 3fc0478b8 -> d4b1caa92


HIVE-20316: Skip external table file listing for create table event (Sankar 
Hariappan, reviewed by Mahesh Kumar Behera, Anishek Agarwal)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d4b1caa9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d4b1caa9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d4b1caa9

Branch: refs/heads/branch-3
Commit: d4b1caa92e157025ee1bf1a81112de90bf6b849e
Parents: 3fc0478
Author: Sankar Hariappan <sank...@apache.org>
Authored: Fri Aug 10 20:27:19 2018 +0530
Committer: Sankar Hariappan <sank...@apache.org>
Committed: Fri Aug 10 20:27:19 2018 +0530

----------------------------------------------------------------------
 .../hive/hcatalog/listener/DbNotificationListener.java  | 12 ++++++++----
 .../parse/TestReplicationScenariosAcrossInstances.java  |  5 ++++-
 .../messaging/json/JSONAddPartitionMessage.java         |  3 ++-
 .../messaging/json/JSONCreateTableMessage.java          |  2 +-
 4 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d4b1caa9/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
----------------------------------------------------------------------
diff --git 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 717cc8a..2ab59d7 100644
--- 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -84,6 +84,7 @@ import 
org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
 import org.apache.hadoop.hive.metastore.messaging.PartitionFiles;
 import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -155,9 +156,11 @@ public class DbNotificationListener extends 
TransactionalMetaStoreEventListener
   @Override
   public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
     Table t = tableEvent.getTable();
+    FileIterator fileIter = MetaStoreUtils.isExternalTable(t)
+                              ? null : new 
FileIterator(t.getSd().getLocation());
     NotificationEvent event =
-        new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(), 
msgFactory
-            .buildCreateTableMessage(t, new 
FileIterator(t.getSd().getLocation())).toString());
+        new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(),
+                msgFactory.buildCreateTableMessage(t, fileIter).toString());
     event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME);
     event.setDbName(t.getDbName());
     event.setTableName(t.getTableName());
@@ -301,9 +304,10 @@ public class DbNotificationListener extends 
TransactionalMetaStoreEventListener
   @Override
   public void onAddPartition(AddPartitionEvent partitionEvent) throws 
MetaException {
     Table t = partitionEvent.getTable();
+    PartitionFilesIterator fileIter = MetaStoreUtils.isExternalTable(t)
+            ? null : new 
PartitionFilesIterator(partitionEvent.getPartitionIterator(), t);
     String msg = msgFactory
-        .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(),
-            new PartitionFilesIterator(partitionEvent.getPartitionIterator(), 
t)).toString();
+        .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(), 
fileIter).toString();
     NotificationEvent event =
         new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), 
msg);
     event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME);

http://git-wip-us.apache.org/repos/asf/hive/blob/d4b1caa9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index b2e1e25..6c45641 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1387,6 +1387,7 @@ public class TestReplicationScenariosAcrossInstances {
     tuple = primary.run("use " + primaryDbName)
             .run("create external table t3 (id int)")
             .run("insert into table t3 values (10)")
+            .run("create external table t4 as select id from t3")
             .dump("repl dump " + primaryDbName + " from " + 
tuple.lastReplicationId
                     + " with ('hive.repl.include.external.tables'='true')");
 
@@ -1395,6 +1396,8 @@ public class TestReplicationScenariosAcrossInstances {
             .run("show tables like 't3'")
             .verifyResult("t3")
             .run("select id from t3")
-            .verifyResult("10");
+            .verifyResult("10")
+            .run("select id from t4")
+            .verifyResult(null); // Returns null as create table event doesn't 
list files
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/d4b1caa9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
index c232cce..805c691 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
@@ -88,7 +88,8 @@ public class JSONAddPartitionMessage extends 
AddPartitionMessage {
     } catch (TException e) {
       throw new IllegalArgumentException("Could not serialize: ", e);
     }
-    this.partitionFiles = Lists.newArrayList(partitionFileIter);
+    this.partitionFiles = (partitionFileIter != null) ? 
Lists.newArrayList(partitionFileIter)
+                                                      : Lists.newArrayList();
     checkValid();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/d4b1caa9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
index d894af9..6d3ee5b 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
@@ -72,7 +72,7 @@ public class JSONCreateTableMessage extends 
CreateTableMessage {
     } catch (TException e) {
       throw new IllegalArgumentException("Could not serialize: ", e);
     }
-    this.files = Lists.newArrayList(fileIter);
+    this.files = (fileIter != null) ? Lists.newArrayList(fileIter) : 
Lists.newArrayList();
   }
 
   @Override

Reply via email to