Repository: hive Updated Branches: refs/heads/branch-2 8aca411ad -> 7530ff5c4
HIVE-16759 : Add table type information to HMS log notifications (Janaki Lahorani, reviewed by Sergio Pena and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7530ff5c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7530ff5c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7530ff5c Branch: refs/heads/branch-2 Commit: 7530ff5c46f9e3e082c1896427737474fb2e9c17 Parents: 8aca411 Author: Janaki Lahorani <jan...@cloudera.com> Authored: Mon Jul 31 12:48:28 2017 -0700 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Mon Jul 31 12:48:33 2017 -0700 ---------------------------------------------------------------------- .../listener/DbNotificationListener.java | 7 ++--- .../hcatalog/messaging/AddPartitionMessage.java | 2 ++ .../messaging/AlterPartitionMessage.java | 2 ++ .../hcatalog/messaging/AlterTableMessage.java | 1 + .../hcatalog/messaging/CreateTableMessage.java | 2 ++ .../messaging/DropPartitionMessage.java | 1 + .../hcatalog/messaging/DropTableMessage.java | 1 + .../hive/hcatalog/messaging/InsertMessage.java | 1 + .../hive/hcatalog/messaging/MessageFactory.java | 12 +++++++++ .../messaging/json/JSONAddPartitionMessage.java | 15 +++++++++-- .../json/JSONAlterPartitionMessage.java | 19 +++++++++++-- .../messaging/json/JSONAlterTableMessage.java | 18 +++++++++++-- .../messaging/json/JSONCreateTableMessage.java | 16 +++++++++-- .../json/JSONDropPartitionMessage.java | 15 +++++++++-- .../messaging/json/JSONDropTableMessage.java | 11 ++++++-- .../messaging/json/JSONInsertMessage.java | 14 +++++++++- .../messaging/json/JSONMessageFactory.java | 28 +++++++++++++------- .../listener/TestNotificationListener.java | 7 +++++ .../api/TestHCatClientNotification.java | 5 ++++ .../listener/TestDbNotificationListener.java | 18 ++++++++++++- .../hive/metastore/events/InsertEvent.java | 4 +++ .../messaging/AddPartitionMessage.java | 2 ++ .../messaging/AlterPartitionMessage.java | 2 ++ .../metastore/messaging/AlterTableMessage.java | 2 ++ .../metastore/messaging/CreateTableMessage.java | 2 ++ .../messaging/DropPartitionMessage.java | 2 ++ .../metastore/messaging/DropTableMessage.java | 2 ++ .../hive/metastore/messaging/InsertMessage.java | 2 ++ .../metastore/messaging/MessageFactory.java | 3 +++ .../messaging/json/JSONAddPartitionMessage.java | 8 +++++- .../json/JSONAlterPartitionMessage.java | 8 +++++- .../messaging/json/JSONAlterTableMessage.java | 8 +++++- .../messaging/json/JSONCreateTableMessage.java | 18 ++++++++++--- .../json/JSONDropPartitionMessage.java | 15 +++++++++-- .../messaging/json/JSONDropTableMessage.java | 13 ++++++++- .../messaging/json/JSONInsertMessage.java | 14 +++++++++- .../messaging/json/JSONMessageFactory.java | 9 ++++++- 37 files changed, 272 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index f08b970..41347c2 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -439,9 +439,10 @@ public class DbNotificationListener extends MetaStoreEventListener { @Override public void onInsert(InsertEvent insertEvent) throws MetaException { NotificationEvent event = - new NotificationEvent(0, now(), EventType.INSERT.toString(), msgFactory.buildInsertMessage( - insertEvent.getDb(), insertEvent.getTable(), insertEvent.getPartitionKeyValues(), - new FileChksumIterator(insertEvent.getFiles(), insertEvent.getFileChecksums())) + new NotificationEvent(0, now(), EventType.INSERT.toString(), + msgFactory.buildInsertMessage(insertEvent.getDb(), insertEvent.getTable(), + insertEvent.getTableType(), insertEvent.getPartitionKeyValues(), + new FileChksumIterator(insertEvent.getFiles(), insertEvent.getFileChecksums())) .toString()); event.setDbName(insertEvent.getDb()); event.setTableName(insertEvent.getTable()); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java index 5b114b5..9dacbf6 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AddPartitionMessage.java @@ -37,6 +37,8 @@ public abstract class AddPartitionMessage extends HCatEventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + /** * Getter for list of partitions added. * @return List of maps, where each map identifies values for each partition-key, for every added partition. http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java index 10a300d..8d9575e 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterPartitionMessage.java @@ -31,6 +31,8 @@ public abstract class AlterPartitionMessage extends HCatEventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Map<String,String> getKeyValues(); @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java index 0b58f29..94f07cd 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterTableMessage.java @@ -28,6 +28,7 @@ public abstract class AlterTableMessage extends HCatEventMessage { } public abstract String getTable(); + public abstract String getTableType(); @Override public HCatEventMessage checkValid() { http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java index 6c8e2a4..6442340 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateTableMessage.java @@ -34,6 +34,8 @@ public abstract class CreateTableMessage extends HCatEventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + @Override public HCatEventMessage checkValid() { if (getTable() == null) http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java index ee222ec..d7b74f7 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropPartitionMessage.java @@ -32,6 +32,7 @@ public abstract class DropPartitionMessage extends HCatEventMessage { } public abstract String getTable(); + public abstract String getTableType(); public abstract List<Map<String, String>> getPartitions (); @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java index e47b572..345345e 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropTableMessage.java @@ -33,6 +33,7 @@ public abstract class DropTableMessage extends HCatEventMessage { * @return Table-name (String). */ public abstract String getTable(); + public abstract String getTableType(); @Override public HCatEventMessage checkValid() { http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java index be7ea10..d2aae7b 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/InsertMessage.java @@ -36,6 +36,7 @@ public abstract class InsertMessage extends HCatEventMessage { * @return Table-name (String). */ public abstract String getTable(); + public abstract String getTableType(); /** * Get the map of partition keyvalues. Will be null if this insert is to a table and not a http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index 44574fe..28026db 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -211,4 +211,16 @@ public abstract class MessageFactory { */ public abstract InsertMessage buildInsertMessage(String db, String table, Map<String,String> partVals, List<String> files); + + /** + * Factory method for building insert message + * @param db Name of the database the insert occurred in + * @param table Table the insert occurred in + * @param partVals Partition values for the partition that the insert occurred in, may be null + * if the insert was done into a non-partitioned table + * @param files List of files created as a result of the insert, may be null. + * @return instance of InsertMessage + */ + public abstract InsertMessage buildInsertMessage(String db, Table table, + Map<String,String> partVals, List<String> files); } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java index ac7dcd9..5d201f4 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONAddPartitionMessage extends AddPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -45,11 +45,17 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { public JSONAddPartitionMessage() {} public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String,String>> partitions, Long timestamp) { + List<Map<String,String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String,String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -68,6 +74,11 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java index 4f1d104..7ae7d75 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java @@ -34,7 +34,7 @@ import java.util.Map; public class JSONAlterPartitionMessage extends AlterPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -53,16 +53,26 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { String table, Map<String,String> keyValues, Long timestamp) { + this(server, servicePrincipal, db, table, null, keyValues, timestamp); + } + + public JSONAlterPartitionMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + Map<String,String> keyValues, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; this.keyValues = keyValues; checkValid(); } - @Override public String getServer() { return server; @@ -89,6 +99,11 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Map<String,String> getKeyValues() { return keyValues; } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java index b057d4a..f23aedf 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java @@ -32,7 +32,7 @@ import java.util.List; public class JSONAlterTableMessage extends AlterTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -47,15 +47,24 @@ public class JSONAlterTableMessage extends AlterTableMessage { String db, String table, Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONAlterTableMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } - @Override public String getServer() { return server; @@ -82,6 +91,11 @@ public class JSONAlterTableMessage extends AlterTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String toString() { try { return JSONMessageDeserializer.mapper.writeValueAsString(this); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java index 9c66730..327dc97 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONCreateTableMessage extends CreateTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -38,11 +38,18 @@ public class JSONCreateTableMessage extends CreateTableMessage { */ public JSONCreateTableMessage() {} - public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -63,6 +70,11 @@ public class JSONCreateTableMessage extends CreateTableMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String toString() { try { return JSONMessageDeserializer.mapper.writeValueAsString(this); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java index a4d6400..ecdc39d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONDropPartitionMessage extends DropPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -45,11 +45,17 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage() {} public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String,String>> partitions, Long timestamp) { + List<Map<String,String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String,String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -69,6 +75,11 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java index 3b62023..4300625 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONDropTableMessage extends DropTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -38,11 +38,13 @@ public class JSONDropTableMessage extends DropTableMessage { */ public JSONDropTableMessage() {} - public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -52,6 +54,11 @@ public class JSONDropTableMessage extends DropTableMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java index 8a4db15..667df4d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONInsertMessage.java @@ -31,7 +31,7 @@ import java.util.Map; public class JSONInsertMessage extends InsertMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -49,10 +49,17 @@ public class JSONInsertMessage extends InsertMessage { public JSONInsertMessage(String server, String servicePrincipal, String db, String table, Map<String,String> partKeyVals, List<String> files, Long timestamp) { + this(server, servicePrincipal, db, table, null, partKeyVals, files, timestamp); + } + + public JSONInsertMessage(String server, String servicePrincipal, String db, String table, + String tableType, Map<String,String> partKeyVals, List<String> files, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; this.partKeyVals = partKeyVals; this.files = files; @@ -64,6 +71,11 @@ public class JSONInsertMessage extends InsertMessage { public String getTable() { return table; } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 251084f..7962732 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -95,37 +95,40 @@ public class JSONMessageFactory extends MessageFactory { @Override public CreateTableMessage buildCreateTableMessage(Table table) { return new JSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), now()); + table.getTableName(), table.getTableType(), now()); } @Override public AlterTableMessage buildAlterTableMessage(Table before, Table after) { return new JSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), now()); + before.getTableName(), before.getTableType(), now()); } @Override public DropTableMessage buildDropTableMessage(Table table) { - return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), table.getTableName(), - now()); + return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), table.getTableType(), now()); } @Override public AddPartitionMessage buildAddPartitionMessage(Table table, Iterator<Partition> partitionsIterator) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionsIterator), now()); + table.getTableName(), table.getTableType(), + getPartitionKeyValues(table, partitionsIterator), now()); } @Override public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition before, Partition after) { return new JSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, - before.getDbName(), before.getTableName(), getPartitionKeyValues(table,before),now()); + before.getDbName(), before.getTableName(), table.getTableType(), + getPartitionKeyValues(table,before),now()); } @Override public DropPartitionMessage buildDropPartitionMessage(Table table, Iterator<Partition> partitions) { return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitions), now()); + table.getTableName(), table.getTableType(), + getPartitionKeyValues(table, partitions), now()); } @Override @@ -161,8 +164,15 @@ public class JSONMessageFactory extends MessageFactory { @Override public InsertMessage buildInsertMessage(String db, String table, Map<String,String> partKeyVals, List<String> files) { - return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, partKeyVals, - files, now()); + return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, null, + partKeyVals, files, now()); + } + + @Override + public InsertMessage buildInsertMessage(String db, Table table, Map<String,String> partKeyVals, + List<String> files) { + return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), table.getTableType(), partKeyVals, files, now()); } private long now() { http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java b/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java index ef7b575..3a33403 100644 --- a/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java +++ b/hcatalog/server-extensions/src/test/java/org/apache/hive/hcatalog/listener/TestNotificationListener.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; @@ -168,6 +169,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis CreateTableMessage message = deserializer.getCreateTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof CreateTableMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -181,6 +183,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getPartitions().size()); Assert.assertEquals("2011", message.getPartitions().get(0).get("b")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AddPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -195,6 +198,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getKeyValues().size()); Assert.assertTrue(message.getKeyValues().values().contains("2011")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -210,6 +214,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis Assert.assertEquals("mydb", message.getDB()); Assert.assertEquals(1, message.getPartitions().size()); Assert.assertEquals("2011", message.getPartitions().get(0).get("b")); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof DropPartitionMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -223,6 +228,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis DropTableMessage message = deserializer.getDropTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof DropTableMessage); Assert.assertEquals("mydb", message2.getDB()); @@ -241,6 +247,7 @@ public class TestNotificationListener extends HCatBaseTest implements MessageLis AlterTableMessage message = deserializer.getAlterTableMessage(messageBody); Assert.assertEquals("mytbl", message.getTable()); Assert.assertEquals("mydb", message.getDB()); + Assert.assertEquals(TableType.MANAGED_TABLE.toString(), message.getTableType()); HCatEventMessage message2 = MessagingUtils.getMessage(msg); Assert.assertTrue("Unexpected message-type.", message2 instanceof AlterTableMessage); Assert.assertEquals("mydb", message2.getDB()); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java index c09e687..b9a3218 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; @@ -137,6 +138,7 @@ public class TestHCatClientNotification { CreateTableMessage createTableMessage = md.getCreateTableMessage(event.getMessage()); assertEquals(dbName, createTableMessage.getDB()); assertEquals(tableName, createTableMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), createTableMessage.getTableType()); // fetch the table marked by the message and compare HCatTable createdTable = hCatClient.getTable(dbName,tableName); @@ -167,6 +169,7 @@ public class TestHCatClientNotification { DropTableMessage dropTableMessage = md.getDropTableMessage(event.getMessage()); assertEquals(dbName, dropTableMessage.getDB()); assertEquals(tableName, dropTableMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropTableMessage.getTableType()); } @Test @@ -198,6 +201,7 @@ public class TestHCatClientNotification { AddPartitionMessage addPartitionMessage = md.getAddPartitionMessage(event.getMessage()); assertEquals(dbName, addPartitionMessage.getDB()); assertEquals(tableName, addPartitionMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), addPartitionMessage.getTableType()); List<Map<String,String>> ptndescs = addPartitionMessage.getPartitions(); // fetch the partition referred to by the message and compare @@ -245,6 +249,7 @@ public class TestHCatClientNotification { DropPartitionMessage dropPartitionMessage = md.getDropPartitionMessage(event.getMessage()); assertEquals(dbName, dropPartitionMessage.getDB()); assertEquals(tableName, dropPartitionMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPartitionMessage.getTableType()); List<Map<String, String>> droppedPartSpecs = dropPartitionMessage.getPartitions(); assertNotNull(droppedPartSpecs); assertEquals(1,droppedPartSpecs.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 976c3c5..8de66b3 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FireEventRequest; @@ -365,7 +366,7 @@ public class TestDbNotificationListener { emptyParameters); Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, - emptyParameters, null, null, null); + emptyParameters, null, null, TableType.MANAGED_TABLE.toString()); msClient.createTable(table); // Get notifications from metastore @@ -383,6 +384,7 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, createTblMsg.getDB()); assertEquals(tblName, createTblMsg.getTable()); assertEquals(table, createTblMsg.getTableObj()); + assertEquals(TableType.MANAGED_TABLE.toString(), createTblMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); @@ -442,6 +444,7 @@ public class TestDbNotificationListener { AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage()); assertEquals(table, alterTableMessage.getTableObjAfter()); + assertEquals(TableType.MANAGED_TABLE.toString(), alterTableMessage.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); @@ -496,6 +499,7 @@ public class TestDbNotificationListener { DropTableMessage dropTblMsg = md.getDropTableMessage(event.getMessage()); assertEquals(defaultDbName, dropTblMsg.getDB()); assertEquals(tblName, dropTblMsg.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropTblMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_TABLE, firstEventId + 2); @@ -565,6 +569,7 @@ public class TestDbNotificationListener { Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator(); assertTrue(ptnIter.hasNext()); assertEquals(partition, ptnIter.next()); + assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); @@ -635,6 +640,7 @@ public class TestDbNotificationListener { assertEquals(defaultDbName, alterPtnMsg.getDB()); assertEquals(tblName, alterPtnMsg.getTable()); assertEquals(newPart, alterPtnMsg.getPtnObjAfter()); + assertEquals(TableType.MANAGED_TABLE.toString(), alterPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); @@ -702,6 +708,7 @@ public class TestDbNotificationListener { assertEquals(table.getDbName(), tableObj.getDbName()); assertEquals(table.getTableName(), tableObj.getTableName()); assertEquals(table.getOwner(), tableObj.getOwner()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType()); // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3); @@ -784,6 +791,8 @@ public class TestDbNotificationListener { assertEquals(dbName, addPtnMsg.getDB()); assertEquals(tab2.getTableName(), addPtnMsg.getTable()); Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator(); + assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType()); + assertTrue(ptnIter.hasNext()); Partition msgPart = ptnIter.next(); assertEquals(part1.getValues(), msgPart.getValues()); @@ -801,6 +810,7 @@ public class TestDbNotificationListener { DropPartitionMessage dropPtnMsg = md.getDropPartitionMessage(event.getMessage()); assertEquals(dbName, dropPtnMsg.getDB()); assertEquals(tab1.getTableName(), dropPtnMsg.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType()); Iterator<Map<String, String>> parts = dropPtnMsg.getPartitions().iterator(); assertTrue(parts.hasNext()); assertEquals(part1.getValues(), Lists.newArrayList(parts.next().values())); @@ -1186,6 +1196,12 @@ public class TestDbNotificationListener { // Parse the message field verifyInsert(event, defaultDbName, tblName); + // Parse the message field + InsertMessage insertMessage = md.getInsertMessage(event.getMessage()); + assertEquals(defaultDbName, insertMessage.getDB()); + assertEquals(tblName, insertMessage.getTable()); + assertEquals(TableType.MANAGED_TABLE.toString(), insertMessage.getTableType()); + // Verify the eventID was passed to the non-transactional listener MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2); MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java index 7bc0e04..0690c1b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java @@ -37,6 +37,7 @@ public class InsertEvent extends ListenerEvent { // we have just the string names, but that's fine for what we need. private final String db; private final String table; + private final String tableType; private final Map<String, String> keyValues; private final List<String> files; private List<String> fileChecksums = new ArrayList<String>(); @@ -60,6 +61,7 @@ public class InsertEvent extends ListenerEvent { GetTableRequest req = new GetTableRequest(db, table); req.setCapabilities(HiveMetaStoreClient.TEST_VERSION); Table t = handler.get_table_req(req).getTable(); + tableType = t.getTableType(); keyValues = new LinkedHashMap<String, String>(); if (partVals != null) { for (int i = 0; i < partVals.size(); i++) { @@ -82,6 +84,8 @@ public class InsertEvent extends ListenerEvent { return table; } + public String getTableType() {return tableType; } + /** * @return List of values for the partition keys. */ http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java index 28f1101..774c020 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java @@ -37,6 +37,8 @@ public abstract class AddPartitionMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; /** http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java index ed6080b..8443244 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java @@ -31,6 +31,8 @@ public abstract class AlterPartitionMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Map<String,String> getKeyValues(); public abstract Table getTableObj() throws Exception; http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java index 5487123..60e4c9b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java @@ -28,6 +28,8 @@ public abstract class AlterTableMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObjBefore() throws Exception; public abstract Table getTableObjAfter() throws Exception; http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java index 441fd84..b75caa6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java @@ -33,6 +33,8 @@ public abstract class CreateTableMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; /** http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java index 0dd3e50..d254ad9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java @@ -32,6 +32,8 @@ public abstract class DropPartitionMessage extends EventMessage { public abstract String getTable(); + public abstract String getTableType(); + public abstract Table getTableObj() throws Exception; public abstract List<Map<String, String>> getPartitions (); http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java index 64a8cc5..03f73f1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java @@ -31,6 +31,8 @@ public abstract class DropTableMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + @Override public EventMessage checkValid() { if (getTable() == null) http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java index 3d16721..3adee52 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java @@ -36,6 +36,8 @@ public abstract class InsertMessage extends EventMessage { */ public abstract String getTable(); + public abstract String getTableType(); + /** * Get the map of partition keyvalues. Will be null if this insert is to a table and not a * partition. http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java index aa770f2..ff57326 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java @@ -236,4 +236,7 @@ public abstract class MessageFactory { */ public abstract InsertMessage buildInsertMessage(String db, String table, Map<String, String> partVals, Iterator<String> files); + + public abstract InsertMessage buildInsertMessage(String db, String table, String tableType, + Map<String, String> partVals, Iterator<String> files); } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java index a488205..db3431e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java @@ -43,7 +43,7 @@ import java.util.Map; public class JSONAddPartitionMessage extends AddPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @@ -73,6 +73,7 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { this.servicePrincipal = servicePrincipal; this.db = tableObj.getDbName(); this.table = tableObj.getTableName(); + this.tableType = tableObj.getTableType(); this.timestamp = timestamp; partitions = new ArrayList<Map<String, String>>(); partitionListJson = new ArrayList<String>(); @@ -112,6 +113,11 @@ public class JSONAddPartitionMessage extends AddPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Table getTableObj() throws Exception { return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java index dd1bf3c..f545bf2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -34,7 +34,7 @@ import java.util.Map; public class JSONAlterPartitionMessage extends AlterPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @@ -57,6 +57,7 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { this.servicePrincipal = servicePrincipal; this.db = tableObj.getDbName(); this.table = tableObj.getTableName(); + this.tableType = tableObj.getTableType(); this.timestamp = timestamp; this.keyValues = JSONMessageFactory.getPartitionKeyValues(tableObj, partitionObjBefore); try { @@ -95,6 +96,11 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Map<String, String> getKeyValues() { return keyValues; } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java index 792015e..8f4b018 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java @@ -29,7 +29,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONAlterTableMessage extends AlterTableMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjBeforeJson, tableObjAfterJson; + String server, servicePrincipal, db, table, tableType, tableObjBeforeJson, tableObjAfterJson; @JsonProperty Long timestamp; @@ -46,6 +46,7 @@ public class JSONAlterTableMessage extends AlterTableMessage { this.servicePrincipal = servicePrincipal; this.db = tableObjBefore.getDbName(); this.table = tableObjBefore.getTableName(); + this.tableType = tableObjBefore.getTableType(); this.timestamp = timestamp; try { this.tableObjBeforeJson = JSONMessageFactory.createTableObjJson(tableObjBefore); @@ -82,6 +83,11 @@ public class JSONAlterTableMessage extends AlterTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Table getTableObjBefore() throws Exception { return (Table) JSONMessageFactory.getTObj(tableObjBeforeJson,Table.class); } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java index dbc3dd4..9dd7a74 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java @@ -35,7 +35,7 @@ import com.google.common.collect.Lists; public class JSONCreateTableMessage extends CreateTableMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @JsonProperty @@ -48,18 +48,25 @@ public class JSONCreateTableMessage extends CreateTableMessage { } public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp) { + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + public JSONCreateTableMessage(String server, String servicePrincipal, Table tableObj, Iterator<String> fileIter, Long timestamp) { - this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), timestamp); + this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), + tableObj.getTableType(), timestamp); try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); } catch (TException e) { @@ -94,6 +101,11 @@ public class JSONCreateTableMessage extends CreateTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Table getTableObj() throws Exception { return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java index f1860af..576806c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java @@ -33,7 +33,7 @@ import java.util.Map; public class JSONDropPartitionMessage extends DropPartitionMessage { @JsonProperty - String server, servicePrincipal, db, table, tableObjJson; + String server, servicePrincipal, db, table, tableType, tableObjJson; @JsonProperty Long timestamp; @@ -49,10 +49,16 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, List<Map<String, String>> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, null, partitions, timestamp); + } + + public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List<Map<String, String>> partitions, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; checkValid(); @@ -61,7 +67,7 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { public JSONDropPartitionMessage(String server, String servicePrincipal, Table tableObj, List<Map<String, String>> partitionKeyValues, long timestamp) { this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), - partitionKeyValues, timestamp); + tableObj.getTableType(), partitionKeyValues, timestamp); try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); } catch (TException e) { @@ -90,6 +96,11 @@ public class JSONDropPartitionMessage extends DropPartitionMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public Long getTimestamp() { return timestamp; } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java index 635ab61..17f38ba 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java @@ -28,7 +28,7 @@ import org.codehaus.jackson.annotate.JsonProperty; public class JSONDropTableMessage extends DropTableMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -41,10 +41,16 @@ public class JSONDropTableMessage extends DropTableMessage { public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + this(server, servicePrincipal, db, table, null, timestamp); + } + + public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; checkValid(); } @@ -55,6 +61,11 @@ public class JSONDropTableMessage extends DropTableMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java index e1316a4..329f320 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java @@ -34,7 +34,7 @@ import java.util.Map; public class JSONInsertMessage extends InsertMessage { @JsonProperty - String server, servicePrincipal, db, table; + String server, servicePrincipal, db, table, tableType; @JsonProperty Long timestamp; @@ -53,10 +53,17 @@ public class JSONInsertMessage extends InsertMessage { public JSONInsertMessage(String server, String servicePrincipal, String db, String table, Map<String, String> partKeyVals, Iterator<String> fileIter, Long timestamp) { + this(server, servicePrincipal, db, table, null, partKeyVals, fileIter, timestamp); + } + + public JSONInsertMessage(String server, String servicePrincipal, String db, String table, + String tableType, Map<String, String> partKeyVals, Iterator<String> fileIter, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; + this.tableType = tableType; this.timestamp = timestamp; this.partKeyVals = partKeyVals; this.files = Lists.newArrayList(fileIter); @@ -69,6 +76,11 @@ public class JSONInsertMessage extends InsertMessage { } @Override + public String getTableType() { + if (tableType != null) return tableType; else return ""; + } + + @Override public String getServer() { return server; } http://git-wip-us.apache.org/repos/asf/hive/blob/7530ff5c/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java index 3406afb..7308adb 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java @@ -111,7 +111,7 @@ public class JSONMessageFactory extends MessageFactory { @Override public DropTableMessage buildDropTableMessage(Table table) { return new JSONDropTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), now()); + table.getTableName(), table.getTableType(), now()); } @Override @@ -167,6 +167,13 @@ public class JSONMessageFactory extends MessageFactory { fileIter, now()); } + @Override + public InsertMessage buildInsertMessage(String db, String table, String tableType, Map<String, + String> partKeyVals, Iterator<String> fileIter) { + return new JSONInsertMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, db, table, tableType, + partKeyVals, fileIter, now()); + } + private long now() { return System.currentTimeMillis() / 1000; }