HIVE-13622 WriteSet tracking optimizations (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0b532fc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0b532fc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0b532fc Branch: refs/heads/branch-1 Commit: c0b532fceec3ae06f06eb94c0075ac7e86187c35 Parents: 6c65832 Author: Eugene Koifman <[email protected]> Authored: Thu May 19 12:50:06 2016 -0700 Committer: Eugene Koifman <[email protected]> Committed: Thu May 19 12:50:06 2016 -0700 ---------------------------------------------------------------------- .../hive/hcatalog/streaming/HiveEndPoint.java | 4 +- .../hive/metastore/TestHiveMetaStoreTxns.java | 7 + metastore/if/hive_metastore.thrift | 12 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 1814 +++++++++--------- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 905 +++++---- .../gen/thrift/gen-cpp/hive_metastore_types.h | 52 +- .../metastore/api/AddDynamicPartitions.java | 127 +- .../hive/metastore/api/DataOperationType.java | 57 + .../hive/metastore/api/LockComponent.java | 234 ++- .../src/gen/thrift/gen-php/metastore/Types.php | 86 + .../gen/thrift/gen-py/hive_metastore/ttypes.py | 69 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 27 +- .../hive/metastore/HiveMetaStoreClient.java | 9 + .../hadoop/hive/metastore/IMetaStoreClient.java | 11 +- .../hive/metastore/LockComponentBuilder.java | 10 + .../hive/metastore/LockRequestBuilder.java | 9 + .../hadoop/hive/metastore/txn/TxnHandler.java | 108 +- .../metastore/txn/TestCompactionTxnHandler.java | 16 +- .../hive/metastore/txn/TestTxnHandler.java | 84 + .../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 18 +- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 34 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- .../hive/ql/lockmgr/TestDbTxnManager2.java | 181 +- .../hive/ql/txn/compactor/TestCleaner.java | 6 + .../hive/ql/txn/compactor/TestInitiator.java | 20 + .../gen-py/hive_service/ThriftHive-remote | 7 + 27 files changed, 2473 insertions(+), 1443 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java ---------------------------------------------------------------------- diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java index 41d2834..b9c1e23 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.LockComponentBuilder; import org.apache.hadoop.hive.metastore.LockRequestBuilder; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.LockRequest; @@ -964,7 +965,8 @@ public class HiveEndPoint { LockComponentBuilder lockCompBuilder = new LockComponentBuilder() .setDbName(hiveEndPoint.database) .setTableName(hiveEndPoint.table) - .setShared(); + .setShared() + .setOperationType(DataOperationType.INSERT); if (partNameForLock!=null && !partNameForLock.isEmpty() ) { lockCompBuilder.setPartitionName(partNameForLock); } http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index d5ecf98..f0646f6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -21,6 +21,7 @@ import junit.framework.Assert; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; @@ -151,14 +152,17 @@ public class TestHiveMetaStoreTxns { .setTableName("mytable") .setPartitionName("mypartition") .setExclusive() + .setOperationType(DataOperationType.NO_TXN) .build()); rqstBuilder.addLockComponent(new LockComponentBuilder() .setDbName("mydb") .setTableName("yourtable") .setSemiShared() + .setOperationType(DataOperationType.NO_TXN) .build()); rqstBuilder.addLockComponent(new LockComponentBuilder() .setDbName("yourdb") + .setOperationType(DataOperationType.NO_TXN) .setShared() .build()); rqstBuilder.setUser("fred"); @@ -187,15 +191,18 @@ public class TestHiveMetaStoreTxns { .setTableName("mytable") .setPartitionName("mypartition") .setSemiShared() + .setOperationType(DataOperationType.UPDATE) .build()) .addLockComponent(new LockComponentBuilder() .setDbName("mydb") .setTableName("yourtable") .setSemiShared() + .setOperationType(DataOperationType.UPDATE) .build()) .addLockComponent(new LockComponentBuilder() .setDbName("yourdb") .setShared() + .setOperationType(DataOperationType.SELECT) .build()) .setUser("fred"); http://git-wip-us.apache.org/repos/asf/hive/blob/c0b532fc/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 4b5d207..f3d1d42 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -106,6 +106,15 @@ enum GrantRevokeType { REVOKE = 2, } +enum DataOperationType { + SELECT = 1, + INSERT = 2 + UPDATE = 3, + DELETE = 4, + UNSET = 5,//this is the default to distinguish from NULL from old clients + NO_TXN = 6,//drop table, insert overwrite, etc - something non-transactional +} + // Types of events the client can request that the metastore fire. For now just support DML operations, as the metastore knows // about DDL operations and there's no reason for the client to request such an event. enum EventRequestType { @@ -587,6 +596,8 @@ struct LockComponent { 3: required string dbname, 4: optional string tablename, 5: optional string partitionname, + 6: optional DataOperationType operationType = DataOperationType.UNSET, + 7: optional bool isAcid = false } struct LockRequest { @@ -692,6 +703,7 @@ struct AddDynamicPartitions { 2: required string dbname, 3: required string tablename, 4: required list<string> partitionnames, + 5: optional DataOperationType operationType = DataOperationType.UNSET } struct NotificationEventRequest {
