Author: hashutosh
Date: Wed Mar 5 00:20:53 2014
New Revision: 1574266
URL: http://svn.apache.org/r1574266
Log:
HIVE-5843 : Transaction manager for Hive (Alan Gates via Ashutosh Chauhan)
Added:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
hive/trunk/metastore/scripts/upgrade/derby/hive-txn-schema-0.13.0.derby.sql
hive/trunk/metastore/scripts/upgrade/mysql/hive-txn-schema-0.13.0.mysql.sql
hive/trunk/metastore/scripts/upgrade/oracle/hive-txn-schema-0.13.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/postgres/hive-txn-schema-0.13.0.postgres.sql
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AbortTxnRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CheckLockRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CompactionType.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetOpenTxnsInfoResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetOpenTxnsResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/HeartbeatRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockComponent.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockLevel.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockState.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LockType.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NoSuchLockException.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NoSuchTxnException.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnsResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowLocksRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowLocksResponse.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowLocksResponseElement.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnAbortedException.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnInfo.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnOpenException.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnState.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UnlockRequest.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/txn/
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/TxnManagerFactory.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
hive/trunk/ql/src/test/queries/clientnegative/dbtxnmgr_nodblock.q
hive/trunk/ql/src/test/queries/clientnegative/dbtxnmgr_nodbunlock.q
hive/trunk/ql/src/test/queries/clientnegative/dbtxnmgr_notablelock.q
hive/trunk/ql/src/test/queries/clientnegative/dbtxnmgr_notableunlock.q
hive/trunk/ql/src/test/queries/clientpositive/dbtxnmgr_query1.q
hive/trunk/ql/src/test/queries/clientpositive/dbtxnmgr_query2.q
hive/trunk/ql/src/test/queries/clientpositive/dbtxnmgr_query3.q
hive/trunk/ql/src/test/queries/clientpositive/dbtxnmgr_query4.q
hive/trunk/ql/src/test/queries/clientpositive/dbtxnmgr_query5.q
hive/trunk/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
hive/trunk/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
hive/trunk/ql/src/test/results/clientnegative/dbtxnmgr_notablelock.q.out
hive/trunk/ql/src/test/results/clientnegative/dbtxnmgr_notableunlock.q.out
hive/trunk/ql/src/test/results/clientpositive/dbtxnmgr_query1.q.out
hive/trunk/ql/src/test/results/clientpositive/dbtxnmgr_query2.q.out
hive/trunk/ql/src/test/results/clientpositive/dbtxnmgr_query3.q.out
hive/trunk/ql/src/test/results/clientpositive/dbtxnmgr_query4.q.out
hive/trunk/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out
Removed:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/TestDriver.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/metastore/if/hive_metastore.thrift
hive/trunk/metastore/pom.xml
hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
hive/trunk/metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddPartitionsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddPartitionsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ColumnStatistics.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DropPartitionsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/HiveObjectRef.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsByExprResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsStatsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionsStatsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PrincipalPrivilegeSet.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PrivilegeBag.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/RequestPartsSpec.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Schema.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SkewedInfo.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TableStatsRequest.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TableStatsResult.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Type.java
hive/trunk/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/EmbeddedLockManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockMode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed
Mar 5 00:20:53 2014
@@ -18,26 +18,6 @@
package org.apache.hadoop.hive.conf;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.security.auth.login.LoginException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,6 +27,14 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
+import javax.security.auth.login.LoginException;
+import java.io.*;
+import java.net.URL;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* Hive Configuration.
*/
@@ -134,7 +122,12 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
HiveConf.ConfVars.USERS_IN_ADMIN_ROLE,
- HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER
+ HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ HiveConf.ConfVars.HIVE_TXN_MANAGER,
+ HiveConf.ConfVars.HIVE_TXN_JDBC_DRIVER,
+ HiveConf.ConfVars.HIVE_TXN_JDBC_CONNECT_STRING,
+ HiveConf.ConfVars.HIVE_TXN_TIMEOUT,
+ HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH,
};
/**
@@ -703,6 +696,48 @@ public class HiveConf extends Configurat
HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace",
"hive_zookeeper_namespace"),
HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes",
false),
+ // Transactions
+ HIVE_TXN_MANAGER("hive.txn.manager",
+ "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager"),
+ HIVE_TXN_JDBC_DRIVER("hive.txn.driver", ""),
+ HIVE_TXN_JDBC_CONNECT_STRING("hive.txn.connection.string", ""),
+ // time after which transactions are declared aborted if the client has
+ // not sent a heartbeat, in seconds.
+ HIVE_TXN_TIMEOUT("hive.txn.timeout", 300),
+
+ // Maximum number of transactions that can be fetched in one call to
+ // open_txns().
+ // Increasing this will decrease the number of delta files created when
+ // streaming data into Hive. But it will also increase the number of
+ // open transactions at any given time, possibly impacting read
+ // performance.
+ HIVE_TXN_MAX_OPEN_BATCH("hive.txn.max.open.batch", 1000),
+
+ // Whether to run the compactor's initiator thread in this metastore
instance or not.
+ HIVE_COMPACTOR_INITIATOR_ON("hive.compactor.initiator.on", false),
+
+ // Number of compactor worker threads to run on this metastore instance.
+ HIVE_COMPACTOR_WORKER_THREADS("hive.compactor.worker.threads", 0),
+
+ // Time, in seconds, before a given compaction in working state is
declared a failure and
+ // returned to the initiated state.
+ HIVE_COMPACTOR_WORKER_TIMEOUT("hive.compactor.worker.timeout", 86400L),
+
+ // Time in seconds between checks to see if any partitions need compacted.
This should be
+ // kept high because each check for compaction requires many calls against
the NameNode.
+ HIVE_COMPACTOR_CHECK_INTERVAL("hive.compactor.check.interval", 300L),
+
+ // Number of delta files that must exist in a directory before the
compactor will attempt a
+ // minor compaction.
+ HIVE_COMPACTOR_DELTA_NUM_THRESHOLD("hive.compactor.delta.num.threshold",
10),
+
+ // Percentage (by size) of base that deltas can be before major compaction
is initiated.
+ HIVE_COMPACTOR_DELTA_PCT_THRESHOLD("hive.compactor.delta.pct.threshold",
0.1f),
+
+ // Number of aborted transactions involving a particular table or
partition before major
+ // compaction is initiated.
+ HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD("hive.compactor.abortedtxn.threshold",
1000),
+
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true),
Added:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java?rev=1574266&view=auto
==============================================================================
---
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
(added)
+++
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
Wed Mar 5 00:20:53 2014
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import junit.framework.Assert;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.LockComponentBuilder;
+import org.apache.hadoop.hive.metastore.LockRequestBuilder;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link
org.apache.hadoop.hive.metastore.HiveMetaStoreClient}. For now this just has
+ * transaction and locking tests. The goal here is not to test all
+ * functionality possible through the interface, as all permutations of DB
+ * operations should be tested in the appropriate DB handler classes. The
+ * goal is to test that we can properly pass the messages through the thrift
+ * service.
+ *
+ * This is in the ql directory rather than the metastore directory because it
+ * required the hive-exec jar, and hive-exec jar already depends on
+ * hive-metastore jar, thus I can't make hive-metastore depend on hive-exec.
+ */
+public class TestHiveMetaStoreTxns {
+
+ private HiveConf conf = new HiveConf();
+ private IMetaStoreClient client;
+
+ public TestHiveMetaStoreTxns() throws Exception {
+ TxnDbUtil.setConfValues(conf);
+ LogManager.getRootLogger().setLevel(Level.DEBUG);
+ tearDown();
+ }
+
+ @Test
+ public void testTxns() throws Exception {
+ List<Long> tids = client.openTxns("me", 3).getTxn_ids();
+ Assert.assertEquals(1L, (long) tids.get(0));
+ Assert.assertEquals(2L, (long) tids.get(1));
+ Assert.assertEquals(3L, (long) tids.get(2));
+ client.rollbackTxn(1);
+ client.commitTxn(2);
+ IMetaStoreClient.ValidTxnList validTxns = client.getValidTxns();
+ Assert.assertFalse(validTxns.isTxnCommitted(1));
+ Assert.assertTrue(validTxns.isTxnCommitted(2));
+ Assert.assertFalse(validTxns.isTxnCommitted(3));
+ Assert.assertFalse(validTxns.isTxnCommitted(4));
+ }
+
+ @Test
+ public void testTxnRange() throws Exception {
+ IMetaStoreClient.ValidTxnList validTxns = client.getValidTxns();
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.NONE,
+ validTxns.isTxnRangeCommitted(1L, 3L));
+
+ List<Long> tids = client.openTxns("me", 5).getTxn_ids();
+ client.rollbackTxn(1L);
+ client.commitTxn(2L);
+ client.commitTxn(3L);
+ client.commitTxn(4L);
+ validTxns = client.getValidTxns();
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.ALL,
+ validTxns.isTxnRangeCommitted(2L, 2L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.ALL,
+ validTxns.isTxnRangeCommitted(2L, 3L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.ALL,
+ validTxns.isTxnRangeCommitted(2L, 4L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.ALL,
+ validTxns.isTxnRangeCommitted(3L, 4L));
+
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.SOME,
+ validTxns.isTxnRangeCommitted(1L, 4L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.SOME,
+ validTxns.isTxnRangeCommitted(2L, 5L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.SOME,
+ validTxns.isTxnRangeCommitted(1L, 2L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.SOME,
+ validTxns.isTxnRangeCommitted(4L, 5L));
+
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.NONE,
+ validTxns.isTxnRangeCommitted(1L, 1L));
+ Assert.assertEquals(IMetaStoreClient.ValidTxnList.RangeResponse.NONE,
+ validTxns.isTxnRangeCommitted(5L, 10L));
+
+ }
+
+ @Test
+ public void testLocks() throws Exception {
+ LockRequestBuilder rqstBuilder = new LockRequestBuilder();
+ rqstBuilder.addLockComponent(new LockComponentBuilder()
+ .setDbName("mydb")
+ .setTableName("mytable")
+ .setPartitionName("mypartition")
+ .setExclusive()
+ .build());
+ rqstBuilder.addLockComponent(new LockComponentBuilder()
+ .setDbName("mydb")
+ .setTableName("yourtable")
+ .setSemiShared()
+ .build());
+ rqstBuilder.addLockComponent(new LockComponentBuilder()
+ .setDbName("yourdb")
+ .setShared()
+ .build());
+ rqstBuilder.setUser("fred");
+
+ LockResponse res = client.lock(rqstBuilder.build());
+ Assert.assertEquals(1L, res.getLockid());
+ Assert.assertEquals(LockState.ACQUIRED, res.getState());
+
+ res = client.checkLock(1);
+ Assert.assertEquals(1L, res.getLockid());
+ Assert.assertEquals(LockState.ACQUIRED, res.getState());
+
+ client.heartbeat(0, 1);
+
+ client.unlock(1);
+ }
+
+ @Test
+ public void testLocksWithTxn() throws Exception {
+ long txnid = client.openTxn("me");
+
+ LockRequestBuilder rqstBuilder = new LockRequestBuilder();
+ rqstBuilder.setTransactionId(txnid)
+ .addLockComponent(new LockComponentBuilder()
+ .setDbName("mydb")
+ .setTableName("mytable")
+ .setPartitionName("mypartition")
+ .setExclusive()
+ .build())
+ .addLockComponent(new LockComponentBuilder()
+ .setDbName("mydb")
+ .setTableName("yourtable")
+ .setSemiShared()
+ .build())
+ .addLockComponent(new LockComponentBuilder()
+ .setDbName("yourdb")
+ .setShared()
+ .build())
+ .setUser("fred");
+
+ LockResponse res = client.lock(rqstBuilder.build());
+ Assert.assertEquals(1L, res.getLockid());
+ Assert.assertEquals(LockState.ACQUIRED, res.getState());
+
+ res = client.checkLock(1);
+ Assert.assertEquals(1L, res.getLockid());
+ Assert.assertEquals(LockState.ACQUIRED, res.getState());
+
+ client.heartbeat(txnid, 1);
+
+ client.commitTxn(txnid);
+ }
+
+ @Test
+ public void stringifyValidTxns() throws Exception {
+ // Test with just high water mark
+ GetOpenTxnsResponse rsp = new GetOpenTxnsResponse();
+ rsp.setTxn_high_water_mark(1);
+ IMetaStoreClient.ValidTxnList validTxns = new
HiveMetaStoreClient.ValidTxnListImpl(rsp);
+ String asString = validTxns.toString();
+ Assert.assertEquals("1:", asString);
+ validTxns = new HiveMetaStoreClient.ValidTxnListImpl();
+ validTxns.fromString(asString);
+ Assert.assertEquals(1, validTxns.getOpenTxns().getTxn_high_water_mark());
+ Assert.assertNotNull(validTxns.getOpenTxns().getOpen_txns());
+ Assert.assertEquals(0, validTxns.getOpenTxns().getOpen_txnsSize());
+ asString = validTxns.toString();
+ Assert.assertEquals("1:", asString);
+ validTxns = new HiveMetaStoreClient.ValidTxnListImpl();
+ validTxns.fromString(asString);
+ Assert.assertEquals(1, validTxns.getOpenTxns().getTxn_high_water_mark());
+ Assert.assertNotNull(validTxns.getOpenTxns().getOpen_txns());
+ Assert.assertEquals(0, validTxns.getOpenTxns().getOpen_txnsSize());
+
+ // Test with open transactions
+ rsp = new GetOpenTxnsResponse();
+ rsp.setTxn_high_water_mark(10);
+ Set<Long> openTxns = new HashSet<Long>(2);
+ openTxns.add(3L);
+ openTxns.add(5L);
+ rsp.setOpen_txns(openTxns);
+ validTxns = new HiveMetaStoreClient.ValidTxnListImpl(rsp);
+ asString = validTxns.toString();
+ if (!asString.equals("10:3:5") && !asString.equals("10:5:3")) {
+ Assert.fail("Unexpected string value " + asString);
+ }
+ validTxns = new HiveMetaStoreClient.ValidTxnListImpl();
+ validTxns.fromString(asString);
+ Assert.assertEquals(10, validTxns.getOpenTxns().getTxn_high_water_mark());
+ Assert.assertNotNull(validTxns.getOpenTxns().getOpen_txns());
+ Assert.assertEquals(2, validTxns.getOpenTxns().getOpen_txnsSize());
+ boolean sawThree = false, sawFive = false;
+ for (long tid : validTxns.getOpenTxns().getOpen_txns()) {
+ if (tid == 3) sawThree = true;
+ else if (tid == 5) sawFive = true;
+ else Assert.fail("Unexpected value " + tid);
+ }
+ Assert.assertTrue(sawThree);
+ Assert.assertTrue(sawFive);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ TxnDbUtil.prepDb();
+ client = new HiveMetaStoreClient(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TxnDbUtil.cleanDb();
+ }
+}
Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Wed Mar 5 00:20:53 2014
@@ -70,6 +70,37 @@ enum PartitionEventType {
LOAD_DONE = 1,
}
+// Enums for transaction and lock management
+enum TxnState {
+ COMMITTED = 1,
+ ABORTED = 2,
+ OPEN = 3,
+}
+
+enum LockLevel {
+ DB = 1,
+ TABLE = 2,
+ PARTITION = 3,
+}
+
+enum LockState {
+ ACQUIRED = 1, // requester has the lock
+ WAITING = 2, // requester is waiting for the lock and should call
checklock at a later point to see if the lock has been obtained.
+ ABORT = 3, // the lock has been aborted, most likely due to
timeout
+ NOT_ACQUIRED = 4, // returned only with lockNoWait, indicates the lock
was not available and was not acquired
+}
+
+enum LockType {
+ SHARED_READ = 1,
+ SHARED_WRITE = 2,
+ EXCLUSIVE = 3,
+}
+
+enum CompactionType {
+ MINOR = 1,
+ MAJOR = 2,
+}
+
struct HiveObjectRef{
1: HiveObjectType objectType,
2: string dbName,
@@ -383,6 +414,122 @@ struct Function {
8: list<ResourceUri> resourceUris,
}
+// Structs for transaction and locks
+struct TxnInfo {
+ 1: required i64 id,
+ 2: required TxnState state,
+ 3: required string user, // used in 'show transactions' to help
admins find who has open transactions
+ 4: required string hostname, // used in 'show transactions' to help
admins find who has open transactions
+}
+
+struct GetOpenTxnsInfoResponse {
+ 1: required i64 txn_high_water_mark,
+ 2: required list<TxnInfo> open_txns,
+}
+
+struct GetOpenTxnsResponse {
+ 1: required i64 txn_high_water_mark,
+ 2: required set<i64> open_txns,
+}
+
+struct OpenTxnRequest {
+ 1: required i32 num_txns,
+ 2: required string user,
+ 3: required string hostname,
+}
+
+struct OpenTxnsResponse {
+ 1: required list<i64> txn_ids,
+}
+
+struct AbortTxnRequest {
+ 1: required i64 txnid,
+}
+
+struct CommitTxnRequest {
+ 1: required i64 txnid,
+}
+
+struct LockComponent {
+ 1: required LockType type,
+ 2: required LockLevel level,
+ 3: required string dbname,
+ 4: optional string tablename,
+ 5: optional string partitionname,
+}
+
+struct LockRequest {
+ 1: required list<LockComponent> component,
+ 2: optional i64 txnid,
+ 3: required string user, // used in 'show locks' to help admins find
who has open locks
+ 4: required string hostname, // used in 'show locks' to help admins find
who has open locks
+}
+
+struct LockResponse {
+ 1: required i64 lockid,
+ 2: required LockState state,
+}
+
+struct CheckLockRequest {
+ 1: required i64 lockid,
+}
+
+struct UnlockRequest {
+ 1: required i64 lockid,
+}
+
+struct ShowLocksRequest {
+}
+
+struct ShowLocksResponseElement {
+ 1: required i64 lockid,
+ 2: required string dbname,
+ 3: optional string tablename,
+ 4: optional string partname,
+ 5: required LockState state,
+ 6: required LockType type,
+ 7: optional i64 txnid,
+ 8: required i64 lastheartbeat,
+ 9: optional i64 acquiredat,
+ 10: required string user,
+ 11: required string hostname,
+}
+
+struct ShowLocksResponse {
+ 1: list<ShowLocksResponseElement> locks,
+}
+
+struct HeartbeatRequest {
+ 1: optional i64 lockid,
+ 2: optional i64 txnid
+}
+
+struct CompactionRequest {
+ 1: required string dbname,
+ 2: required string tablename,
+ 3: optional string partitionname,
+ 4: required CompactionType type,
+ 5: optional string runas,
+}
+
+struct ShowCompactRequest {
+}
+
+struct ShowCompactResponseElement {
+ 1: required string dbname,
+ 2: required string tablename,
+ 3: required string partitionname,
+ 4: required CompactionType type,
+ 5: required string state,
+ 6: required string workerid,
+ 7: required i64 start,
+ 8: required string runAs,
+}
+
+struct ShowCompactResponse {
+ 1: required list<ShowCompactResponseElement> compacts,
+}
+
exception MetaException {
1: string message
}
@@ -431,6 +578,23 @@ exception InvalidInputException {
1: string message
}
+// Transaction and lock exceptions
+exception NoSuchTxnException {
+ 1: string message
+}
+
+exception TxnAbortedException {
+ 1: string message
+}
+
+exception TxnOpenException {
+ 1: string message
+}
+
+exception NoSuchLockException {
+ 1: string message
+}
+
/**
* This interface is live.
*/
@@ -776,6 +940,23 @@ service ThriftHiveMetastore extends fb30
// method to cancel delegation token obtained from metastore server
void cancel_delegation_token(1:string token_str_form) throws
(1:MetaException o1)
+
+ // Transaction and lock management calls
+ // Get just list of open transactions
+ GetOpenTxnsResponse get_open_txns()
+ // Get list of open transactions with state (open, aborted)
+ GetOpenTxnsInfoResponse get_open_txns_info()
+ OpenTxnsResponse open_txns(1:OpenTxnRequest rqst)
+ void abort_txn(1:AbortTxnRequest rqst) throws (1:NoSuchTxnException o1)
+ void commit_txn(1:CommitTxnRequest rqst) throws (1:NoSuchTxnException o1,
2:TxnAbortedException o2)
+ LockResponse lock(1:LockRequest rqst) throws (1:NoSuchTxnException o1,
2:TxnAbortedException o2)
+ LockResponse check_lock(1:CheckLockRequest rqst)
+ throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2,
3:NoSuchLockException o3)
+ void unlock(1:UnlockRequest rqst) throws (1:NoSuchLockException o1,
2:TxnOpenException o2)
+ ShowLocksResponse show_locks(1:ShowLocksRequest rqst)
+ void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1,
2:NoSuchTxnException o2, 3:TxnAbortedException o3)
+ void compact(1:CompactionRequest rqst)
+ ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
}
// * Note about the DDL_TIME: When creating or altering a table or a partition,
Modified: hive/trunk/metastore/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/pom.xml?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/metastore/pom.xml (original)
+++ hive/trunk/metastore/pom.xml Wed Mar 5 00:20:53 2014
@@ -54,7 +54,6 @@
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
<version>${bonecp.version}</version>
- <scope>runtime</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
Modified:
hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
(original)
+++ hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql Wed
Mar 5 00:20:53 2014
@@ -315,3 +315,8 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK
(IS_COMPRESSED IN ('Y','N'));
INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES
(1, '0.13.0', 'Hive release version 0.13.0');
+
+-- ----------------------------
+-- Transaction and Lock Tables
+-- ----------------------------
+RUN 'hive-txn-schema-0.13.0.derby.sql';
Added:
hive/trunk/metastore/scripts/upgrade/derby/hive-txn-schema-0.13.0.derby.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/derby/hive-txn-schema-0.13.0.derby.sql?rev=1574266&view=auto
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/derby/hive-txn-schema-0.13.0.derby.sql
(added)
+++ hive/trunk/metastore/scripts/upgrade/derby/hive-txn-schema-0.13.0.derby.sql
Wed Mar 5 00:20:53 2014
@@ -0,0 +1,88 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the License); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an AS IS BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+--
+-- Tables for transaction management
+--
+CREATE TABLE TXNS (
+ TXN_ID bigint PRIMARY KEY,
+ TXN_STATE char(1) NOT NULL,
+ TXN_STARTED bigint NOT NULL,
+ TXN_LAST_HEARTBEAT bigint NOT NULL,
+ TXN_USER varchar(128) NOT NULL,
+ TXN_HOST varchar(128) NOT NULL
+);
+
+CREATE TABLE TXN_COMPONENTS (
+ TC_TXNID bigint REFERENCES TXNS (TXN_ID),
+ TC_DATABASE varchar(128) NOT NULL,
+ TC_TABLE varchar(128),
+ TC_PARTITION varchar(767)
+);
+
+CREATE TABLE COMPLETED_TXN_COMPONENTS (
+ CTC_TXNID bigint,
+ CTC_DATABASE varchar(128) NOT NULL,
+ CTC_TABLE varchar(128),
+ CTC_PARTITION varchar(767)
+);
+
+CREATE TABLE NEXT_TXN_ID (
+ NTXN_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_TXN_ID VALUES(1);
+
+CREATE TABLE HIVE_LOCKS (
+ HL_LOCK_EXT_ID bigint NOT NULL,
+ HL_LOCK_INT_ID bigint NOT NULL,
+ HL_TXNID bigint,
+ HL_DB varchar(128) NOT NULL,
+ HL_TABLE varchar(128),
+ HL_PARTITION varchar(767),
+ HL_LOCK_STATE char(1) NOT NULL,
+ HL_LOCK_TYPE char(1) NOT NULL,
+ HL_LAST_HEARTBEAT bigint NOT NULL,
+ HL_ACQUIRED_AT bigint,
+ HL_USER varchar(128) NOT NULL,
+ HL_HOST varchar(128) NOT NULL,
+ PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID)
+);
+
+CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
+
+CREATE TABLE NEXT_LOCK_ID (
+ NL_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_LOCK_ID VALUES(1);
+
+CREATE TABLE COMPACTION_QUEUE (
+ CQ_ID bigint PRIMARY KEY,
+ CQ_DATABASE varchar(128) NOT NULL,
+ CQ_TABLE varchar(128) NOT NULL,
+ CQ_PARTITION varchar(767),
+ CQ_STATE char(1) NOT NULL,
+ CQ_TYPE char(1) NOT NULL,
+ CQ_WORKER_ID varchar(128),
+ CQ_START bigint,
+ CQ_RUN_AS varchar(128)
+);
+
+CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+ NCQ_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+
+
+
Modified:
hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
(original)
+++
hive/trunk/metastore/scripts/upgrade/derby/upgrade-0.12.0-to-0.13.0.derby.sql
Wed Mar 5 00:20:53 2014
@@ -1,4 +1,5 @@
-- Upgrade MetaStore schema from 0.12.0 to 0.13.0
+RUN 'hive-txn-schema-0.13.0.derby.sql';
RUN '016-HIVE-6386.derby.sql';
RUN '017-HIVE-6458.derby.sql';
Modified:
hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
(original)
+++ hive/trunk/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql Wed
Mar 5 00:20:53 2014
@@ -792,7 +792,12 @@ CREATE TABLE IF NOT EXISTS `FUNC_RU` (
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1,
'0.13.0', 'Hive release version 0.13.0');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1,
'0.12.0', 'Hive release version 0.12.0');
+
+------------------------------
+-- Transaction and Lock Tables
+------------------------------
+SOURCE hive-txn-schema-0.13.0.mysql.sql;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
Added:
hive/trunk/metastore/scripts/upgrade/mysql/hive-txn-schema-0.13.0.mysql.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/mysql/hive-txn-schema-0.13.0.mysql.sql?rev=1574266&view=auto
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/mysql/hive-txn-schema-0.13.0.mysql.sql
(added)
+++ hive/trunk/metastore/scripts/upgrade/mysql/hive-txn-schema-0.13.0.mysql.sql
Wed Mar 5 00:20:53 2014
@@ -0,0 +1,90 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+--
+-- Tables for transaction management
+--
+
+CREATE TABLE TXNS (
+ TXN_ID bigint PRIMARY KEY,
+ TXN_STATE char(1) NOT NULL,
+ TXN_STARTED bigint NOT NULL,
+ TXN_LAST_HEARTBEAT bigint NOT NULL,
+ TXN_USER varchar(128) NOT NULL,
+ TXN_HOST varchar(128) NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE TXN_COMPONENTS (
+ TC_TXNID bigint,
+ TC_DATABASE varchar(128) NOT NULL,
+ TC_TABLE varchar(128),
+ TC_PARTITION varchar(767),
+ FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID)
+) ENGINE=InnoDB;
+
+CREATE TABLE COMPLETED_TXN_COMPONENTS (
+ CTC_TXNID bigint,
+ CTC_DATABASE varchar(128) NOT NULL,
+ CTC_TABLE varchar(128),
+ CTC_PARTITION varchar(767)
+) ENGINE=InnoDB;
+
+CREATE TABLE NEXT_TXN_ID (
+ NTXN_NEXT bigint NOT NULL
+) ENGINE=InnoDB;
+INSERT INTO NEXT_TXN_ID VALUES(1);
+
+CREATE TABLE HIVE_LOCKS (
+ HL_LOCK_EXT_ID bigint NOT NULL,
+ HL_LOCK_INT_ID bigint NOT NULL,
+ HL_TXNID bigint,
+ HL_DB varchar(128) NOT NULL,
+ HL_TABLE varchar(128),
+ HL_PARTITION varchar(767),
+ HL_LOCK_STATE char(1) not null,
+ HL_LOCK_TYPE char(1) not null,
+ HL_LAST_HEARTBEAT bigint NOT NULL,
+ HL_ACQUIRED_AT bigint,
+ HL_USER varchar(128) NOT NULL,
+ HL_HOST varchar(128) NOT NULL,
+ PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID),
+ KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID)
+) ENGINE=InnoDB;
+
+CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID) USING HASH;
+
+CREATE TABLE NEXT_LOCK_ID (
+ NL_NEXT bigint NOT NULL
+) ENGINE=InnoDB;
+INSERT INTO NEXT_LOCK_ID VALUES(1);
+
+CREATE TABLE COMPACTION_QUEUE (
+ CQ_ID bigint PRIMARY KEY,
+ CQ_DATABASE varchar(128) NOT NULL,
+ CQ_TABLE varchar(128) NOT NULL,
+ CQ_PARTITION varchar(767),
+ CQ_STATE char(1) NOT NULL,
+ CQ_TYPE char(1) NOT NULL,
+ CQ_WORKER_ID varchar(128),
+ CQ_START bigint,
+ CQ_RUN_AS varchar(128)
+);
+
+CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+ NCQ_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+
+
Modified:
hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
(original)
+++
hive/trunk/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql
Wed Mar 5 00:20:53 2014
@@ -3,6 +3,7 @@ SELECT 'Upgrading MetaStore schema from
SOURCE 015-HIVE-5700.mysql.sql;
SOURCE 016-HIVE-6386.mysql.sql;
SOURCE 017-HIVE-6458.mysql.sql;
+SOURCE hive-txn-schema-0.13.0.mysql.sql;
UPDATE VERSION SET SCHEMA_VERSION='0.13.0', VERSION_COMMENT='Hive release
version 0.13.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.12.0 to 0.13.0' AS ' ';
Modified:
hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
(original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
Wed Mar 5 00:20:53 2014
@@ -754,3 +754,7 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUN
INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1,
'0.13.0', 'Hive release version 0.13.0');
+------------------------------
+-- Transaction and lock tables
+------------------------------
[email protected];
Added:
hive/trunk/metastore/scripts/upgrade/oracle/hive-txn-schema-0.13.0.oracle.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/hive-txn-schema-0.13.0.oracle.sql?rev=1574266&view=auto
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/oracle/hive-txn-schema-0.13.0.oracle.sql
(added)
+++
hive/trunk/metastore/scripts/upgrade/oracle/hive-txn-schema-0.13.0.oracle.sql
Wed Mar 5 00:20:53 2014
@@ -0,0 +1,88 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the License); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an AS IS BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+--
+-- Tables for transaction management
+--
+
+CREATE TABLE TXNS (
+ TXN_ID NUMBER(10) PRIMARY KEY,
+ TXN_STATE char(1) NOT NULL,
+ TXN_STARTED NUMBER(10) NOT NULL,
+ TXN_LAST_HEARTBEAT NUMBER(10) NOT NULL,
+ TXN_USER varchar(128) NOT NULL,
+ TXN_HOST varchar(128) NOT NULL
+);
+
+CREATE TABLE TXN_COMPONENTS (
+ TC_TXNID NUMBER(10) REFERENCES TXNS (TXN_ID),
+ TC_DATABASE VARCHAR2(128) NOT NULL,
+ TC_TABLE VARCHAR2(128),
+ TC_PARTITION VARCHAR2(767) NULL
+);
+
+CREATE TABLE COMPLETED_TXN_COMPONENTS (
+ CTC_TXNID NUMBER(10),
+ CTC_DATABASE varchar(128) NOT NULL,
+ CTC_TABLE varchar(128),
+ CTC_PARTITION varchar(767)
+);
+
+CREATE TABLE NEXT_TXN_ID (
+ NTXN_NEXT NUMBER(10) NOT NULL
+);
+INSERT INTO NEXT_TXN_ID VALUES(1);
+
+CREATE TABLE HIVE_LOCKS (
+ HL_LOCK_EXT_ID NUMBER(10) NOT NULL,
+ HL_LOCK_INT_ID NUMBER(10) NOT NULL,
+ HL_TXNID NUMBER(10),
+ HL_DB VARCHAR2(128) NOT NULL,
+ HL_TABLE VARCHAR2(128),
+ HL_PARTITION VARCHAR2(767),
+ HL_LOCK_STATE CHAR(1) NOT NULL,
+ HL_LOCK_TYPE CHAR(1) NOT NULL,
+ HL_LAST_HEARTBEAT NUMBER(10) NOT NULL,
+ HL_ACQUIRED_AT NUMBER(10),
+ HL_USER varchar(128) NOT NULL,
+ HL_HOST varchar(128) NOT NULL,
+ PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID)
+);
+
+CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
+
+CREATE TABLE NEXT_LOCK_ID (
+ NL_NEXT NUMBER(10) NOT NULL
+);
+INSERT INTO NEXT_LOCK_ID VALUES(1);
+
+CREATE TABLE COMPACTION_QUEUE (
+ CQ_ID NUMBER(10) PRIMARY KEY,
+ CQ_DATABASE varchar(128) NOT NULL,
+ CQ_TABLE varchar(128) NOT NULL,
+ CQ_PARTITION varchar(767),
+ CQ_STATE char(1) NOT NULL,
+ CQ_TYPE char(1) NOT NULL,
+ CQ_WORKER_ID varchar(128),
+ CQ_START NUMBER(10),
+ CQ_RUN_AS varchar(128)
+);
+
+CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+ NCQ_NEXT NUMBER(10) NOT NULL
+);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+
+
Modified:
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
(original)
+++
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
Wed Mar 5 00:20:53 2014
@@ -3,6 +3,7 @@ SELECT 'Upgrading MetaStore schema from
@015-HIVE-5700.oracle.sql;
@016-HIVE-6386.oracle.sql;
@017-HIVE-6458.oracle.sql;
[email protected];
UPDATE VERSION SET SCHEMA_VERSION='0.13.0', VERSION_COMMENT='Hive release
version 0.13.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.12.0 to 0.13.0' AS Status
from dual;
Modified:
hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
(original)
+++
hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.13.0.postgres.sql
Wed Mar 5 00:20:53 2014
@@ -1456,3 +1456,7 @@ INSERT INTO "VERSION" ("VER_ID", "SCHEMA
-- PostgreSQL database dump complete
--
+------------------------------
+-- Transaction and lock tables
+------------------------------
+\i hive-txn-schema-0.13.0.postgres.sql;
Added:
hive/trunk/metastore/scripts/upgrade/postgres/hive-txn-schema-0.13.0.postgres.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/hive-txn-schema-0.13.0.postgres.sql?rev=1574266&view=auto
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/postgres/hive-txn-schema-0.13.0.postgres.sql
(added)
+++
hive/trunk/metastore/scripts/upgrade/postgres/hive-txn-schema-0.13.0.postgres.sql
Wed Mar 5 00:20:53 2014
@@ -0,0 +1,89 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+--
+-- Tables for transaction management
+--
+
+CREATE TABLE TXNS (
+ TXN_ID bigint PRIMARY KEY,
+ TXN_STATE char(1) NOT NULL,
+ TXN_STARTED bigint NOT NULL,
+ TXN_LAST_HEARTBEAT bigint NOT NULL,
+ TXN_USER varchar(128) NOT NULL,
+ TXN_HOST varchar(128) NOT NULL
+);
+
+CREATE TABLE TXN_COMPONENTS (
+ TC_TXNID bigint REFERENCES TXNS (TXN_ID),
+ TC_DATABASE varchar(128) NOT NULL,
+ TC_TABLE varchar(128),
+ TC_PARTITION varchar(767) DEFAULT NULL
+);
+
+CREATE TABLE COMPLETED_TXN_COMPONENTS (
+ CTC_TXNID bigint,
+ CTC_DATABASE varchar(128) NOT NULL,
+ CTC_TABLE varchar(128),
+ CTC_PARTITION varchar(767)
+);
+
+CREATE TABLE NEXT_TXN_ID (
+ NTXN_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_TXN_ID VALUES(1);
+
+CREATE TABLE HIVE_LOCKS (
+ HL_LOCK_EXT_ID bigint NOT NULL,
+ HL_LOCK_INT_ID bigint NOT NULL,
+ HL_TXNID bigint,
+ HL_DB varchar(128) NOT NULL,
+ HL_TABLE varchar(128),
+ HL_PARTITION varchar(767) DEFAULT NULL,
+ HL_LOCK_STATE char(1) NOT NULL,
+ HL_LOCK_TYPE char(1) NOT NULL,
+ HL_LAST_HEARTBEAT bigint NOT NULL,
+ HL_ACQUIRED_AT bigint,
+ HL_USER varchar(128) NOT NULL,
+ HL_HOST varchar(128) NOT NULL,
+ PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID)
+);
+
+CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS USING hash (HL_TXNID);
+
+CREATE TABLE NEXT_LOCK_ID (
+ NL_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_LOCK_ID VALUES(1);
+
+CREATE TABLE COMPACTION_QUEUE (
+ CQ_ID bigint PRIMARY KEY,
+ CQ_DATABASE varchar(128) NOT NULL,
+ CQ_TABLE varchar(128) NOT NULL,
+ CQ_PARTITION varchar(767),
+ CQ_STATE char(1) NOT NULL,
+ CQ_TYPE char(1) NOT NULL,
+ CQ_WORKER_ID varchar(128),
+ CQ_START bigint,
+ CQ_RUN_AS varchar(128)
+);
+
+CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+ NCQ_NEXT bigint NOT NULL
+);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+
+
+
Modified:
hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql?rev=1574266&r1=1574265&r2=1574266&view=diff
==============================================================================
---
hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
(original)
+++
hive/trunk/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql
Wed Mar 5 00:20:53 2014
@@ -3,6 +3,7 @@ SELECT 'Upgrading MetaStore schema from
\i 015-HIVE-5700.postgres.sql;
\i 016-HIVE-6386.postgres.sql;
\i 017-HIVE-6458.postgres.sql;
+\i hive-txn-schema-0.13.0.postgres.sql;
UPDATE "VERSION" SET "SCHEMA_VERSION"='0.13.0', "VERSION_COMMENT"='Hive
release version 0.13.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 0.12.0 to 0.13.0';
