http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index 14ff58e..e46e65b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -87,7 +87,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -102,7 +102,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -121,7 +121,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(3, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -140,7 +140,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(4, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -155,7 +155,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -171,7 +171,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -307,7 +307,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(4, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -322,7 +322,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -337,7 +337,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -352,7 +352,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.rollbackTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -367,7 +367,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.rollbackTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -382,7 +382,7 @@ public class TestDbTxnManager { List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, - TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); + TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId)); txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -481,7 +481,7 @@ public class TestDbTxnManager { @Before public void setUp() throws Exception { - TxnDbUtil.prepDb(); + TxnDbUtil.prepDb(conf); txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr.getLockManager();//init lock manager Assert.assertTrue(txnMgr instanceof DbTxnManager); @@ -497,7 +497,7 @@ public class TestDbTxnManager { public void tearDown() throws Exception { if(houseKeeperService != null) houseKeeperService.stop(); if (txnMgr != null) txnMgr.closeTxnManager(); - TxnDbUtil.cleanDb(); + TxnDbUtil.cleanDb(conf); } private static class MockQueryPlan extends QueryPlan {
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 6d1cdcb..e9833cb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -104,8 +104,8 @@ public class TestDbTxnManager2 { ctx = new Context(conf); driver = new Driver(conf); driver.init(); - TxnDbUtil.cleanDb(); - TxnDbUtil.prepDb(); + TxnDbUtil.cleanDb(conf); + TxnDbUtil.prepDb(conf); SessionState ss = SessionState.get(); ss.initTxnMgr(conf); txnMgr = ss.getTxnMgr(); @@ -445,9 +445,9 @@ public class TestDbTxnManager2 { checkCmdOnDriver(cpr); cpr = driver.run("insert into temp.T13p partition (ds='tomorrow', hour='2') values (8, 8)"); checkCmdOnDriver(cpr); - int count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')"); + int count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')"); Assert.assertEquals(4, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')"); Assert.assertEquals(4, count); // Fail some inserts, so that we have records in TXN_COMPONENTS @@ -460,132 +460,132 @@ public class TestDbTxnManager2 { checkCmdOnDriver(cpr); cpr = driver.run("insert into temp.T13p partition (ds='today', hour='1') values (12, 12)"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(4, count); conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); // Drop a table/partition; corresponding records in TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS should disappear - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); Assert.assertEquals(2, count); cpr = driver.run("drop table temp.T10"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); Assert.assertEquals(1, count); cpr = driver.run("alter table temp.T12p drop partition (ds='today', hour='1')"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); Assert.assertEquals(0, count); // Successfully perform compaction on a table/partition, so that we have successful records in COMPLETED_COMPACTIONS cpr = driver.run("alter table temp.T11 compact 'minor'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'"); Assert.assertEquals(1, count); cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'minor'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'"); Assert.assertEquals(1, count); // Fail compaction, so that we have failed records in COMPLETED_COMPACTIONS conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, true); cpr = driver.run("alter table temp.T11 compact 'major'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'"); Assert.assertEquals(1, count); cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(1, count); org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'"); Assert.assertEquals(1, count); conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false); // Put 2 records into COMPACTION_QUEUE and do nothing cpr = driver.run("alter table temp.T11 compact 'major'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(1, count); cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(1, count); // Drop a table/partition, corresponding records in COMPACTION_QUEUE and COMPLETED_COMPACTIONS should disappear cpr = driver.run("drop table temp.T11"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'"); Assert.assertEquals(0, count); cpr = driver.run("alter table temp.T12p drop partition (ds='tomorrow', hour='2')"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'"); Assert.assertEquals(0, count); // Put 1 record into COMPACTION_QUEUE and do nothing cpr = driver.run("alter table temp.T13p partition (ds='today', hour='1') compact 'major'"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'"); Assert.assertEquals(1, count); // Drop database, everything in all 4 meta tables should disappear - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(2, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); cpr = driver.run("drop database if exists temp cascade"); checkCmdOnDriver(cpr); - count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); } @@ -959,7 +959,7 @@ public class TestDbTxnManager2 { @Test public void testWriteSetTracking4() throws Exception { dropTable(new String[] {"TAB_PART", "TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); @@ -986,14 +986,14 @@ public class TestDbTxnManager2 { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks); //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab2", Collections.EMPTY_LIST); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); txnMgr2.commitTxn(); //Short Running updated nothing, so we expect 0 rows in WRITE_SET - Assert.assertEquals( 0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals( 0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); txnMgr2.openTxn(ctx, "T3"); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'"));//pretend this partition exists @@ -1003,19 +1003,19 @@ public class TestDbTxnManager2 { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);//since TAB2 is empty //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(), "default", "tab2", Collections.singletonList("p=two")); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp);//simulate partition update txnMgr2.commitTxn(); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); AcidWriteSetService houseKeeper = new AcidWriteSetService(); TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); //since T3 overlaps with Long Running (still open) GC does nothing - Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));//no rows match txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running"); //so generate empty Dyn Part call @@ -1028,7 +1028,7 @@ public class TestDbTxnManager2 { locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 0, locks.size()); TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } /** * overlapping txns updating the same resource but 1st one rolls back; 2nd commits @@ -1037,7 +1037,7 @@ public class TestDbTxnManager2 { @Test public void testWriteSetTracking5() throws Exception { dropTable(new String[] {"TAB_PART"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); @@ -1063,9 +1063,9 @@ public class TestDbTxnManager2 { Arrays.asList("p=blah")); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); txnMgr2.commitTxn();//since conflicting txn rolled back, commit succeeds - Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } /** * check that read query concurrent with txn works ok @@ -1073,7 +1073,7 @@ public class TestDbTxnManager2 { @Test public void testWriteSetTracking6() throws Exception { dropTable(new String[] {"TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); CommandProcessorResponse cpr = driver.run("create table if not exists TAB2(a int, b int) clustered " + "by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); @@ -1086,19 +1086,19 @@ public class TestDbTxnManager2 { swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 101")); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Horton"); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr2.commitTxn();//no conflict - Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr.commitTxn(); TestTxnCommands2.runHouseKeeperService(new AcidWriteSetService(), conf); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } /** @@ -1108,7 +1108,7 @@ public class TestDbTxnManager2 { @Test public void testWriteSetTracking7() throws Exception { dropTable(new String[] {"tab2", "TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); CommandProcessorResponse cpr = driver.run("create table if not exists tab2 (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); @@ -1150,13 +1150,13 @@ public class TestDbTxnManager2 { txnMgr.commitTxn();//txnid:idTxnUpdate2 //now both txns concurrently updated TAB2 but different partitions. - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'")); //2 from txnid:1, 1 from txnid:2, 1 from txnid:3 - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null")); //================ //test with predicates such that partition pruning doesn't kick in @@ -1204,13 +1204,13 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn();//txnid:idTxnUpdate4 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); //2 from insert + 1 for each update stmt - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); } /** * Concurrent updates with partition pruning predicate and w/o one @@ -1261,12 +1261,12 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn();//txnid:idTxnUpdate2 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); } /** * Concurrent update/delete of different partitions - should pass @@ -1317,18 +1317,18 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn();//txnid:idTxnUpdate2 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + " and ctc_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + " and ctc_table='tab1' and ctc_partition='p=one'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + " and ctc_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + " and ctc_table='tab1' and ctc_partition='p=one'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); } /** * Concurrent update/delete of same partition - should fail to commit @@ -1387,10 +1387,10 @@ public class TestDbTxnManager2 { "Aborting [txnid:5,5] due to a write conflict on default/tab1/p=two committed by [txnid:4,5] d/u", exception.getCause().getMessage()); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 3, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); } /** * Concurrent delte/detele of same partition - should pass @@ -1448,16 +1448,16 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn();//"select * from tab1" txn - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete)); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect)); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete)); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect)); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete)); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect)); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete)); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect)); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); } @Test public void testCompletedTxnComponents() throws Exception { @@ -1470,11 +1470,11 @@ public class TestDbTxnManager2 { checkCmdOnDriver(driver.run("insert into tab_not_acid2 values(1,1),(2,2)")); //writing both acid and non-acid resources in the same txn checkCmdOnDriver(driver.run("from tab_not_acid2 insert into tab1 partition(p='two')(a,b) select a,b insert into tab_not_acid2(a,b) select a,b "));//txnid:1 - Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); //only expect transactional components to be in COMPLETED_TXN_COMPONENTS - Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'")); } /** @@ -1495,13 +1495,13 @@ public class TestDbTxnManager2 { //writing both acid and non-acid resources in the same txn //tab1 write is a dynamic partition insert checkCmdOnDriver(driver.run("from tab_not_acid insert into tab1 partition(p)(a,b,p) select a,b,p insert into tab_not_acid(a,b) select a,b where p='two'"));//txnid:9 - Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); //only expect transactional components to be in COMPLETED_TXN_COMPONENTS - Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9")); - Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'")); } //todo: Concurrent insert/update of same partition - should pass @@ -1611,9 +1611,9 @@ public class TestDbTxnManager2 { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); //complete 1st txn AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target", Collections.singletonList("p=1/q=3"));//update clause @@ -1629,39 +1629,39 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + " and tc_operation_type='u'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 2, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + " and tc_operation_type='d'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 3, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + " and tc_operation_type='i'")); txnMgr.commitTxn();//commit T1 Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), 6, - TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1, - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + " and ws_operation_type='u'")); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 2, - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + " and ws_operation_type='d'")); //re-check locks which were in Waiting state - should now be Acquired @@ -1676,9 +1676,9 @@ public class TestDbTxnManager2 { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2)); //complete 2nd txn adp = new AddDynamicPartitions(txnId2, "default", "target", Collections.singletonList(cc ? "p=1/q=3" : "p=1/p=2"));//update clause @@ -1694,21 +1694,21 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + " and tc_operation_type='u'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 2, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + " and tc_operation_type='d'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 3, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + " and tc_operation_type='i'")); LockException expectedException = null; @@ -1725,33 +1725,33 @@ public class TestDbTxnManager2 { "committed by [txnid:10,11] u/u", expectedException.getMessage()); Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), 0, - TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 0, - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2)); } else { Assert.assertNull("Unexpected exception " + expectedException, expectedException); Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), 6, - TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1, - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 + " and ws_operation_type='u'")); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 2, - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 + " and ws_operation_type='d'")); } @@ -1790,9 +1790,9 @@ public class TestDbTxnManager2 { txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1,//no DP, so it's populated from lock info - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); @@ -1820,9 +1820,9 @@ public class TestDbTxnManager2 { txnMgr.commitTxn();//commit T1 Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), causeConflict ? 1 : 0,//Inserts are not tracked by WRITE_SET - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid1 + " and ws_operation_type=" + (causeConflict ? "'u'" : "'i'"))); @@ -1835,14 +1835,14 @@ public class TestDbTxnManager2 { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1,// - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1,// - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + "and tc_operation_type='d'")); //complete T2 txn @@ -1862,9 +1862,9 @@ public class TestDbTxnManager2 { } else { Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1,//Unpartitioned table: 1 row for Delete; Inserts are not tracked in WRITE_SET - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 + " and ws_operation_type='d'")); } } @@ -1887,15 +1887,15 @@ public class TestDbTxnManager2 { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks); Assert.assertEquals( "HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString("select * from HIVE_LOCKS"), + TxnDbUtil.queryToString(conf, "select * from HIVE_LOCKS"), 1, - TxnDbUtil.countQueryAgent("select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1)); txnMgr.rollbackTxn(); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); //now actually write to table to generate some partitions checkCmdOnDriver(driver.run("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)")); driver.run("select count(*) from target"); @@ -1904,10 +1904,10 @@ public class TestDbTxnManager2 { Assert.assertEquals("", "4", r.get(0)); Assert.assertEquals(//look in COMPLETED_TXN_COMPONENTS because driver.run() committed!!!! "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1 + 1) + "): " + - TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), 2,//2 distinct partitions created //txnid+1 because we want txn used by previous driver.run("insert....) - TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1))); + TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1))); long txnid2 = txnMgr.openTxn(ctx, "T1"); @@ -1922,9 +1922,9 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 2,//2 distinct partitions modified - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); txnMgr.commitTxn(); } @Test @@ -1986,9 +1986,9 @@ public class TestDbTxnManager2 { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0,//because it's using a DP write - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); //complete T1 transaction (simulate writing to 2 partitions) AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target", Arrays.asList("p=1/q=2","p=1/q=3")); @@ -1996,15 +1996,15 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 2, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + " and tc_operation_type='u'")); txnMgr.commitTxn();//commit T1 Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 2,//2 partitions updated - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + " and ws_operation_type='u'")); @@ -2021,9 +2021,9 @@ public class TestDbTxnManager2 { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 0,//because it's using a DP write - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); //complete T2 txn //simulate Insert into 2 partitions adp = new AddDynamicPartitions(txnid2, "default", "target", @@ -2032,9 +2032,9 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 2, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'")); //simulate Update of 1 partitions; depending on causeConflict, choose one of the partitions //which was modified by the T1 update stmt or choose a non-conflicting one adp = new AddDynamicPartitions(txnid2, "default", "target", @@ -2043,9 +2043,9 @@ public class TestDbTxnManager2 { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), 1, - TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'")); LockException expectedException = null; @@ -2064,14 +2064,14 @@ public class TestDbTxnManager2 { } else { Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1,//1 partitions updated - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 + + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 + " and ws_operation_type='u'")); Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString("select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), 1,//1 partitions updated (and no other entries) - TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2)); } } http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java index af8a63d..2784009 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java @@ -95,7 +95,7 @@ public abstract class CompactorTest { protected CompactorTest() throws Exception { conf = new HiveConf(); TxnDbUtil.setConfValues(conf); - TxnDbUtil.cleanDb(); + TxnDbUtil.cleanDb(conf); ms = new HiveMetaStoreClient(conf); txnHandler = TxnUtils.getTxnStore(conf); tmpdir = new File (Files.createTempDirectory("compactor_test_table_").toString()); http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/pom.xml ---------------------------------------------------------------------- diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index d91b22d..acc50ca 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -33,6 +33,11 @@ <dependencies> <dependency> + <groupId>com.jolbox</groupId> + <artifactId>bonecp</artifactId> + <version>${bonecp.version}</version> + </dependency> + <dependency> <groupId>com.github.joshelser</groupId> <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId> <version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version> @@ -48,6 +53,16 @@ <version>${protobuf.version}</version> </dependency> <dependency> + <groupId>com.zaxxer</groupId> + <artifactId>HikariCP</artifactId> + <version>${hikaricp.version}</version> + </dependency> + <dependency> + <groupId>commons-dbcp</groupId> + <artifactId>commons-dbcp</artifactId> + <version>${commons-dbcp.version}</version> + </dependency> + <dependency> <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> <version>${dropwizard.version}</version> @@ -68,6 +83,11 @@ <version>${commons-lang3.version}</version> </dependency> <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + <version>${derby.version}</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java new file mode 100644 index 0000000..9fa5cab --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configurable; + +/** + * Combination of Runnable and Configurable + */ +public interface RunnableConfigurable extends Configurable, Runnable { +} http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index d3c714b..5a6ef99 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -54,11 +55,8 @@ public class MetastoreConf { private static URL hiveSiteURL = null; private static URL hiveMetastoreSiteURL = null; private static URL metastoreSiteURL = null; + private static AtomicBoolean beenDumped = new AtomicBoolean(); - /* - private static Map<String, String> metaToHiveKeys; - private static Map<String, String> hiveToMetaKeys; - */ private static Map<String, ConfVars> keyToVars; @VisibleForTesting @@ -406,6 +404,9 @@ public class MetastoreConf { "not blocked.\n" + "\n" + "See HIVE-4409 for more details."), + DUMP_CONFIG_ON_CREATION("metastore.dump.config.on.creation", NO_SUCH_KEY, true, + "If true, a printout of the config file (minus sensitive values) will be dumped to the " + + "log whenever newMetastoreConf() is called. Can produce a lot of logs"), END_FUNCTION_LISTENERS("metastore.end.function.listeners", "hive.metastore.end.function.listeners", "", "List of comma separated listeners for the end of metastore functions."), @@ -932,6 +933,11 @@ public class MetastoreConf { if (getBoolVar(conf, ConfVars.SCHEMA_VERIFICATION)) { setBoolVar(conf, ConfVars.AUTO_CREATE_ALL, false); } + + if (!beenDumped.getAndSet(true) && getBoolVar(conf, ConfVars.DUMP_CONFIG_ON_CREATION) && + LOG.isInfoEnabled()) { + LOG.info(dumpConfig(conf)); + } return conf; } @@ -1304,8 +1310,8 @@ public class MetastoreConf { } /** - * Return the configuration value as a String. This does not work on time based values as it - * doesn't have a time unit. + * Return the configuration value as a String. For time based values it will be returned in + * the default time unit appended with an appropriate abbreviation (eg s for seconds, ...) * @param conf configuration to read * @param var variable to read * @return value as an object @@ -1319,8 +1325,12 @@ public class MetastoreConf { return Long.toString(getLongVar(conf, var)); } else if (var.defaultVal.getClass() == Double.class) { return Double.toString(getDoubleVar(conf, var)); + } else if (var.defaultVal.getClass() == TimeValue.class) { + TimeUnit timeUnit = (var.defaultVal.getClass() == TimeValue.class) ? + ((TimeValue)var.defaultVal).unit : null; + return Long.toString(getTimeVar(conf, var, timeUnit)) + timeAbbreviationFor(timeUnit); } else { - throw new RuntimeException("Invalid type for getObject " + var.defaultVal.getClass().getName()); + throw new RuntimeException("Unknown type for getObject " + var.defaultVal.getClass().getName()); } } @@ -1356,6 +1366,45 @@ public class MetastoreConf { } /** + * Dump the configuration file to the log. It will be dumped at an INFO level. This can + * potentially produce a lot of logs, so you might want to be careful when and where you do it. + * It takes care not to dump hidden keys. + * @param conf Configuration file to dump + * @return String containing dumped config file. + */ + public static String dumpConfig(Configuration conf) { + StringBuilder buf = new StringBuilder("MetastoreConf object:\n"); + if (hiveSiteURL != null) { + buf.append("Used hive-site file: ") + .append(hiveSiteURL) + .append('\n'); + } + if (hiveMetastoreSiteURL != null) { + buf.append("Used hivemetastore-site file: ") + .append(hiveMetastoreSiteURL) + .append('\n'); + } + if (metastoreSiteURL != null) { + buf.append("Used metastore-site file: ") + .append(metastoreSiteURL) + .append('\n'); + } + for (ConfVars var : ConfVars.values()) { + if (!unprintables.contains(var.varname)) { + buf.append("Key: <") + .append(var.varname) + .append("> old hive key: <") + .append(var.hiveName) + .append("> value: <") + .append(getAsString(conf, var)) + .append(">\n"); + } + } + buf.append("Finished MetastoreConf object.\n"); + return buf.toString(); + } + + /** * validate value for a ConfVar, return non-null string for fail message */ public interface Validator { http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java new file mode 100644 index 0000000..6a2f770 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.jolbox.bonecp.BoneCPConfig; +import com.jolbox.bonecp.BoneCPDataSource; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.Properties; + +/** + * DataSourceProvider for the BoneCP connection pool. + */ +public class BoneCPDataSourceProvider implements DataSourceProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class); + + public static final String BONECP = "bonecp"; + private static final String CONNECTION_TIMEOUT_PROPERTY= "bonecp.connectionTimeoutInMs"; + private static final String PARTITION_COUNT_PROPERTY= "bonecp.partitionCount"; + + @Override + public DataSource create(Configuration hdpConfig) throws SQLException { + + LOG.debug("Creating BoneCP connection pool for the MetaStore"); + + String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig); + String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig); + String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig); + int maxPoolSize = MetastoreConf.getIntVar(hdpConfig, + MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS); + + Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP); + long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L); + String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1"); + + BoneCPConfig config = null; + try { + config = new BoneCPConfig(properties); + } catch (Exception e) { + throw new SQLException("Cannot create BoneCP configuration: ", e); + } + config.setJdbcUrl(driverUrl); + //if we are waiting for connection for a long time, something is really wrong + //better raise an error than hang forever + //see DefaultConnectionStrategy.getConnectionInternal() + config.setConnectionTimeoutInMs(connectionTimeout); + config.setMaxConnectionsPerPartition(maxPoolSize); + config.setPartitionCount(Integer.parseInt(partitionCount)); + config.setUser(user); + config.setPassword(passwd); + return new BoneCPDataSource(config); + } + + @Override + public boolean mayReturnClosedConnection() { + // See HIVE-11915 for details + return true; + } + + @Override + public boolean supports(Configuration configuration) { + String poolingType = + MetastoreConf.getVar(configuration, + MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE).toLowerCase(); + if (BONECP.equals(poolingType)) { + int boneCpPropsNr = DataSourceProvider.getPrefixedProperties(configuration, BONECP).size(); + LOG.debug("Found " + boneCpPropsNr + " nr. of bonecp specific configurations"); + return boneCpPropsNr > 0; + } + LOG.debug("Configuration requested " + poolingType + " pooling, BoneCpDSProvider exiting"); + return false; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java new file mode 100644 index 0000000..17ff8d1 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.google.common.collect.Iterables; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + +import javax.sql.DataSource; +import java.io.IOException; +import java.sql.SQLException; +import java.util.Properties; + +public interface DataSourceProvider { + + /** + * @param hdpConfig + * @return the new connection pool + */ + DataSource create(Configuration hdpConfig) throws SQLException; + + /** + * BoneCp has a bug which causes closed connections to be returned to the pool + * under certain conditions. (HIVE-11915) + * @return true if the factory creates BoneCp pools which need "special attention" + */ + boolean mayReturnClosedConnection(); + + /** + * @param configuration Hadoop configuration object + * @return factory able to create a connection pool for the implementation + * specified in the configuration + */ + boolean supports(Configuration configuration); + + /** + * @param hdpConfig + * @return subset of properties prefixed by a connection pool specific substring + */ + static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) { + Properties dataSourceProps = new Properties(); + Iterables.filter( + hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix))) + .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue())); + return dataSourceProps; + } + + static String getMetastoreJdbcUser(Configuration conf) { + return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME); + } + + static String getMetastoreJdbcPasswd(Configuration conf) throws SQLException { + try { + return MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD); + } catch (IOException err) { + throw new SQLException("Error getting metastore password", err); + } + } + + static String getMetastoreJdbcDriverUrl(Configuration conf) throws SQLException { + return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTURLKEY); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java new file mode 100644 index 0000000..e3c18e3 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.conf.Configuration; + +/** + * Create a DataSourceProvider for a connectionPool configured in a hadoop + * Configuration object. + */ +public abstract class DataSourceProviderFactory { + + private static final ImmutableList<DataSourceProvider> FACTORIES = + ImmutableList.<DataSourceProvider>builder().add(new HikariCPDataSourceProvider(), new BoneCPDataSourceProvider()).build(); + + /** + * @param hdpConfig hadoop configuration + * @return factory for the configured datanucleus.connectionPoolingType + */ + public static DataSourceProvider getDataSourceProvider(Configuration hdpConfig) { + + for (DataSourceProvider factory : FACTORIES) { + + if (factory.supports(hdpConfig)) { + return factory; + } + } + return null; + } + +}
