Repository: lens Updated Branches: refs/heads/master 18999b974 -> 0455a5a3d
LENS-1141 : Add user cost ceiling tests to regression Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0455a5a3 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0455a5a3 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0455a5a3 Branch: refs/heads/master Commit: 0455a5a3dff539b01f09b58b72e2243b307546e5 Parents: 18999b9 Author: Archana H <[email protected]> Authored: Wed May 18 17:42:34 2016 +0530 Committer: Deepak Kumar Barr <[email protected]> Committed: Wed May 18 17:42:34 2016 +0530 ---------------------------------------------------------------------- lens-regression/pom.xml | 18 + .../regression/core/constants/DriverConfig.java | 8 + .../src/main/resources/queries.properties | 27 ++ .../lens/regression/ITStreamingTests.java | 3 +- .../regression/config/ITServerConfigTests.java | 31 +- .../regression/config/ITSessionConfigTests.java | 2 + .../lens/regression/throttling/ITCostTests.java | 465 +++++++++++++++++++ 7 files changed, 534 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/pom.xml ---------------------------------------------------------------------- diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml index b9841e8..b9bee97 100644 --- a/lens-regression/pom.xml +++ b/lens-regression/pom.xml @@ -45,6 +45,19 @@ </dependency> <dependency> <groupId>org.apache.lens</groupId> + <artifactId>lens-server</artifactId> + <version>${project.version}</version> + <classifier>classes</classifier> + </dependency> + <dependency> + <groupId>org.apache.lens</groupId> + <artifactId>lens-server</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.lens</groupId> <artifactId>lens-server-api</artifactId> <version>${project.version}</version> </dependency> @@ -54,6 +67,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.lens</groupId> + <artifactId>lens-driver-hive</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>com.jcraft</groupId> <artifactId>jsch</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java index c0fc15a..d80fba9 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java @@ -19,8 +19,10 @@ package org.apache.lens.regression.core.constants; +import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.driver.jdbc.JDBCDriverConfConstants; import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory; +import org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory; public class DriverConfig { @@ -36,5 +38,11 @@ public class DriverConfig { MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY; public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties. JDBC_POOL_MAX_SIZE.getConfigKey(); + public static final String HIVE_CONSTRAINT_FACTORIES = HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY; + + + public static final String MAX_CONCURRENT_CONSTRAINT_FACTORY = MaxConcurrentDriverQueriesConstraintFactory + .class.getName(); + public static final String USER_COST_CONSTRAINT_FACTORY = TotalQueryCostCeilingConstraintFactory.class.getName(); } http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/main/resources/queries.properties ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/resources/queries.properties b/lens-regression/src/main/resources/queries.properties new file mode 100644 index 0000000..033c504 --- /dev/null +++ b/lens-regression/src/main/resources/queries.properties @@ -0,0 +1,27 @@ + + +HIVE.COST_95=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-01', '2014-09-04') +HIVE.COST_60=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-01', '2014-07-31') +HIVE.COST_30=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-01', '2014-06-31') +HIVE.COST_20=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-20', '2014-07-10') +HIVE.COST_10=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-20', '2014-06-30') +HIVE.COST_5=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-21', '2014-06-26') +HIVE.COST_3=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-23', '2014-06-26') +HIVE.COST_2=cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24', '2014-06-26') + + +HIVE.SLEEP_COST_95=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-01', '2014-09-04') +HIVE.SLEEP_COST_60=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-01', '2014-07-31') +HIVE.SLEEP_COST_30=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-01', '2014-06-31') +HIVE.SLEEP_COST_20=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-20', '2014-07-10') +HIVE.SLEEP_COST_10=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-20', '2014-06-30') +HIVE.SLEEP_COST_5=cube select sample_dim_chain.name, sleepTime(measure4,%s) from sample_cube where time_range_in(dt, '2014-06-21', '2014-06-26') + + +JDBC.QUERY1 = cube select product_id from sales where time_range_in(delivery_time,'2015-04-12','2015-04-13') +JDBC.QUERY2 = cube select product_id from sales where time_range_in(delivery_time,'2015-04-12','2015-04-13') +JDBC.QUERY3 = cube select product_id from sales where time_range_in(delivery_time,'2015-04-12','2015-04-13') +JDBC.QUERY4 = cube select product_id from sales where time_range_in(delivery_time,'2015-04-12','2015-04-13') + + + http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java index d942e3c..d10f011 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/ITStreamingTests.java @@ -132,7 +132,7 @@ public class ITStreamingTests extends BaseTestClass { sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout( - QueryInventory.getSleepQuery("10"), "1000").getData(); + QueryInventory.getSleepQuery("10"), "100").getData(); InMemoryQueryResult inmemoryResult = (InMemoryQueryResult) qhr.getResult(); Assert.assertNull(inmemoryResult); @@ -162,5 +162,6 @@ public class ITStreamingTests extends BaseTestClass { PersistentQueryResult result = (PersistentQueryResult) qhr.getResult(); Assert.assertNotNull(result); Assert.assertEquals(result.getNumRows().intValue(), 8); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java index f326a4f..48ea8ec 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/config/ITServerConfigTests.java @@ -202,28 +202,21 @@ public class ITServerConfigTests extends BaseTestClass { @Test(enabled = true) public void negativeTestSnapshotInterval() throws Exception { - String sessionHandle = null; - try { - HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS, - "50000"); - Util.changeConfig(map, confFilePath); - lens.restart(); - - sessionHandle = sHelper.openNewSession("user", "pass"); - sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS, + "50000"); + Util.changeConfig(map, confFilePath); + lens.restart(); - Util.runRemoteCommand(lensKillCmd); - lens.restart(); + String sessionHandle = sHelper.openNewSession("user", "pass"); + sHelper.setAndValidateParam(sessionHandle, LensConfConstants.SESSION_CLUSTER_USER, "test"); - MapBuilder query = new MapBuilder("sessionid", sessionHandle, "key", LensConfConstants.SESSION_CLUSTER_USER); - Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); - Assert.assertEquals(response.getStatus(), Response.Status.GONE.getStatusCode(), "Snapshot interval test failed"); + //killing so that lens is not stopped gracefully. + Util.runRemoteCommand(lensKillCmd); + lens.restart(); - } finally { - if (sessionHandle != null) { - sHelper.closeNewSession(sessionHandle); - } - } + MapBuilder query = new MapBuilder("sessionid", sessionHandle, "key", LensConfConstants.SESSION_CLUSTER_USER); + Response response = lens.sendQuery("get", SessionURL.SESSION_PARAMS_URL, query); + Assert.assertEquals(response.getStatus(), Response.Status.GONE.getStatusCode(), "Snapshot interval test failed"); } http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/test/java/org/apache/lens/regression/config/ITSessionConfigTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/config/ITSessionConfigTests.java b/lens-regression/src/test/java/org/apache/lens/regression/config/ITSessionConfigTests.java index d3b8ffb..bf9de6c 100644 --- a/lens-regression/src/test/java/org/apache/lens/regression/config/ITSessionConfigTests.java +++ b/lens-regression/src/test/java/org/apache/lens/regression/config/ITSessionConfigTests.java @@ -76,6 +76,8 @@ public class ITSessionConfigTests extends BaseTestClass{ @AfterMethod(alwaysRun=true) public void restoreConfig() throws JSchException, IOException, JAXBException, LensException{ + qHelper.killQuery(null, "QUEUED", "all"); + qHelper.killQuery(null, "RUNNING", "all"); sHelper.closeSession(); } http://git-wip-us.apache.org/repos/asf/lens/blob/0455a5a3/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java new file mode 100644 index 0000000..4da4c3a --- /dev/null +++ b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java @@ -0,0 +1,465 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.regression.throttling; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import javax.ws.rs.client.WebTarget; + +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryPlan; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.cube.parse.CubeQueryConfUtil; +import org.apache.lens.regression.core.constants.DriverConfig; +import org.apache.lens.regression.core.constants.QueryInventory; +import org.apache.lens.regression.core.helpers.*; +import org.apache.lens.regression.core.testHelper.BaseTestClass; +import org.apache.lens.regression.util.Util; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; + +import org.apache.log4j.Logger; + +import org.testng.Assert; +import org.testng.annotations.*; + +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.SftpException; + +public class ITCostTests extends BaseTestClass { + + private WebTarget servLens; + private String sessionHandleString; + + LensServerHelper lens = getLensServerHelper(); + MetastoreHelper mHelper = getMetastoreHelper(); + SessionHelper sHelper = getSessionHelper(); + QueryHelper qHelper = getQueryHelper(); + + public static final String COST_95 = QueryInventory.getQueryFromInventory("HIVE.COST_95"); + public static final String COST_60 = QueryInventory.getQueryFromInventory("HIVE.COST_60"); + public static final String COST_30 = QueryInventory.getQueryFromInventory("HIVE.COST_30"); + public static final String COST_20 = QueryInventory.getQueryFromInventory("HIVE.COST_20"); + public static final String COST_10 = QueryInventory.getQueryFromInventory("HIVE.COST_10"); + public static final String COST_5 = QueryInventory.getQueryFromInventory("HIVE.COST_5"); + public static final String COST_3 = QueryInventory.getQueryFromInventory("HIVE.COST_3"); + public static final String COST_2 = QueryInventory.getQueryFromInventory("HIVE.COST_2"); + public static final String JDBC_QUERY1 = QueryInventory.getQueryFromInventory("JDBC.QUERY1"); + + private static String hiveDriver = "hive/hive1"; + private String hiveDriverSitePath = lens.getServerDir() + "/conf/drivers/hive/hive1/hivedriver-site.xml"; + private static final long SECONDS_IN_A_MINUTE = 60; + + private static Logger logger = Logger.getLogger(ITCostTests.class); + + @BeforeClass(alwaysRun = true) + public void initialize() throws Exception { + servLens = ServiceManagerHelper.init(); + logger.info("Creating a new Session"); + sessionHandleString = lens.openSession(lens.getCurrentDB()); + sHelper.setAndValidateParam(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + } + + @BeforeMethod(alwaysRun = true) + public void setUp(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + } + + @AfterMethod(alwaysRun = true) + public void afterMethod(Method method) throws Exception { + logger.info("Test Name: " + method.getName()); + qHelper.killQuery(null, "QUEUED", "all"); + qHelper.killQuery(null, "RUNNING", "all"); + qHelper.killQuery(null, "EXECUTED", "all"); + } + + @AfterClass(alwaysRun = true) + public void closeSession() throws Exception { + logger.info("Closing Session"); + lens.closeSession(); + } + + @BeforeGroups("user-cost-ceiling") + public void setUserCeilingconfig() throws Exception { + try{ + HashMap<String, String> map = LensUtil.getHashMap(LensConfConstants.TOTAL_QUERY_COST_CEILING_PER_USER_KEY, "60", + DriverConfig.HIVE_CONSTRAINT_FACTORIES, + DriverConfig.MAX_CONCURRENT_CONSTRAINT_FACTORY + "," + DriverConfig.USER_COST_CONSTRAINT_FACTORY, + DriverConfig.MAX_CONCURRENT_QUERIES, "10"); + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + }catch (Exception e){ + logger.info(e); + } + } + + @AfterGroups("user-cost-ceiling") + public void restoreConfig() throws SftpException, JSchException, InterruptedException, LensException, IOException { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + + + @Test(enabled = true, groups= "user-cost-ceiling") + public void testUserCostCeiling() throws Exception { + + QueryHandle h1 = (QueryHandle) qHelper.executeQuery(COST_60).getData(); + QueryHandle h2 = (QueryHandle) qHelper.executeQuery(COST_95).getData(); + + Assert.assertEquals(qHelper.getQueryStatus(h1).getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(qHelper.getQueryStatus(h2).getStatus(), QueryStatus.Status.QUEUED); + + LensQuery lq1 = qHelper.waitForCompletion(h1); + LensQuery lq2 = qHelper.waitForCompletion(h2); + + Assert.assertEquals(lq1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + Assert.assertEquals(lq2.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + @Test(enabled = true, groups= "user-cost-ceiling") + public void hiveJdbcUserCostCeiling() throws Exception { + + List<QueryStatus.Status> passStatus = Arrays.asList(QueryStatus.Status.SUCCESSFUL, QueryStatus.Status.RUNNING, + QueryStatus.Status.EXECUTED); + + QueryHandle h1 = (QueryHandle) qHelper.executeQuery(COST_10).getData(); + QueryHandle jq1 = (QueryHandle) qHelper.executeQuery(JDBC_QUERY1).getData(); + QueryHandle h2 = (QueryHandle) qHelper.executeQuery(COST_60).getData(); + QueryHandle jq2 = (QueryHandle) qHelper.executeQuery(JDBC_QUERY1).getData(); + QueryHandle h3 = (QueryHandle) qHelper.executeQuery(COST_30).getData(); + + Assert.assertEquals(qHelper.getQueryStatus(h1).getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(qHelper.getQueryStatus(h2).getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(qHelper.getQueryStatus(h3).getStatus(), QueryStatus.Status.QUEUED); + Assert.assertTrue(passStatus.contains(qHelper.getQueryStatus(jq1).getStatus())); + Assert.assertTrue(passStatus.contains(qHelper.getQueryStatus(jq2).getStatus())); + + LensQuery lq1 = qHelper.waitForCompletion(h1); + LensQuery lq2 = qHelper.waitForCompletion(h2); + LensQuery lq3 = qHelper.waitForCompletion(h3); + + Assert.assertEquals(lq1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + Assert.assertEquals(lq2.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + Assert.assertEquals(lq3.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + @Test(enabled = true, groups= "user-cost-ceiling") + public void runMultipleQueriesCostGreaterThanCeiling() throws Exception { + + int userCeiling = 60; + Map<QueryHandle, Double> queryhandleCostMap = new HashMap<QueryHandle, Double>(); + QueryPlan qp = null; + + String[] queries = {COST_20, JDBC_QUERY1, COST_95, JDBC_QUERY1, COST_5, COST_10, JDBC_QUERY1, + COST_30, COST_60, }; + + for (int i = 1; i < 3; i++) { + for(int j = 0; j < queries.length; j++) { + qp = (QueryPlan) qHelper.explainQuery(queries[j]).getData(); + queryhandleCostMap.put((QueryHandle) qHelper.executeQuery(queries[j]).getData(), + qp.getQueryCost().getEstimatedResourceUsage()); + } + } + + List<QueryHandle> running = qHelper.getQueryHandleList(null, "RUNNING", "all"); + List<QueryHandle> queued = qHelper.getQueryHandleList(null, "QUEUED", "all"); + + while (running.size() > 0 || queued.size() > 0) { + if (running.size() > 1) { + Double costSum = 0.0, maxcost = 0.0; + for (QueryHandle qH : running) { + Double currentHandleCost = queryhandleCostMap.get(qH); + costSum += currentHandleCost; + if (currentHandleCost > maxcost) { + maxcost = currentHandleCost; + } + } + + logger.info("max-cost : " + maxcost + " cost-sum : " + costSum); + Assert.assertTrue((costSum - maxcost) < userCeiling); + } + + Thread.sleep(1000); + running = qHelper.getQueryHandleList(null, "RUNNING", "all"); + queued = qHelper.getQueryHandleList(null, "QUEUED", "all"); + } + + for (QueryHandle q : queryhandleCostMap.keySet()) { + LensQuery lq = qHelper.waitForCompletion(q); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + } + + + + @Test(enabled = true, groups= "user-cost-ceiling") + public void testCostCeilingWithProrityMaxConcurrent() throws Exception { + + String query = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_5"), "10"); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "5", + DriverConfig.PRIORITY_MAX_CONCURRENT, "HIGH=3"); + + try { + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + for(int i=1; i<=5; i++){ + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + } + + QueryStatus s0 = qHelper.getQueryStatus(handleList.get(0)); + QueryStatus s1 = qHelper.getQueryStatus(handleList.get(1)); + QueryStatus s2 = qHelper.getQueryStatus(handleList.get(2)); + QueryStatus s3 = qHelper.getQueryStatus(handleList.get(3)); + QueryStatus s4 = qHelper.getQueryStatus(handleList.get(4)); + + Assert.assertEquals(s0.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s1.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s2.getStatus(), QueryStatus.Status.RUNNING); + Assert.assertEquals(s3.getStatus(), QueryStatus.Status.QUEUED); + Assert.assertEquals(s4.getStatus(), QueryStatus.Status.QUEUED); + + qHelper.waitForCompletion(handleList.get(0)); + Assert.assertEquals(qHelper.getQueryStatus(handleList.get(3)).getStatus(), QueryStatus.Status.RUNNING); + + qHelper.waitForCompletion(handleList.get(1)); + Assert.assertEquals(qHelper.getQueryStatus(handleList.get(4)).getStatus(), QueryStatus.Status.RUNNING); + + for(QueryHandle q :handleList){ + LensQuery lq = qHelper.waitForCompletion(q); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + } finally { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + } + + + @Test(enabled = true, groups= "user-cost-ceiling") + public void multipleUserConcurrentPriorityThrottling() throws Exception { + + String query = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_5"), "10"); + long timeToWait= 7 * SECONDS_IN_A_MINUTE; //in seconds + int sleepTime = 5; //in seconds + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "5", + DriverConfig.PRIORITY_MAX_CONCURRENT, "HIGH=3"); + + try { + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + + String session1 = sHelper.openNewSession("diff1", "diff1", lens.getCurrentDB()); + String session2 = sHelper.openNewSession("diff2", "diff2", lens.getCurrentDB()); + sHelper.setAndValidateParam(session1, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + sHelper.setAndValidateParam(session2, CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); + + List<QueryHandle> handleList = new ArrayList<QueryHandle>(); + for (int i = 1; i <= 3; i++) { + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + handleList.add((QueryHandle) qHelper.executeQuery(query, "", session1).getData()); + handleList.add((QueryHandle) qHelper.executeQuery(query, "", session2).getData()); + } + Thread.sleep(50); + + List<QueryHandle> running = null, queued = null; + for (int t = 0; t < timeToWait; t = t + sleepTime) { + + running = qHelper.getQueryHandleList(null, "RUNNING", "all", sessionHandleString, null, null, + hiveDriver); + queued = qHelper.getQueryHandleList(null, "QUEUED", "all", sessionHandleString, null, null, + hiveDriver); + logger.info("Running query count : " + running.size() + "\t Queued query count : " + queued.size()); + + if (running.isEmpty() && queued.isEmpty()) { + break; + } + + Assert.assertTrue(running.size() < 4); + TimeUnit.SECONDS.sleep(sleepTime); + } + + Assert.assertTrue(running.isEmpty() && queued.isEmpty(), "Queries are taking very long time"); + + for (QueryHandle q : handleList) { + LensQuery lq = qHelper.waitForCompletion(q); + Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + } finally { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + } + + //TODO : Add queue level throttling along with user ceiling constraint + + /* + * Queue number shouldn't change with in the same prority + */ + + @Test(enabled = true) + public void queueNumberChangeWithInSamePriority() throws Exception { + + String longRunningQuery = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_95"), "20"); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1"); + String[] queries = {longRunningQuery, COST_5, COST_5, COST_3, COST_2}; + + try { + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + + List<QueryHandle> handleList = new ArrayList<>(); + for(String query : queries){ + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + } + + LensQuery lq1 = qHelper.getLensQuery(sessionHandleString, handleList.get(1)); + LensQuery lq2 = qHelper.getLensQuery(sessionHandleString, handleList.get(2)); + LensQuery lq3 = qHelper.getLensQuery(sessionHandleString, handleList.get(3)); + LensQuery lq4 = qHelper.getLensQuery(sessionHandleString, handleList.get(4)); + + Assert.assertEquals(lq1.getStatus().getQueueNumber().intValue(), 1); + Assert.assertEquals(lq2.getStatus().getQueueNumber().intValue(), 2); + Assert.assertEquals(lq3.getStatus().getQueueNumber().intValue(), 3); + Assert.assertEquals(lq4.getStatus().getQueueNumber().intValue(), 4); + + LensQuery lq0 = qHelper.waitForCompletion(handleList.get(0)); + + lq1 = qHelper.getLensQuery(sessionHandleString, handleList.get(1)); + lq2 = qHelper.getLensQuery(sessionHandleString, handleList.get(2)); + lq3 = qHelper.getLensQuery(sessionHandleString, handleList.get(3)); + lq4 = qHelper.getLensQuery(sessionHandleString, handleList.get(4)); + + Assert.assertEquals(lq0.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + Assert.assertEquals(lq2.getStatus().getQueueNumber().intValue(), 1); + Assert.assertEquals(lq3.getStatus().getQueueNumber().intValue(), 2); + Assert.assertEquals(lq4.getStatus().getQueueNumber().intValue(), 3); + + lq1 = qHelper.waitForCompletion(handleList.get(1)); + + lq2 = qHelper.getLensQuery(sessionHandleString, handleList.get(2)); + lq3 = qHelper.getLensQuery(sessionHandleString, handleList.get(3)); + lq4 = qHelper.getLensQuery(sessionHandleString, handleList.get(4)); + + Assert.assertEquals(lq1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + Assert.assertEquals(lq3.getStatus().getQueueNumber().intValue(), 1); + Assert.assertEquals(lq4.getStatus().getQueueNumber().intValue(), 2); + + }finally { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + } + + + @Test(enabled = true) + public void queueNumberChangeDifferentPriority() throws Exception { + + String longRunningQuery = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_95"), "20"); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1"); + try { + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + + QueryHandle q0 = (QueryHandle) qHelper.executeQuery(longRunningQuery).getData(); + QueryHandle q1 = (QueryHandle) qHelper.executeQuery(COST_20).getData(); + QueryHandle q2 = (QueryHandle) qHelper.executeQuery(COST_2).getData(); + + LensQuery normal1 = qHelper.getLensQuery(sessionHandleString, q1); + LensQuery high1 = qHelper.getLensQuery(sessionHandleString, q2); + + Assert.assertEquals(normal1.getStatus().getQueueNumber().intValue(), 2); + Assert.assertEquals(high1.getStatus().getQueueNumber().intValue(), 1); + + QueryHandle q3 = (QueryHandle) qHelper.executeQuery(COST_5).getData(); + + LensQuery high2 = qHelper.getLensQuery(sessionHandleString, q3); + high1 = qHelper.getLensQuery(sessionHandleString, q2); + normal1 = qHelper.getLensQuery(sessionHandleString, q1); + + Assert.assertEquals(normal1.getStatus().getQueueNumber().intValue(), 3); + Assert.assertEquals(high1.getStatus().getQueueNumber().intValue(), 1); + Assert.assertEquals(high2.getStatus().getQueueNumber().intValue(), 2); + + QueryHandle q4 = (QueryHandle) qHelper.executeQuery(COST_20).getData(); + + LensQuery normal2 = qHelper.getLensQuery(sessionHandleString, q4); + normal1 = qHelper.getLensQuery(sessionHandleString, q1); + high1 = qHelper.getLensQuery(sessionHandleString, q2); + high2 = qHelper.getLensQuery(sessionHandleString, q3); + + Assert.assertEquals(high1.getStatus().getQueueNumber().intValue(), 1); + Assert.assertEquals(high2.getStatus().getQueueNumber().intValue(), 2); + Assert.assertEquals(normal1.getStatus().getQueueNumber().intValue(), 3); + Assert.assertEquals(normal2.getStatus().getQueueNumber().intValue(), 4); + + }finally { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + } + + + @Test(enabled = true) + public void queueNumberChangeDifferentPriorityWithJdbc() throws Exception { + + String longRunningQuery = String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_95"), "20"); + HashMap<String, String> map = LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1"); + List<QueryHandle> handleList = new ArrayList<>(); + + try { + Util.changeConfig(map, hiveDriverSitePath); + lens.restart(); + + String[] queries = {COST_20, COST_2, COST_3, COST_60, COST_5, COST_10, COST_3}; + // Queue order is determined from priority and order in which queries are fired. + int[] queueNo = {5, 1, 2, 7, 3, 6, 4}; + + qHelper.executeQuery(longRunningQuery); + for(String query : queries){ + handleList.add((QueryHandle) qHelper.executeQuery(query).getData()); + qHelper.executeQuery(JDBC_QUERY1).getData(); + } + + List<LensQuery> lqList = new ArrayList<>(); + for(QueryHandle qh : handleList){ + lqList.add(qHelper.getLensQuery(sessionHandleString, qh)); + } + + for(int i = 0; i < lqList.size(); i++) { + Assert.assertEquals(lqList.get(i).getStatus().getQueueNumber().intValue(), queueNo[i]); + } + + }finally { + Util.changeConfig(hiveDriverSitePath); + lens.restart(); + } + } + +} +
