Repository: lens Updated Branches: refs/heads/master dce5a812d -> 34223e880
LENS-1027: Extend MaxConcurrentDriverQueriesConstraint to support limit on ANY queue Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/34223e88 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/34223e88 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/34223e88 Branch: refs/heads/master Commit: 34223e880df7f08912a519349ea6053932f014ea Parents: dce5a81 Author: Puneet Gupta <[email protected]> Authored: Tue May 3 12:15:41 2016 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Tue May 3 12:15:41 2016 +0530 ---------------------------------------------------------------------- .../MaxConcurrentDriverQueriesConstraint.java | 7 ++++++- .../MaxConcurrentDriverQueriesConstraintFactory.java | 4 +++- .../MaxConcurrentDriverQueriesConstraintTest.java | 13 ++++++++----- .../ThreadSafeEstimatedQueryCollectionTest.java | 3 ++- 4 files changed, 19 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java index 58ebd9a..b2319a9 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java @@ -36,6 +36,7 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst private final int maxConcurrentQueries; private final Map<String, Integer> maxConcurrentQueriesPerQueue; private final Map<Priority, Integer> maxConcurrentQueriesPerPriority; + private final Integer defaultMaxConcurrentQueriesPerQueueLimit; @Override public boolean allowsLaunchOf( @@ -57,7 +58,11 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst String queue = candidateQuery.getQueue(); Integer limit = maxConcurrentQueriesPerQueue.get(queue); if (limit == null) { - return true; + if (defaultMaxConcurrentQueriesPerQueueLimit != null) { //Check if any default limit is enabled for all queues + limit = defaultMaxConcurrentQueriesPerQueueLimit; + } else { + return true; + } } int launchedOnQueue = 0; for (QueryContext context : launchedQueries.getQueries(candidateQuery.getSelectedDriver())) { http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java index 6db7da7..442cd99 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java @@ -36,6 +36,7 @@ public class MaxConcurrentDriverQueriesConstraintFactory public static final String MAX_CONCURRENT_QUERIES_KEY = "driver.max.concurrent.launched.queries"; private static final String PREFIX = MAX_CONCURRENT_QUERIES_KEY + ".per."; public static final String MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY = PREFIX + "queue"; + public static final String DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY = "*"; public static final String MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY = PREFIX + "priority"; private static final EntryParser<String, Integer> STRING_INT_PARSER = new EntryParser<String, Integer>() { @Override @@ -72,7 +73,8 @@ public class MaxConcurrentDriverQueriesConstraintFactory maxConcurrentQueries = Integer.parseInt(maxConcurrentQueriesValue); } return new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, maxConcurrentQueriesPerQueue, - maxConcurrentQueriesPerPriority); + maxConcurrentQueriesPerPriority, + maxConcurrentQueriesPerQueue.get(DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY)); } } http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java index 4031122..38b74ae 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java @@ -46,9 +46,10 @@ public class MaxConcurrentDriverQueriesConstraintTest { QueryLaunchingConstraint constraint = factory.create(getConfiguration( "driver.max.concurrent.launched.queries", 10 )); + QueryLaunchingConstraint perQueueConstraint = factory.create(getConfiguration( "driver.max.concurrent.launched.queries", 4, - "driver.max.concurrent.launched.queries.per.queue", "q1=2,q2=3" + "driver.max.concurrent.launched.queries.per.queue", "*=1,q1=2,q2=3" )); QueryLaunchingConstraint perPriorityConstraint = factory.create(getConfiguration( @@ -73,12 +74,14 @@ public class MaxConcurrentDriverQueriesConstraintTest { {queues("q1", "q1"), "q2", true}, {queues("q1", "q1"), "q3", true}, {queues("q1", "q1", "q1"), "q2", true}, // hypothetical - {queues("q1", "q1", "q2"), "q1", false}, + {queues("q1", "q1", "q2"), "q1", false}, //q1 limit breached {queues("q1", "q2", "q2"), "q1", true}, {queues("q1", "q2", "q2"), "q2", true}, - {queues("q1", "q2", "q1", "q2"), "q2", false}, - {queues("q1", "q2", "q1", "q2"), "q1", false}, - {queues("q1", "q2", "q1", "q2"), "q3", false}, + {queues("q1", "q2", "q1", "q2"), "q2", false}, // driver.max.concurrent.launched.queries breached + {queues("q1", "q2", "q1", "q2"), "q1", false}, // driver.max.concurrent.launched.queries breached + {queues("q1", "q2", "q1", "q2"), "q3", false}, // driver.max.concurrent.launched.queries breached + {queues("q1", "q2", "q2"), "q3", true}, + {queues("q1", "q2", "q3"), "q3", false}, //default max concurrent queries per queue limit breached }; } http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java index e1bf350..c8ebd0c 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java @@ -52,7 +52,8 @@ public class ThreadSafeEstimatedQueryCollectionTest { LensDriver mockDriver = mock(LensDriver.class); LensDriver mockDriver2 = mock(LensDriver.class); - QueryLaunchingConstraint constraint = new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, null, null); + QueryLaunchingConstraint constraint = new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, null, + null, null); ThreadSafeEstimatedQueryCollection col = new ThreadSafeEstimatedQueryCollection(new DefaultEstimatedQueryCollection(new DefaultQueryCollection()));
