Repository: lens Updated Branches: refs/heads/master 7707a25b1 -> 776a2c331
LENS-1168 : Add query launcher pool to launch queries in parallel Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/776a2c33 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/776a2c33 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/776a2c33 Branch: refs/heads/master Commit: 776a2c331e8e43e11b188046a1369b1d640355eb Parents: 7707a25 Author: Amareshwari Sriramadasu <[email protected]> Authored: Mon Jul 25 11:26:38 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Mon Jul 25 11:26:38 2016 +0530 ---------------------------------------------------------------------- .../lens/server/api/LensConfConstants.java | 20 ++ .../lens/server/api/metrics/MetricsService.java | 2 + .../lens/server/api/query/QueryContext.java | 8 + .../server/api/query/QueryExecutionService.java | 7 + .../MaxConcurrentDriverQueriesConstraint.java | 28 ++- ...oncurrentDriverQueriesConstraintFactory.java | 16 +- ...axConcurrentDriverQueriesConstraintTest.java | 32 ++- .../lens/server/metrics/MetricsServiceImpl.java | 11 ++ .../server/query/QueryExecutionServiceImpl.java | 196 ++++++++++++++----- .../lens/server/session/LensSessionImpl.java | 12 +- .../src/main/resources/lensserver-default.xml | 22 +++ .../org/apache/lens/server/LensJerseyTest.java | 2 +- .../lens/server/common/FailingQueryDriver.java | 9 + .../lens/server/query/TestQueryConstraints.java | 19 +- .../lens/server/query/TestQueryService.java | 24 +++ .../ThreadSafeEstimatedQueryCollectionTest.java | 2 +- lens-server/src/test/resources/logback.xml | 2 +- src/site/apt/admin/config.apt | 174 ++++++++-------- 18 files changed, 416 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 9f51565..43bd7e4 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -886,6 +886,26 @@ public final class LensConfConstants { public static final String ESTIMATE_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX + "estimate.pool.keepalive.millis"; public static final int DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS = 60000; // 1 minute + /** + * Key used to get minimum number of threads in the launcher thread pool + */ + public static final String LAUNCHER_POOL_MIN_THREADS = SERVER_PFX + "launcher.pool.min.threads"; + public static final int DEFAULT_LAUNCHER_POOL_MIN_THREADS = 3; + + /** + * Key used to get maximum number of threads in the laucnher thread pool + */ + public static final String LAUNCHER_POOL_MAX_THREADS = SERVER_PFX + "launcher.pool.max.threads"; + // keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on + // all drivers together. + public static final int DEFAULT_LAUNCHER_POOL_MAX_THREADS = 100; + + /** + * Key used to get keep alive time for threads in the launcher thread pool + */ + public static final String LAUNCHER_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX + "launcher.pool.keepalive.millis"; + public static final int DEFAULT_LAUNCHER_POOL_KEEP_ALIVE_MILLIS = 60000; // 1 minute + public static final String QUERY_PHASE1_REWRITERS = SERVER_PFX + "query.phase1.rewriters"; /** http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java index 7fd2d81..35daa21 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java @@ -124,6 +124,8 @@ public interface MetricsService extends LensService { */ String WAITING_QUERIES = "waiting-queries"; + String LAUNCHING_QUERIES = "launching-queries"; + /** * The Constant FINISHED_QUERIES. */ http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 8ba0689..7f323dc 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -26,6 +26,7 @@ import static org.apache.lens.server.api.LensConfConstants.PREFETCH_INMEMORY_RES import java.util.Collection; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Future; import org.apache.lens.api.LensConf; import org.apache.lens.api.query.LensQuery; @@ -199,6 +200,13 @@ public class QueryContext extends AbstractQueryContext { transient FailureContext statusUpdateFailures = new FailureContext(); + @Getter + @Setter + private transient boolean isLaunching = false; + + @Getter + @Setter + private transient Future queryLauncher; /** * Creates context from query * http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java index d10ad09..4f4d4aa 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java @@ -317,4 +317,11 @@ public interface QueryExecutionService extends LensService, SessionValidator { * @return finished queries count */ long getFinishedQueriesCount(); + + /** + * Get queries being launched count + * + * @return Queries being launched count + */ + long getLaunchingQueriesCount(); } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java index b2319a9..e0f1376 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java @@ -20,6 +20,7 @@ package org.apache.lens.server.api.query.constraint; import java.util.Map; +import java.util.Set; import org.apache.lens.api.Priority; import org.apache.lens.server.api.driver.LensDriver; @@ -37,21 +38,23 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst private final Map<String, Integer> maxConcurrentQueriesPerQueue; private final Map<Priority, Integer> maxConcurrentQueriesPerPriority; private final Integer defaultMaxConcurrentQueriesPerQueueLimit; + private final int maxConcurrentLaunches; @Override public boolean allowsLaunchOf( final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) { final LensDriver selectedDriver = candidateQuery.getSelectedDriver(); + final Set<QueryContext> driverLaunchedQueries = launchedQueries.getQueries(selectedDriver); final boolean canLaunch = (launchedQueries.getQueriesCount(selectedDriver) < maxConcurrentQueries) - && canLaunchWithQueueConstraint(candidateQuery, launchedQueries) - && canLaunchWithPriorityConstraint(candidateQuery, launchedQueries); + && (getIsLaunchingCount(driverLaunchedQueries) < maxConcurrentLaunches) + && canLaunchWithQueueConstraint(candidateQuery, driverLaunchedQueries) + && canLaunchWithPriorityConstraint(candidateQuery, driverLaunchedQueries); log.debug("canLaunch:{}", canLaunch); return canLaunch; } - private boolean canLaunchWithQueueConstraint(QueryContext candidateQuery, EstimatedImmutableQueryCollection - launchedQueries) { + private boolean canLaunchWithQueueConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) { if (maxConcurrentQueriesPerQueue == null) { return true; } @@ -65,7 +68,7 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst } } int launchedOnQueue = 0; - for (QueryContext context : launchedQueries.getQueries(candidateQuery.getSelectedDriver())) { + for (QueryContext context : launchedQueries) { if (context.getQueue().equals(queue)) { launchedOnQueue++; } @@ -73,8 +76,7 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst return launchedOnQueue < limit; } - private boolean canLaunchWithPriorityConstraint(QueryContext candidateQuery, EstimatedImmutableQueryCollection - launchedQueries) { + private boolean canLaunchWithPriorityConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) { if (maxConcurrentQueriesPerPriority == null) { return true; } @@ -84,11 +86,21 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst return true; } int launchedOnPriority = 0; - for (QueryContext context : launchedQueries.getQueries(candidateQuery.getSelectedDriver())) { + for (QueryContext context : launchedQueries) { if (context.getPriority().equals(priority)) { launchedOnPriority++; } } return launchedOnPriority < limit; } + + private int getIsLaunchingCount(final Set<QueryContext> launchedQueries) { + int launcherCount = 0; + for (QueryContext ctx : launchedQueries) { + if (ctx.isLaunching()) { + launcherCount++; + } + } + return launcherCount; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java index 442cd99..d149622 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java @@ -27,7 +27,6 @@ import org.apache.lens.api.Priority; import org.apache.lens.api.util.CommonUtils.EntryParser; import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; public class MaxConcurrentDriverQueriesConstraintFactory @@ -38,6 +37,8 @@ public class MaxConcurrentDriverQueriesConstraintFactory public static final String MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY = PREFIX + "queue"; public static final String DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY = "*"; public static final String MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY = PREFIX + "priority"; + public static final String MAX_CONCURRENT_LAUNCHES = "driver.max.concurrent.launches"; + private static final EntryParser<String, Integer> STRING_INT_PARSER = new EntryParser<String, Integer>() { @Override public String parseKey(String str) { @@ -63,18 +64,17 @@ public class MaxConcurrentDriverQueriesConstraintFactory @Override public MaxConcurrentDriverQueriesConstraint create(final Configuration conf) { - String maxConcurrentQueriesValue = conf.get(MAX_CONCURRENT_QUERIES_KEY); + int maxConcurrentQueries = conf.getInt(MAX_CONCURRENT_QUERIES_KEY, Integer.MAX_VALUE); Map<String, Integer> maxConcurrentQueriesPerQueue = parseMapFromString( conf.get(MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY), STRING_INT_PARSER); Map<Priority, Integer> maxConcurrentQueriesPerPriority = parseMapFromString( conf.get(MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY), PRIORITY_INT_PARSER); - int maxConcurrentQueries = Integer.MAX_VALUE; - if (!StringUtils.isBlank(maxConcurrentQueriesValue)) { - maxConcurrentQueries = Integer.parseInt(maxConcurrentQueriesValue); - } - return new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, maxConcurrentQueriesPerQueue, + + return new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, + maxConcurrentQueriesPerQueue, maxConcurrentQueriesPerPriority, - maxConcurrentQueriesPerQueue.get(DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY)); + maxConcurrentQueriesPerQueue.get(DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY), + conf.getInt(MAX_CONCURRENT_LAUNCHES, maxConcurrentQueries)); } } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java index 38b74ae..122409b 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java @@ -44,7 +44,8 @@ public class MaxConcurrentDriverQueriesConstraintTest { MaxConcurrentDriverQueriesConstraintFactory factory = new MaxConcurrentDriverQueriesConstraintFactory(); QueryLaunchingConstraint constraint = factory.create(getConfiguration( - "driver.max.concurrent.launched.queries", 10 + "driver.max.concurrent.launched.queries", 10, + "driver.max.concurrent.launches", 4 )); QueryLaunchingConstraint perQueueConstraint = factory.create(getConfiguration( @@ -64,7 +65,12 @@ public class MaxConcurrentDriverQueriesConstraintTest { @DataProvider public Object[][] dpTestAllowsLaunchOfQuery() { - return new Object[][]{{2, true}, {10, false}, {11, false}}; + return new Object[][]{{2, true}, {3, true}, {4, true}, {5, true}, {10, false}, {11, false}}; + } + + @DataProvider + public Object[][] dpTestConcurrentLaunches() { + return new Object[][]{{2, true}, {3, true}, {4, false}, {5, false}, {10, false}, {11, false}}; } @DataProvider @@ -163,6 +169,28 @@ public class MaxConcurrentDriverQueriesConstraintTest { assertEquals(actualCanLaunch, expectedCanLaunch); } + @Test(dataProvider = "dpTestConcurrentLaunches") + public void testConcurrentLaunches(final int currentDriverLaunchedQueries, final boolean expectedCanLaunch) { + + QueryContext mockCandidateQuery = mock(QueryContext.class); + EstimatedImmutableQueryCollection mockLaunchedQueries = mock(EstimatedImmutableQueryCollection.class); + LensDriver mockDriver = mock(LensDriver.class); + + Set<QueryContext> queries = new HashSet<>(currentDriverLaunchedQueries); + for (int i = 0; i < currentDriverLaunchedQueries; i++) { + QueryContext mQuery = mock(QueryContext.class); + when(mQuery.isLaunching()).thenReturn(true); + queries.add(mQuery); + } + when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver); + when(mockLaunchedQueries.getQueriesCount(mockDriver)).thenReturn(currentDriverLaunchedQueries); + when(mockLaunchedQueries.getQueries(mockDriver)).thenReturn(queries); + + boolean actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries); + + assertEquals(actualCanLaunch, expectedCanLaunch); + } + @Test(dataProvider = "dpTestPerQueueConstraints") public void testPerQueueConstraints(final String[] launchedQueues, final String candidateQueue, final boolean expectedCanLaunch) { http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java index 1e8d540..f1ee169 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java @@ -139,6 +139,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic /** The waiting queries. */ private Gauge<Long> waitingQueries; + /** The queries being launched. */ + private Gauge<Long> launchingQueries; + /** The finished queries. */ private Gauge<Long> finishedQueries; @@ -358,6 +361,14 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic } }); + launchingQueries = metricRegistry.register(MetricRegistry.name(QueryExecutionService.class, LAUNCHING_QUERIES), + new Gauge<Long>() { + @Override + public Long getValue() { + return getQuerySvc().getLaunchingQueriesCount(); + } + }); + finishedQueries = metricRegistry.register(MetricRegistry.name(QueryExecutionService.class, FINISHED_QUERIES), new Gauge<Long>() { @Override http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index be0e2c8..cf0c168 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -264,6 +264,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE private ExecutorService estimatePool; /** + * Thread pool used for launching queries in parallel. + */ + private ExecutorService queryLauncherPool; + + /** * The pool used for cancelling timed out queries. */ private ExecutorService queryCancellationPool; @@ -647,17 +652,13 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE */ private boolean pausedForTest = false; - private final ErrorCollection errorCollection; - private final EstimatedQueryCollection waitingQueries; private final QueryLaunchingConstraintsChecker constraintsChecker; - public QuerySubmitter(@NonNull final ErrorCollection errorCollection, - @NonNull final EstimatedQueryCollection waitingQueries, - @NonNull final QueryLaunchingConstraintsChecker constraintsChecker) { + public QuerySubmitter(@NonNull final EstimatedQueryCollection waitingQueries, + @NonNull final QueryLaunchingConstraintsChecker constraintsChecker) { - this.errorCollection = errorCollection; this.waitingQueries = waitingQueries; this.constraintsChecker = constraintsChecker; } @@ -688,37 +689,32 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } log.info("Processing query:{}", query.getUserQuery()); + /* Check javadoc of QueryExecutionServiceImpl#removalFromLaunchedQueriesLock for reason for existence + of this lock. */ + log.debug("Acquiring lock in QuerySubmitter"); + removalFromLaunchedQueriesLock.lock(); try { - // acquire session before any query operation. - acquire(query.getLensSessionIdentifier()); - - /* Check javadoc of QueryExecutionServiceImpl#removalFromLaunchedQueriesLock for reason for existence - of this lock. */ - log.debug("Acquiring lock in QuerySubmitter"); - removalFromLaunchedQueriesLock.lock(); - try { - if (this.constraintsChecker.canLaunch(query, launchedQueries)) { - /* Query is not going to be added to waiting queries. No need to keep the lock. - First release lock, then launch query */ - removalFromLaunchedQueriesLock.unlock(); - launchQuery(query); - } else { - /* Query is going to be added to waiting queries. Keep holding the lock to avoid any removal from - launched queries. First add to waiting queries, then release lock */ - addToWaitingQueries(query); - removalFromLaunchedQueriesLock.unlock(); - } - } finally { - if (removalFromLaunchedQueriesLock.isHeldByCurrentThread()) { - removalFromLaunchedQueriesLock.unlock(); - } + if (this.constraintsChecker.canLaunch(query, launchedQueries)) { + + /* Query is not going to be added to waiting queries. No need to keep the lock. + First release lock, then launch query */ + removalFromLaunchedQueriesLock.unlock(); + // add it to launched queries data structure immediately sothat other constraint checks can start seeing + // this query + query.setLaunching(true); + launchedQueries.add(query); + Future launcherFuture = queryLauncherPool.submit(new QueryLauncher(query)); + query.setQueryLauncher(launcherFuture); + } else { + /* Query is going to be added to waiting queries. Keep holding the lock to avoid any removal from + launched queries. First add to waiting queries, then release lock */ + addToWaitingQueries(query); + removalFromLaunchedQueriesLock.unlock(); } - } catch (Exception e) { - log.error("Error launching query: {}", query.getQueryHandle(), e); - setFailedStatus(query, "Launching query failed", e); - continue; } finally { - release(query.getLensSessionIdentifier()); + if (removalFromLaunchedQueriesLock.isHeldByCurrentThread()) { + removalFromLaunchedQueriesLock.unlock(); + } } } } catch (InterruptedException e) { @@ -732,15 +728,61 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE log.info("QuerySubmitter exited"); } - private void launchQuery(final QueryContext query) throws LensException { + private void addToWaitingQueries(final QueryContext query) throws LensException { checkEstimatedQueriesState(query); + this.waitingQueries.add(query); + log.info("Added to waiting queries. QueryId:{}", query.getQueryHandleString()); + } + } + + private class QueryLauncher implements Runnable { + QueryContext query; + + QueryLauncher(QueryContext query) { + this.query = query; + } + + @Override + public void run() { + synchronized (query) { + try { + logSegregationContext.setLogSegragationAndQueryId(query.getQueryHandleString()); + // acquire session before launching query. + acquire(query.getLensSessionIdentifier()); + if (query.getStatus().cancelled()) { + return; + } else { + launchQuery(query); + } + } catch (Exception e) { + if (!query.getStatus().cancelled()) { + log.error("Error launching query: {}", query.getQueryHandle(), e); + incrCounter(QUERY_SUBMITTER_COUNTER); + try { + setFailedStatus(query, "Launching query failed", e); + } catch (LensException e1) { + log.error("Error in setting failed status", e1); + } + } + } finally { + query.setLaunching(false); + try { + release(query.getLensSessionIdentifier()); + } catch (LensException e) { + log.error("Error releasing session", e); + } + } + } + } + + private void launchQuery(final QueryContext query) throws LensException { + checkEstimatedQueriesState(query); query.getSelectedDriver().getQueryHook().preLaunch(query); QueryStatus oldStatus = query.getStatus(); QueryStatus newStatus = new QueryStatus(query.getStatus().getProgress(), null, QueryStatus.Status.LAUNCHED, "Query is launched on driver", false, null, null, null); query.validateTransition(newStatus); - // Check if we need to pass session's effective resources to selected driver addSessionResourcesToDriver(query); query.getSelectedDriver().executeAsync(query); @@ -748,27 +790,18 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE query.setLaunchTime(System.currentTimeMillis()); query.clearTransientStateAfterLaunch(); - launchedQueries.add(query); log.info("Added to launched queries. QueryId:{}", query.getQueryHandleString()); fireStatusChangeEvent(query, newStatus, oldStatus); } + } - private void addToWaitingQueries(final QueryContext query) throws LensException { - - checkEstimatedQueriesState(query); - this.waitingQueries.add(query); - log.info("Added to waiting queries. QueryId:{}", query.getQueryHandleString()); - } - - private void checkEstimatedQueriesState(final QueryContext query) throws LensException { - if (query.getSelectedDriver() == null || query.getSelectedDriverQueryCost() == null) { - throw new LensException("selected driver: " + query.getSelectedDriver() + " OR selected driver query cost: " - + query.getSelectedDriverQueryCost() + " is null. Query doesn't appear to be an estimated query."); - } + private void checkEstimatedQueriesState(final QueryContext query) throws LensException { + if (query.getSelectedDriver() == null || query.getSelectedDriverQueryCost() == null) { + throw new LensException("selected driver: " + query.getSelectedDriver() + " OR selected driver query cost: " + + query.getSelectedDriverQueryCost() + " is null. Query doesn't appear to be an estimated query."); } } - /** * Pause query submitter. * note : used in tests only @@ -803,6 +836,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE if (stopped || statusPoller.isInterrupted()) { return; } + if (ctx.isLaunching()) { + continue; + } logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString()); log.debug("Polling status for {}", ctx.getQueryHandle()); @@ -1167,8 +1203,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE this.queryConstraintsChecker = new DefaultQueryLaunchingConstraintsChecker(queryConstraints); - this.querySubmitterRunnable = new QuerySubmitter(LensServices.get().getErrorCollection(), this.waitingQueries, - this.queryConstraintsChecker); + this.querySubmitterRunnable = new QuerySubmitter(this.waitingQueries, this.queryConstraintsChecker); this.querySubmitter = new Thread(querySubmitterRunnable, "QuerySubmitter"); ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies = getImplementations( @@ -1266,6 +1301,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE waitingQueriesSelectionSvc.shutdownNow(); // Soft shutdown, Wait for current estimate tasks estimatePool.shutdown(); + // shutdown launcher pool + queryLauncherPool.shutdown(); // Soft shutdown for result purger too. Purging shouldn't take much time. if (null != queryResultPurger) { queryResultPurger.shutdown(); @@ -1285,6 +1322,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } // Needs to be done before queries' states are persisted, hence doing here. Await of other // executor services can be done after persistence, hence they are done in #stop + awaitTermination(queryLauncherPool); awaitTermination(queryCancellationPool); } @@ -1375,6 +1413,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE prepareQueryPurger.start(); startEstimatePool(); + startLauncherPool(); startQueryCancellationPool(); if (conf.getBoolean(RESULTSET_PURGE_ENABLED, DEFAULT_RESULTSET_PURGE_ENABLED)) { @@ -1409,11 +1448,39 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS, new SynchronousQueue<Runnable>(), threadFactory); - estimatePool.allowCoreThreadTimeOut(true); + estimatePool.allowCoreThreadTimeOut(false); estimatePool.prestartCoreThread(); this.estimatePool = estimatePool; } + private void startLauncherPool() { + int minPoolSize = conf.getInt(LAUNCHER_POOL_MIN_THREADS, + DEFAULT_LAUNCHER_POOL_MIN_THREADS); + int maxPoolSize = conf.getInt(LAUNCHER_POOL_MAX_THREADS, + DEFAULT_LAUNCHER_POOL_MAX_THREADS); + int keepAlive = conf.getInt(LAUNCHER_POOL_KEEP_ALIVE_MILLIS, + DEFAULT_LAUNCHER_POOL_KEEP_ALIVE_MILLIS); + + final ThreadFactory defaultFactory = Executors.defaultThreadFactory(); + final AtomicInteger thId = new AtomicInteger(); + // We are creating our own thread factory, just so that we can override thread name for easy debugging + ThreadFactory threadFactory = new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread th = defaultFactory.newThread(r); + th.setName("launcher-" + thId.incrementAndGet()); + return th; + } + }; + + log.debug("starting query launcher pool"); + + ThreadPoolExecutor launcherPool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS, + new SynchronousQueue<Runnable>(), threadFactory); + launcherPool.allowCoreThreadTimeOut(false); + launcherPool.prestartCoreThread(); + this.queryLauncherPool = launcherPool; + } private void startQueryCancellationPool() { ThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern("query-cancellation-pool-Thread-%d") @@ -1429,6 +1496,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE @Override public void run() { try { + logSegregationContext.setLogSegragationAndQueryId(handle.getHandleIdString()); cancelQuery(handle); } catch (Exception e) { log.error("Error while cancelling query {}", handle, e); @@ -2433,6 +2501,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE @Override public boolean cancelQuery(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException { try { + logSegregationContext.setLogSegragationAndQueryId(queryHandle.getHandleIdString()); log.info("CancelQuery: session:{} query:{}", sessionHandle, queryHandle); acquire(sessionHandle); return cancelQuery(queryHandle); @@ -2458,6 +2527,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE return false; } + if (ctx.isLaunching()) { + boolean launchCancelled = ctx.getQueryLauncher().cancel(true); + log.info("query launch cancellation success : {}", launchCancelled); + + } if (ctx.launched() || ctx.running()) { if (!ctx.getSelectedDriver().cancelQuery(queryHandle)) { log.info("Could not cancel query {}", queryHandle); @@ -2845,6 +2919,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE details.append("Query submitter thread is dead."); } + if (this.queryLauncherPool.isShutdown() || this.queryLauncherPool.isTerminated()) { + isHealthy = false; + details.append("Query launcher Pool is dead."); + } + if (this.estimatePool.isShutdown() || this.estimatePool.isTerminated()) { isHealthy = false; details.append("Estimate Pool is dead."); @@ -2968,6 +3047,17 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE return finishedQueries.size(); } + @Override + public long getLaunchingQueriesCount() { + long count = 0; + for (QueryContext ctx : launchedQueries.getQueries()) { + if (ctx.isLaunching()) { + count++; + } + } + return count; + } + @Data @AllArgsConstructor public static class QueryCount { http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java index e77c7fa..db53924 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java @@ -243,11 +243,11 @@ public class LensSessionImpl extends HiveSessionImpl { * * @see org.apache.hive.service.cli.session.HiveSessionImpl#acquire() */ - public synchronized void acquire() { + public void acquire() { this.acquire(true); } @Override - public synchronized void acquire(boolean userAccess) { + public void acquire(boolean userAccess) { super.acquire(userAccess); if (acquireCount.incrementAndGet() == 1) { // first acquire // Update thread's class loader with current DBs class loader @@ -263,13 +263,13 @@ public class LensSessionImpl extends HiveSessionImpl { * * @see org.apache.hive.service.cli.session.HiveSessionImpl#release() */ - public synchronized void release() { - setActive(); + public void release() { this.release(true); } + @Override public synchronized void release(boolean userAccess) { - lastAccessTime = System.currentTimeMillis(); + setActive(); if (acquireCount.decrementAndGet() == 0) { super.release(userAccess); } @@ -280,7 +280,7 @@ public class LensSessionImpl extends HiveSessionImpl { && (!persistInfo.markedForClose|| activeOperationsPresent()); } - public void setActive() { + public synchronized void setActive() { setLastAccessTime(System.currentTimeMillis()); } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index fa2d0dd..a57cad6 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -748,6 +748,28 @@ </property> <property> + <name>lens.server.launcher.pool.min.threads</name> + <value>3</value> + <description>Minimum number of threads in the query launcher thread pool</description> + </property> + + <property> + <name>lens.server.launcher.pool.max.threads</name> + <value>100</value> + <description>Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may + never grow till there, it would go to max for concurrrent queries allowed on all drivers together. + This value should be greater than the max concurrent queries allowed on all drivers. </description> + </property> + + <property> + <name>lens.server.launcher.pool.keepalive.millis</name> + <value>60000</value> + <description>Thread keep alive time in milliseconds for the query launcher thread pool. + If there are no query launches for this period,then cached threads will be released from the pool. + </description> + </property> + + <property> <name>lens.server.session.expiry.service.interval.secs</name> <value>3600</value> <description>Interval at which lens session expiry service runs</description> http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index d5e975a..a55552e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -260,7 +260,7 @@ public abstract class LensJerseyTest extends JerseyTest { } } if (unPurgable.size() > allowUnpurgable) { - throw new RuntimeException("finished queries can't be purged: " + unPurgable); + throw new RuntimeException(unPurgable.size() + " finished queries can't be purged: " + unPurgable); } while (finishedQueries.size() > allowUnpurgable) { Thread.sleep(5000); http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java b/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java index 0b38517..5a8d5e6 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java @@ -53,6 +53,15 @@ public class FailingQueryDriver extends MockDriver { @Override public void executeAsync(final QueryContext ctx) throws LensException { + // simulate wait for execution. + if (ctx.getUserQuery().contains("wait")) { + try { + // wait for 1 second. + Thread.sleep(1000); + } catch (InterruptedException e) { + // ignore interrupted exception + } + } throw new LensException("Simulated Launch Failure"); } } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java index 0df436a..8276ace 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java @@ -50,10 +50,7 @@ import org.apache.hadoop.conf.Configuration; import org.glassfish.jersey.test.TestProperties; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +import org.testng.annotations.*; import com.beust.jcommander.internal.Lists; @@ -107,6 +104,16 @@ public class TestQueryConstraints extends LensJerseyTest { loadData(TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue()); } + @BeforeClass + public void setupTest() { + restartLensServer(); + } + + @AfterClass + public void afterTest() { + restartLensServer(); + } + @Override public Map<String, String> getServerConfOverWrites() { return LensUtil.getHashMap(LensConfConstants.DRIVER_TYPES_AND_CLASSES, "mockHive:" + HiveDriver.class.getName(), @@ -143,7 +150,7 @@ public class TestQueryConstraints extends LensJerseyTest { } /** The test table. */ - public static final String TEST_TABLE = "TEST_TABLE"; + public static final String TEST_TABLE = "QUERY_CONSTRAINTS_TEST_TABLE"; /** * Creates the table. @@ -235,7 +242,7 @@ public class TestQueryConstraints extends LensJerseyTest { Optional.of(conf), mt); } - @AfterMethod + @AfterClass private void waitForPurge() throws InterruptedException { waitForPurge(0, queryService.finishedQueries); } http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 6af4225..e8aa223 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -281,6 +281,30 @@ public class TestQueryService extends LensJerseyTest { assertTrue(lensQuery.getFinishTime() > 0); } + /** + * Test multiple launches and failure in execute operation. + * + * @throws InterruptedException the interrupted exception + */ + @Test(dataProvider = "mediaTypeData") + public void testMultipleLaunches(MediaType mt) throws Exception { + QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId), + Optional.of("select wait,fail from non_exist"), + Optional.<LensConf>absent(), mt); + // launch one more. + QueryHandle handle2 = executeAndGetHandle(target(), Optional.of(lensSessionId), + Optional.of("select wait,fail2 from non_exist"), + Optional.<LensConf>absent(), mt); + assertNotEquals(handle, handle2); + // put a small sleep sothat querysubmitter picks handle2 + Thread.sleep(50); + assertTrue(queryService.getQueryContext(handle).isLaunching()); + assertTrue(queryService.getQueryContext(handle2).isLaunching()); + assertTrue(queryService.getLaunchingQueriesCount() > 1); + waitForQueryToFinish(target(), lensSessionId, handle, mt); + waitForQueryToFinish(target(), lensSessionId, handle2, mt); + } + @Test public void testPriorityOnMockQuery() throws Exception { String query = "select mock, fail from " + TEST_TABLE; http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java index c8ebd0c..d972780 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java @@ -53,7 +53,7 @@ public class ThreadSafeEstimatedQueryCollectionTest { LensDriver mockDriver2 = mock(LensDriver.class); QueryLaunchingConstraint constraint = new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries, null, - null, null); + null, null, maxConcurrentQueries); ThreadSafeEstimatedQueryCollection col = new ThreadSafeEstimatedQueryCollection(new DefaultEstimatedQueryCollection(new DefaultQueryCollection())); http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/logback.xml b/lens-server/src/test/resources/logback.xml index a345054..6b91a77 100644 --- a/lens-server/src/test/resources/logback.xml +++ b/lens-server/src/test/resources/logback.xml @@ -25,7 +25,7 @@ <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern> </encoder> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> + <level>INFO</level> </filter> </appender> <appender name="TEST_LOG_FILE" class="ch.qos.logback.core.FileAppender"> http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index 852955b..d9b2a19 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -95,174 +95,180 @@ Lens server configuration *--+--+---+--+ |34|lens.server.inmemory.resultset.ttl.secs|300|This property defines the TTL(time to live) in seconds for all result sets of type InMemoryResultSet beyond which they are eligible for purging irrespective of whether the result set has been read or not. The default value is 300 seconds (5 minutes).| *--+--+---+--+ -|35|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| +|35|lens.server.launcher.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the query launcher thread pool. If there are no query launches for this period,then cached threads will be released from the pool.| *--+--+---+--+ -|36|lens.server.mail.from.address|[email protected]|The from field in the notifier mail to the submitter.| +|36|lens.server.launcher.pool.max.threads|100|Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on all drivers together. This value should be greater than the max concurrent queries allowed on all drivers.| *--+--+---+--+ -|37|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| +|37|lens.server.launcher.pool.min.threads|3|Minimum number of threads in the query launcher thread pool| *--+--+---+--+ -|38|lens.server.mail.port|25|SMTP Port| +|38|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| *--+--+---+--+ -|39|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| +|39|lens.server.mail.from.address|[email protected]|The from field in the notifier mail to the submitter.| *--+--+---+--+ -|40|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| +|40|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| *--+--+---+--+ -|41|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| +|41|lens.server.mail.port|25|SMTP Port| *--+--+---+--+ -|42|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| +|42|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| *--+--+---+--+ -|43|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| +|43|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| *--+--+---+--+ -|44|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| +|44|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| *--+--+---+--+ -|45|lens.server.metrics.ganglia.host| |The ganglia host name| +|45|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| *--+--+---+--+ -|46|lens.server.metrics.ganglia.port| |The ganglia port| +|46|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| *--+--+---+--+ -|47|lens.server.metrics.graphite.host| |The graphite host name| +|47|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| *--+--+---+--+ -|48|lens.server.metrics.graphite.port| |The graphite port| +|48|lens.server.metrics.ganglia.host| |The ganglia host name| *--+--+---+--+ -|49|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| +|49|lens.server.metrics.ganglia.port| |The ganglia port| *--+--+---+--+ -|50|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| +|50|lens.server.metrics.graphite.host| |The graphite host name| *--+--+---+--+ -|51|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| +|51|lens.server.metrics.graphite.port| |The graphite port| *--+--+---+--+ -|52|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| +|52|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| *--+--+---+--+ -|53|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| +|53|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| *--+--+---+--+ -|54|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| +|54|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| *--+--+---+--+ -|55|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +|55|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| *--+--+---+--+ -|56|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +|56|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| *--+--+---+--+ -|57|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | +|57|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| +*--+--+---+--+ +|58|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +*--+--+---+--+ +|59|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +*--+--+---+--+ +|60|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | | | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ | | | | |Or define shortcuts for certain frequently used queries :) | *--+--+---+--+ -|58|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| +|61|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| *--+--+---+--+ -|59|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +|62|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| *--+--+---+--+ -|60|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +|63|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| *--+--+---+--+ -|61|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| +|64|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| *--+--+---+--+ -|62|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| +|65|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| *--+--+---+--+ -|63|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| +|66|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| *--+--+---+--+ -|64|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| +|67|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| *--+--+---+--+ -|65|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| +|68|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| *--+--+---+--+ -|66|lens.server.resultset.purge.enabled|false|Whether to purge the query results| +|69|lens.server.resultset.purge.enabled|false|Whether to purge the query results| *--+--+---+--+ -|67|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| +|70|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| *--+--+---+--+ -|68|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| +|71|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| *--+--+---+--+ -|69|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| +|72|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| *--+--+---+--+ -|70|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| +|73|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| *--+--+---+--+ -|71|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| +|74|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| *--+--+---+--+ -|72|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| +|75|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| *--+--+---+--+ -|73|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| +|76|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| *--+--+---+--+ -|74|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| +|77|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| *--+--+---+--+ -|75|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| +|78|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| *--+--+---+--+ -|76|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| +|79|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| *--+--+---+--+ -|77|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| +|80|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| *--+--+---+--+ -|78|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| +|81|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|79|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| +|82|lens.server.servicenames|session,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ -|80|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| +|83|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ -|81|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| +|84|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| *--+--+---+--+ -|82|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| +|85|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| *--+--+---+--+ -|83|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| +|86|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| *--+--+---+--+ -|84|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| +|87|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| *--+--+---+--+ -|85|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| +|88|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| *--+--+---+--+ -|86|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| +|89|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| *--+--+---+--+ -|87|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| +|90|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| *--+--+---+--+ -|88|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| +|91|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| *--+--+---+--+ -|89|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| +|92|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| *--+--+---+--+ -|90|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| +|93|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|91|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|94|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|92|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|95|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ -|93|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| +|96|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ -|94|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| +|97|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|95|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server| +|98|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server| *--+--+---+--+ -|96|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.| +|99|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.| *--+--+---+--+ -|97|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server| +|100|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server| *--+--+---+--+ -|98|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from| +|101|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from| *--+--+---+--+ -|99|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|102|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|100|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|103|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|101|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|104|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|102|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|105|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|103|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|106|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|104|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|107|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|105|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|108|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|106|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|109|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|107|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|110|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|108|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|111|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|109|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|112|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|110|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|113|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|111|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|114|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|112|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|115|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|113|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|116|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|114|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|117|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|115|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|118|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|116|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|119|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| *--+--+---+--+ -|117|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +|120|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|118|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|121|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values
