Repository: lens Updated Branches: refs/heads/master 382881940 -> 389daac9a
LENS-1032 : Add option to kill the query on timeout api upon timeout Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/389daac9 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/389daac9 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/389daac9 Branch: refs/heads/master Commit: 389daac9a66b0cb69ac729a56b39b3aa903ad36a Parents: 3828819 Author: Puneet Gupta <puneet.k.gu...@gmail.com> Authored: Thu May 12 10:08:11 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Thu May 12 10:08:11 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/api/query/QueryStatus.java | 8 ++ .../src/main/resources/lens-client-default.xml | 7 ++ .../org/apache/lens/driver/hive/HiveDriver.java | 3 + .../lens/server/api/LensConfConstants.java | 11 ++ .../server/query/QueryExecutionServiceImpl.java | 120 +++++++++++++++---- .../lens/server/query/QueryServiceResource.java | 6 +- .../src/main/resources/lenssession-default.xml | 8 ++ .../lens/server/query/TestQueryService.java | 36 +++++- src/site/apt/admin/session-config.apt | 58 ++++----- 9 files changed, 201 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java index 67d1e79..daedbf9 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java @@ -205,6 +205,14 @@ public class QueryStatus extends ToYAMLString implements Serializable { return status.equals(Status.FAILED); } + public boolean cancelled() { + return status.equals(Status.CANCELED); + } + + public boolean executed() { + return status.equals(Status.EXECUTED); + } + /** * Checks if is valid transition. http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-client/src/main/resources/lens-client-default.xml ---------------------------------------------------------------------- diff --git a/lens-client/src/main/resources/lens-client-default.xml b/lens-client/src/main/resources/lens-client-default.xml index 96506ac..132e1b1 100644 --- a/lens-client/src/main/resources/lens-client-default.xml +++ b/lens-client/src/main/resources/lens-client-default.xml @@ -75,4 +75,11 @@ is used to execute a query. The default value is 10 seconds. </description> </property> + <property> + <name>lens.query.cancel.on.timeout</name> + <value>false</value> + <description>Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout + value specified while submitting the query for execution. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 1463cc2..b85a66f 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -1048,6 +1048,9 @@ public class HiveDriver extends AbstractLensDriver { this.timeoutMillis = timeoutMillis; this.listener = listener; this.pollInterval = timeoutMillis / 10; + if (pollInterval < 5000) { + pollInterval = 5000; //minimum poll interval is 5 secs + } } /* http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 7cf6449..a6df138 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -1070,4 +1070,15 @@ public final class LensConfConstants { * Default value of DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR is 30000 millis (30 seconds) */ public static final long DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR = 30000; + + /** + * Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout value + * specified while submitting the query for execution. + */ + public static final String CANCEL_QUERY_ON_TIMEOUT = QUERY_PFX + "cancel.on.timeout"; + + /** + * Default value of "lens.query.cancel.on.timeout" + */ + public static final boolean DEFAULT_CANCEL_QUERY_ON_TIMEOUT = true; } http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index b12943e..97cab86 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -77,6 +77,7 @@ import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -256,6 +257,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE */ private ExecutorService estimatePool; + /** + * The pool used for cancelling timed out queries. + */ + private ExecutorService queryCancellationPool; + private final LogSegregationContext logSegregationContext; private final ErrorCollection errorCollection = LensServices.get().getErrorCollection(); @@ -1245,6 +1251,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE queryResultPurger.stop(); } + queryCancellationPool.shutdown(); + log.info("Query execution service stopped"); } @@ -1281,6 +1289,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE prepareQueryPurger.start(); startEstimatePool(); + startQueryCancellationPool(); if (conf.getBoolean(RESULTSET_PURGE_ENABLED, DEFAULT_RESULTSET_PURGE_ENABLED)) { queryResultPurger = new QueryResultPurger(); @@ -1319,6 +1328,28 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE this.estimatePool = estimatePool; } + private void startQueryCancellationPool() { + ThreadFactory factory = new BasicThreadFactory.Builder() + .namingPattern("query-cancellation-pool-Thread-%d") + .priority(Thread.NORM_PRIORITY) + .build(); + //Using fixed values for pool . corePoolSize = maximumPoolSize = 3 and keepAliveTime = 60 secs + queryCancellationPool = new ThreadPoolExecutor(3, 3, 60, TimeUnit.SECONDS, new LinkedBlockingQueue(), factory); + } + + @AllArgsConstructor + private class CancelQueryTask implements Runnable { + private QueryHandle handle; + @Override + public void run() { + try { + cancelQuery(handle); + } catch (Exception e) { + log.error("Error while cancelling query {}", handle, e); + } + } + } + private static final String REWRITE_GAUGE = "CUBE_REWRITE"; private static final String DRIVER_ESTIMATE_GAUGE = "DRIVER_ESTIMATE"; private static final String DRIVER_SELECTOR_GAUGE = "DRIVER_SELECTION"; @@ -2028,26 +2059,35 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE long timeOutTime = System.currentTimeMillis() + timeoutMillis; QueryHandleWithResultSet result = new QueryHandleWithResultSet(handle); - while (isQueued(sessionHandle, handle)) { + boolean isQueued = true; + while (isQueued && System.currentTimeMillis() < timeOutTime) { try { Thread.sleep(10); + isQueued = isQueued(sessionHandle, handle); } catch (InterruptedException e) { log.error("Encountered Interrupted exception.", e); } } + if (isQueued) { //query is still queued even after waiting for timeoutMillis + result.setStatus(ctx.getStatus()); + addQueryToCancellationPool(ctx, conf, timeoutMillis); //cancel the timed-out Query + return result; + } QueryContext queryCtx = getUpdatedQueryContext(sessionHandle, handle); if (queryCtx.getSelectedDriver() == null) { result.setStatus(queryCtx.getStatus()); return result; } + QueryCompletionListenerImpl listener = new QueryCompletionListenerImpl(handle); - synchronized (queryCtx) { - if (!queryCtx.getStatus().finished()) { - queryCtx.getSelectedDriver().registerForCompletionNotification(handle, timeoutMillis, listener); + long waitTime = timeOutTime - System.currentTimeMillis(); + if (waitTime > 0 && !queryCtx.getStatus().finished()) { + synchronized (queryCtx) { + queryCtx.getSelectedDriver().registerForCompletionNotification(handle, waitTime, listener); try { synchronized (listener) { - listener.wait(timeoutMillis); + listener.wait(waitTime); } } catch (InterruptedException e) { log.info("Waiting thread interrupted"); @@ -2055,11 +2095,13 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } } + + // At this stage (since the listener waits only for driver completion and not server that may include result // formatting and persistence) the query status can be RUNNING or EXECUTED or FAILED or SUCCESSFUL LensResultSet resultSet = null; queryCtx = getUpdatedQueryContext(sessionHandle, handle, true); // If the query is already purged queryCtx = null - if (queryCtx != null && listener.querySuccessful && queryCtx.getStatus().isResultSetAvailable()) { + if (queryCtx != null && queryCtx.getStatus().isResultSetAvailable()) { resultSet = queryCtx.getSelectedDriver().fetchResultSet(queryCtx); if (resultSet instanceof PartiallyFetchedInMemoryResultSet) { PartiallyFetchedInMemoryResultSet partialnMemoryResult = (PartiallyFetchedInMemoryResultSet) resultSet; @@ -2095,9 +2137,27 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE result.setResult(null); result.setResultMetadata(null); result.setStatus(queryCtx.getStatus()); + + if (!queryCtx.finished()) { + addQueryToCancellationPool(queryCtx, conf, timeoutMillis); //cancel the timed-out Query + } + return result; } + /** + * This method is used to add a timed out query to cancellation pool. + * The query gets cancelled asynchronously + * Note : lens.query.cancel.on.timeout should be true for cancellation + */ + private void addQueryToCancellationPool(QueryContext queryCtx, Configuration config, long timeoutMillis) { + if (config.getBoolean(CANCEL_QUERY_ON_TIMEOUT, DEFAULT_CANCEL_QUERY_ON_TIMEOUT)) { + log.info("Query {} will be cancelled as it could not be completed within the specified timeout interval {}", + queryCtx.getQueryHandle(), timeoutMillis); + queryCancellationPool.submit(new CancelQueryTask(queryCtx.getQueryHandle())); + } + } + private boolean isQueued(final LensSessionHandle sessionHandle, final QueryHandle handle) throws LensException { // getQueryContext calls updateStatus, which fires query events if there's a change in status @@ -2239,26 +2299,38 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE try { log.info("CancelQuery: session:{} query:{}", sessionHandle, queryHandle); acquire(sessionHandle); - QueryContext ctx = getUpdatedQueryContext(sessionHandle, queryHandle); + return cancelQuery(queryHandle); + } finally { + release(sessionHandle); + } + } - synchronized (ctx) { + private boolean cancelQuery(@NonNull QueryHandle queryHandle) throws LensException { + QueryContext ctx = allQueries.get(queryHandle); + if (ctx == null) { + log.info("Could not cancel query {} as it has been purged already", queryHandle); + return false; + } - if (ctx.finished()) { - return false; - } + synchronized (ctx) { - if (ctx.launched() || ctx.running()) { - boolean ret = ctx.getSelectedDriver().cancelQuery(queryHandle); - if (!ret) { - return false; - } - } + updateStatus(queryHandle); - setCancelledStatus(ctx, "Query is cancelled"); - return true; + if (ctx.finished()) { + log.info("Could not cancel query {} as it has finished execution already", queryHandle); + return false; } - } finally { - release(sessionHandle); + + if (ctx.launched() || ctx.running()) { + if (!ctx.getSelectedDriver().cancelQuery(queryHandle)) { + log.info("Could not cancel query {}", queryHandle); + return false; + } + } + + log.info("Query {} cancelled successfully", queryHandle); + setCancelledStatus(ctx, "Query is cancelled"); + return true; } } @@ -2681,6 +2753,12 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE details.append("QueryResultPurger is dead."); } + if (queryCancellationPool.isShutdown() || queryCancellationPool.isTerminated()) { + isHealthy = false; + details.append("Query Cancellation Pool is dead."); + } + + if (!isHealthy) { log.error(details.toString()); } http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index a043550..d384abc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -191,7 +191,11 @@ public class QueryServiceResource { * {@link org.apache.lens.api.query.SubmitOp#EXECUTE} and * {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT} * @param conf The configuration for the query - * @param timeoutmillis The timeout for the query, honored only in case of value {@link + * @param timeoutmillis The timeout for the query. If the query does not finish within the specified + * timeout, it is automatically cancelled unless user specified otherwise + * by setting configuration lens.query.cancel.on.timeout = false. + * <br> + * Note: The timeout parameter is honored only in case of {@link * org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT} operation * @param queryName human readable query name set by user (optional parameter) http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server/src/main/resources/lenssession-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lenssession-default.xml b/lens-server/src/main/resources/lenssession-default.xml index ce296cc..62fca51 100644 --- a/lens-server/src/main/resources/lenssession-default.xml +++ b/lens-server/src/main/resources/lenssession-default.xml @@ -234,6 +234,14 @@ </description> </property> + <property> + <name>lens.query.cancel.on.timeout</name> + <value>true</value> + <description>Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout + value specified while submitting the query for execution. The default value is true. + </description> + </property> + <!-- properties for session --> <property> <name>lens.session.aux.jars</name> http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 62c0280..164c9f8 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.query; +import static javax.ws.rs.core.MediaType.APPLICATION_XML_TYPE; import static javax.ws.rs.core.Response.Status.*; import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS; @@ -1257,7 +1258,7 @@ public class TestQueryService extends LensJerseyTest { @Test(dataProvider = "mediaTypeData") public void testExecuteWithTimeoutQuery(MediaType mt) throws IOException, InterruptedException { final WebTarget target = target().path("queryapi/queries"); - + //1. Validate Persistent result final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, mt)); @@ -1275,6 +1276,7 @@ public class TestQueryService extends LensJerseyTest { assertNotNull(result.getResult()); validatePersistentResult((PersistentQueryResult) result.getResult(), result.getQueryHandle(), true, false); + //2. Validate InMemory result final FormDataMultiPart mp2 = new FormDataMultiPart(); LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); @@ -1291,6 +1293,27 @@ public class TestQueryService extends LensJerseyTest { validateInmemoryResultForTimeoutQuery(target, mp2, mt); } + @Test + public void testAutoCancelOnTimeOut() throws Exception { + queryService.pauseQuerySubmitter(true); + //First query will not be queued. @see QueryExecutionServiceImpl.QuerySubmitter.run + queryService.executeAsync(lensSessionId, "select 1 from "+TEST_TABLE, new LensConf(), "dummyQuery"); + + //Second query after pause will be queued + QueryHandleWithResultSet result = queryService.execute(lensSessionId, "select ID, IDSTR from "+ TEST_TABLE, 100, + new LensConf(), "testQuery"); + assertNotNull(result.getQueryHandle()); + assertTrue(result.getStatus().queued()); + int checkCtr = 0; + boolean cancelled = false; + while (!cancelled && checkCtr++ < 100) { //Max 10 secs wait + Thread.sleep(100); //wait for query to get auto cancelled + cancelled = queryService.getUpdatedQueryContext(lensSessionId, result.getQueryHandle()).getStatus().cancelled(); + } + assertTrue(cancelled); //auto cancelled beyond timeout + queryService.pauseQuerySubmitter(false); + } + private void validateInmemoryResultForTimeoutQuery(WebTarget target, FormDataMultiPart mp, MediaType mt) { if (mt.equals(MediaType.APPLICATION_JSON_TYPE)) { String result = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), String.class); @@ -1338,7 +1361,7 @@ public class TestQueryService extends LensJerseyTest { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); + APPLICATION_XML_TYPE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID, IDSTR from " + TEST_TABLE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute_with_timeout")); @@ -1346,14 +1369,15 @@ public class TestQueryService extends LensJerseyTest { mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("timeoutmillis").build(), timeOutMillis + "")); LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); + conf.addProperty(LensConfConstants.CANCEL_QUERY_ON_TIMEOUT, "false"); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); conf.addProperty(LensConfConstants.PREFETCH_INMEMORY_RESULTSET, "true"); conf.addProperty(LensConfConstants.PREFETCH_INMEMORY_RESULTSET_ROWS, preFetchRows); conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredInMemoryResultFormatter.class.getName()); conf.addProperty("deferPersistenceByMillis", deferPersistenceByMillis); // property used for test only mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - QueryHandleWithResultSet result =target.request(MediaType.APPLICATION_XML_TYPE) + APPLICATION_XML_TYPE)); + QueryHandleWithResultSet result =target.request(APPLICATION_XML_TYPE) .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData(); QueryHandle handle = result.getQueryHandle(); @@ -1374,11 +1398,11 @@ public class TestQueryService extends LensJerseyTest { assertNull(result.getResult()); // Query execution not finished yet } - waitForQueryToFinish(target(), lensSessionId, handle, Status.SUCCESSFUL, MediaType.APPLICATION_XML_TYPE); + waitForQueryToFinish(target(), lensSessionId, handle, Status.SUCCESSFUL, APPLICATION_XML_TYPE); // Test Persistent Result validatePersistedResult(handle, target(), lensSessionId, new String[][] { { "ID", "INT" }, { "IDSTR", "STRING" } }, - false, true, MediaType.APPLICATION_XML_TYPE); + false, true, APPLICATION_XML_TYPE); } public static class DeferredInMemoryResultFormatter extends FileSerdeFormatter { http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/src/site/apt/admin/session-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/session-config.apt b/src/site/apt/admin/session-config.apt index 206d6a4..fc6916f 100644 --- a/src/site/apt/admin/session-config.apt +++ b/src/site/apt/admin/session-config.apt @@ -48,60 +48,62 @@ Lens session configuration *--+--+---+--+ |12|lens.query.add.insert.overwrite|true|Prefix query with insert overwrite clause if the query is persistent. User can disable if user gave the clause himself.| *--+--+---+--+ -|13|lens.query.enable.mail.notify|false|When a query ends, whether to notify the submitter by mail or not.| +|13|lens.query.cancel.on.timeout|true|Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout value specified while submitting the query for execution. The default value is true.| *--+--+---+--+ -|14|lens.query.enable.metrics.per.query|false|Generates gauge metrics for each query to measure time taken with unique id appended for each query. Should be enabled only for performance measurements. Should not be enabled in day to day production environment.| +|14|lens.query.enable.mail.notify|false|When a query ends, whether to notify the submitter by mail or not.| *--+--+---+--+ -|15|lens.query.enable.persistent.resultset|false|Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions| +|15|lens.query.enable.metrics.per.query|false|Generates gauge metrics for each query to measure time taken with unique id appended for each query. Should be enabled only for performance measurements. Should not be enabled in day to day production environment.| *--+--+---+--+ -|16|lens.query.enable.persistent.resultset.indriver|true|Whether the result should be persisted by driver. Currently only HiveDriver persists the results in a HDFS location.| +|16|lens.query.enable.persistent.resultset|false|Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions| *--+--+---+--+ -|17|lens.query.hdfs.output.path|hdfsout|The directory under the parent result directory, in which HiveDriver will persist the results, if persisting by driver is enabled. This directory should exist and should have world writable permissions sothat all users will be able put query outputs here.| +|17|lens.query.enable.persistent.resultset.indriver|true|Whether the result should be persisted by driver. Currently only HiveDriver persists the results in a HDFS location.| *--+--+---+--+ -|18|lens.query.output.charset.encoding|UTF-8|The charset encoding for formatting query result. It supports all the encodings supported by java.io.OutputStreamWriter.| +|18|lens.query.hdfs.output.path|hdfsout|The directory under the parent result directory, in which HiveDriver will persist the results, if persisting by driver is enabled. This directory should exist and should have world writable permissions sothat all users will be able put query outputs here.| *--+--+---+--+ -|19|lens.query.output.compression.codec|org.apache.hadoop.io.compress.GzipCodec|The codec used to compress the query output, if compression is enabled| +|19|lens.query.output.charset.encoding|UTF-8|The charset encoding for formatting query result. It supports all the encodings supported by java.io.OutputStreamWriter.| *--+--+---+--+ -|20|lens.query.output.enable.compression|false|Whether to compress the query result output| +|20|lens.query.output.compression.codec|org.apache.hadoop.io.compress.GzipCodec|The codec used to compress the query output, if compression is enabled| *--+--+---+--+ -|21|lens.query.output.file.extn|.csv|The extension name for the persisted query output file. If file is compressed, the extension from compression codec will be appended to this extension.| +|21|lens.query.output.enable.compression|false|Whether to compress the query result output| *--+--+---+--+ -|22|lens.query.output.footer| |The value of custom footer that should be written, if any. This footer will be added in formatting driver persisted results.| +|22|lens.query.output.file.extn|.csv|The extension name for the persisted query output file. If file is compressed, the extension from compression codec will be appended to this extension.| *--+--+---+--+ -|23|lens.query.output.formatter| |The query result output formatter for the query. If no value is specified, then org.apache.lens.lib.query.FileSerdeFormatter will be used to format in-memory result sets, org.apache.lens.lib.query.FilePersistentFormatter will be used to format driver persisted result sets.| +|23|lens.query.output.footer| |The value of custom footer that should be written, if any. This footer will be added in formatting driver persisted results.| *--+--+---+--+ -|24|lens.query.output.header| |The value of custom header that should be written, if any. If no value column names will be used as header.| +|24|lens.query.output.formatter| |The query result output formatter for the query. If no value is specified, then org.apache.lens.lib.query.FileSerdeFormatter will be used to format in-memory result sets, org.apache.lens.lib.query.FilePersistentFormatter will be used to format driver persisted result sets.| *--+--+---+--+ -|25|lens.query.output.write.footer|false|Whether to write footer as part of query result. When enabled, total number of rows will be written as part of header.| +|25|lens.query.output.header| |The value of custom header that should be written, if any. If no value column names will be used as header.| *--+--+---+--+ -|26|lens.query.output.write.header|false|Whether to write header as part of query result formatting. When enabled the user given header will be added in case of driver persisted results, and column names chosen will be added as header for in-memory results.| +|26|lens.query.output.write.footer|false|Whether to write footer as part of query result. When enabled, total number of rows will be written as part of header.| *--+--+---+--+ -|27|lens.query.prefetch.inmemory.resultset|true|When set to true, specified number of rows of result set will be pre-fetched if the result set is of type InMemoryResultSet and query execution is not asynchronous i.e. query should be launched with operation as EXECUTE_WITH_TIMEOUT. Suggested usage of this property: It can be used by client to stream as well as persist results in server for queries that finish fast and produce results with fewer rows (should be less than number of rows pre-fetched). Note that the results are streamed to the client early, without waiting for persistence to finish. Default value of this property is true.| +|27|lens.query.output.write.header|false|Whether to write header as part of query result formatting. When enabled the user given header will be added in case of driver persisted results, and column names chosen will be added as header for in-memory results.| *--+--+---+--+ -|28|lens.query.prefetch.inmemory.resultset.rows|100|Specifies the number of rows to pre-fetch when lens.query.prefetch.inmemory.resultset is set to true. Default value is 100 rows.| +|28|lens.query.prefetch.inmemory.resultset|true|When set to true, specified number of rows of result set will be pre-fetched if the result set is of type InMemoryResultSet and query execution is not asynchronous i.e. query should be launched with operation as EXECUTE_WITH_TIMEOUT. Suggested usage of this property: It can be used by client to stream as well as persist results in server for queries that finish fast and produce results with fewer rows (should be less than number of rows pre-fetched). Note that the results are streamed to the client early, without waiting for persistence to finish. Default value of this property is true.| *--+--+---+--+ -|29|lens.query.result.email.cc| |When query ends, the result/failure reason will be sent to the user via email. The mail would be cc'ed to the addresses provided in this field.| +|29|lens.query.prefetch.inmemory.resultset.rows|100|Specifies the number of rows to pre-fetch when lens.query.prefetch.inmemory.resultset is set to true. Default value is 100 rows.| *--+--+---+--+ -|30|lens.query.result.fs.read.url| |Http read URL for FileSystem on which result is present, if available. For example webhdfs as http read url should http://host:port/webhdfs/v1. Currently we support only webhdfs url as the http url for HDFS file system| +|30|lens.query.result.email.cc| |When query ends, the result/failure reason will be sent to the user via email. The mail would be cc'ed to the addresses provided in this field.| *--+--+---+--+ -|31|lens.query.result.output.dir.format| |The format of the output if result is persisted in hdfs. The format should be expressed in HQL.| +|31|lens.query.result.fs.read.url| |Http read URL for FileSystem on which result is present, if available. For example webhdfs as http read url should http://host:port/webhdfs/v1. Currently we support only webhdfs url as the http url for HDFS file system| *--+--+---+--+ -|32|lens.query.result.output.serde|org.apache.lens.lib.query.CSVSerde|The default serde class name that should be used by org.apache.lens.lib.query.FileSerdeFormatter for formatting the output| +|32|lens.query.result.output.dir.format| |The format of the output if result is persisted in hdfs. The format should be expressed in HQL.| *--+--+---+--+ -|33|lens.query.result.parent.dir|file:///tmp/lensreports|The directory for storing persisted result of query. This directory should exist and should have writable permissions by lens server| +|33|lens.query.result.output.serde|org.apache.lens.lib.query.CSVSerde|The default serde class name that should be used by org.apache.lens.lib.query.FileSerdeFormatter for formatting the output| *--+--+---+--+ -|34|lens.query.result.size.format.threshold|10737418240|The maximum allowed size of the query result. If exceeds, no server side formatting would be done.| +|34|lens.query.result.parent.dir|file:///tmp/lensreports|The directory for storing persisted result of query. This directory should exist and should have writable permissions by lens server| *--+--+---+--+ -|35|lens.query.result.split.multiple|false|Whether to split the result into multiple files. If enabled, each file will be restricted to max rows configured. All the files will be available as zip.| +|35|lens.query.result.size.format.threshold|10737418240|The maximum allowed size of the query result. If exceeds, no server side formatting would be done.| *--+--+---+--+ -|36|lens.query.result.split.multiple.maxrows|100000|The maximum number of rows allowed in each file, when splitting the result into multiple files is enabled.| +|36|lens.query.result.split.multiple|false|Whether to split the result into multiple files. If enabled, each file will be restricted to max rows configured. All the files will be available as zip.| *--+--+---+--+ -|37|lens.session.aux.jars| |List of comma separated jar paths, which will added to the session| +|37|lens.query.result.split.multiple.maxrows|100000|The maximum number of rows allowed in each file, when splitting the result into multiple files is enabled.| *--+--+---+--+ -|38|lens.session.cluster.user| |Session level config which will determine which cluster user will access hdfs| +|38|lens.session.aux.jars| |List of comma separated jar paths, which will added to the session| *--+--+---+--+ -|39|lens.session.loggedin.user| |The username used to log in to lens. e.g. LDAP user| +|39|lens.session.cluster.user| |Session level config which will determine which cluster user will access hdfs| *--+--+---+--+ -|40|lens.session.metastore.exclude.cubetables.from.nativetables|true|Exclude cube related tables when fetching native tables| +|40|lens.session.loggedin.user| |The username used to log in to lens. e.g. LDAP user| +*--+--+---+--+ +|41|lens.session.metastore.exclude.cubetables.from.nativetables|true|Exclude cube related tables when fetching native tables| *--+--+---+--+ The configuration parameters and their default values
