Repository: lens Updated Branches: refs/heads/master 26d8f57a2 -> 7a6987254
LENS-1511:Enable SSL/TLS for lens server and client communication Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7a698725 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7a698725 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7a698725 Branch: refs/heads/master Commit: 7a6987254e58d64a1b6400174a2c79e13111b4c5 Parents: 26d8f57 Author: Rajitha R <rajit...@apache.org> Authored: Thu May 24 21:54:27 2018 +0530 Committer: Rajitha.R <rajit...@im0318-l0.corp.inmobi.com> Committed: Thu May 24 21:54:27 2018 +0530 ---------------------------------------------------------------------- .../apache/lens/client/LensClientConfig.java | 15 ++ .../org/apache/lens/client/LensConnection.java | 40 +++- .../src/main/resources/lens-client-default.xml | 10 + .../lens/server/api/LensConfConstants.java | 13 ++ .../java/org/apache/lens/server/LensServer.java | 41 +++- .../DelegationTokenAuthenticationFilter.java | 4 +- .../src/main/resources/lensserver-default.xml | 18 ++ lens-server/src/test/resources/lens-site.xml | 18 ++ src/site/apt/admin/config.apt | 186 ++++++++++--------- src/site/apt/admin/hivedriver-config.apt | 6 +- src/site/apt/user/olap-query-conf.apt | 30 +-- 11 files changed, 269 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java index eb12ee3..885870e 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java @@ -174,4 +174,19 @@ public class LensClientConfig extends Configuration { return CLIENT_PFX + filterName + WS_FILTER_IMPL_SFX; } + /** + * client side SSL config to enable https communication between lens server + * and clients. + */ + public static final String SSL_ENABLED = CLIENT_PFX + "ssl.enabled"; + + public static final boolean DEFAULT_SSL_ENABLED_VALUE = false; + + /** + * will skip server cert verification. + */ + public static final String SSL_IGNORE_SERVER_CERT = CLIENT_PFX + "ssl.ignore.server.cert"; + + public static final boolean DEFAULT_SSL_IGNORE_SERVER_CERT_VALUE = true; + } http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-client/src/main/java/org/apache/lens/client/LensConnection.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index ab49831..5fc8c37 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -26,6 +26,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; @@ -118,8 +121,7 @@ public class LensConnection implements AutoCloseable { } public Client buildClient() { - ClientBuilder cb = ClientBuilder.newBuilder().register(MultiPartFeature.class).register(MoxyJsonFeature.class) - .register(MoxyJsonConfigurationContextResolver.class); + ClientBuilder cb = getClientBuilder(params.getConf()); for (Class<?> aClass : params.getRequestFilters()) { cb.register(aClass); } @@ -134,6 +136,40 @@ public class LensConnection implements AutoCloseable { return client; } + /** + * getClientBuilder : initializes client builder + * specific to transfer protocol (TLS or SSL) + * + * @param config : client side config. + * @return Client builder. + */ + private ClientBuilder getClientBuilder(LensClientConfig config) { + + if (Boolean.valueOf(config.get(LensClientConfig.SSL_ENABLED, + String.valueOf(LensClientConfig.DEFAULT_SSL_ENABLED_VALUE)))) { + try { + log.info("SSL is enabled, Creating https client."); + + SSLContext sc = SSLContext.getInstance("TLSv1"); + System.setProperty("https.protocols", "TLSv1"); + + TrustManager[] trustedCerts = {new LensTrustManager(config)}; + sc.init(null, trustedCerts, new java.security.SecureRandom()); + HostnameVerifier trustedHosts = new LensHostnameVerifier(config); + + return ClientBuilder.newBuilder().sslContext(sc).hostnameVerifier(trustedHosts) + .register(MultiPartFeature.class).register(MoxyJsonFeature.class) + .register(MoxyJsonConfigurationContextResolver.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + log.info("SSL is disabled, Creating http client."); + return ClientBuilder.newBuilder().register(MultiPartFeature.class).register(MoxyJsonFeature.class) + .register(MoxyJsonConfigurationContextResolver.class); + } + } + private WebTarget getSessionWebTarget() { return getSessionWebTarget(buildClient()); } http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-client/src/main/resources/lens-client-default.xml ---------------------------------------------------------------------- diff --git a/lens-client/src/main/resources/lens-client-default.xml b/lens-client/src/main/resources/lens-client-default.xml index 132e1b1..559c47d 100644 --- a/lens-client/src/main/resources/lens-client-default.xml +++ b/lens-client/src/main/resources/lens-client-default.xml @@ -82,4 +82,14 @@ value specified while submitting the query for execution. </description> </property> + <property> + <name>lens.client.ssl.enabled</name> + <value>false</value> + <description>Specifies whether https is enabled for lens server or not.</description> + </property> + <property> + <name>lens.client.ssl.ignore.server.cert</name> + <value>true</value> + <description>Specifies whether https is enabled for lens server or not.</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index b81d0a8..5a4e3dd 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -1315,4 +1315,17 @@ public final class LensConfConstants { public static final String DELEGATION_TOKEN_AUTH_HDFS_PATH_TO_CHECK = SERVER_PFX + "delegation.token.auth.hdfs.path.to.check"; + + /** + * SSL config to enable https communication between lens server + * and clients. + */ + public static final String SSL_ENABLED = SERVER_PFX + "ssl.enabled"; + + public static final boolean DEFAULT_SSL_ENABLED_VALUE = false; + + public static final String SSL_KEYSTORE_FILE_PATH = SERVER_PFX + "ssl.file.path"; + + public static final String SSL_KEYSTORE_PASSWORD = SERVER_PFX + "ssl.password"; + } http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-server/src/main/java/org/apache/lens/server/LensServer.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java index 7f8b336..701ebbe 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java @@ -39,6 +39,8 @@ import org.glassfish.grizzly.http.server.HttpServer; import org.glassfish.grizzly.http.server.NetworkListener; import org.glassfish.grizzly.servlet.ServletRegistration; import org.glassfish.grizzly.servlet.WebappContext; +import org.glassfish.grizzly.ssl.SSLContextConfigurator; +import org.glassfish.grizzly.ssl.SSLEngineConfigurator; import org.glassfish.grizzly.threadpool.ThreadPoolConfig; import org.glassfish.jersey.filter.LoggingFilter; import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory; @@ -85,10 +87,10 @@ public class LensServer { * @throws IOException Signals that an I/O exception has occurred. */ private LensServer(HiveConf conf) throws IOException { + startServices(conf); - String baseURI = conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL); - HttpServer server = GrizzlyHttpServerFactory.createHttpServer(UriBuilder.fromUri(baseURI).build(), getApp(), - false); + + HttpServer server = getHttpServer(conf); int corePoolSize = conf.getInt(LensConfConstants.GRIZZLY_CORE_POOL_SIZE, LensConfConstants.DEFAULT_GRIZZLY_CORE_POOL_SIZE); @@ -269,4 +271,37 @@ public class LensServer { } }); } + + private HttpServer getHttpServer(HiveConf conf) throws IOException { + + String baseURI = conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL); + + if (Boolean.valueOf(conf.get(LensConfConstants.SSL_ENABLED, + String.valueOf(LensConfConstants.DEFAULT_SSL_ENABLED_VALUE)))) { + + log.info("SSL is enabled, starting lens server in https mode."); + + SSLContextConfigurator sslCon = new SSLContextConfigurator(); + String keyStoreFile = conf.get(LensConfConstants.SSL_KEYSTORE_FILE_PATH); + String sslPassword = conf.get(LensConfConstants.SSL_KEYSTORE_PASSWORD); + + if (keyStoreFile == null || keyStoreFile.isEmpty()) { + throw new IOException( + String.format("SSL is enabled but cert file is missing, " + + "%s should be initialize with valid cert file path.", + LensConfConstants.SSL_KEYSTORE_FILE_PATH)); + } + + sslCon.setKeyStoreFile(keyStoreFile); + sslCon.setKeyStorePass(sslPassword); + + return GrizzlyHttpServerFactory.createHttpServer(UriBuilder.fromUri(baseURI).build(), getApp(), + true, new SSLEngineConfigurator(sslCon).setClientMode(false).setNeedClientAuth(false)); + + } else { + log.info("SSL is not enabled, starting lens server in http mode."); + return GrizzlyHttpServerFactory.createHttpServer(UriBuilder.fromUri(baseURI).build(), getApp(), false); + } + + } } http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-server/src/main/java/org/apache/lens/server/auth/DelegationTokenAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/auth/DelegationTokenAuthenticationFilter.java b/lens-server/src/main/java/org/apache/lens/server/auth/DelegationTokenAuthenticationFilter.java index 2650bc7..2f7b0cc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/auth/DelegationTokenAuthenticationFilter.java +++ b/lens-server/src/main/java/org/apache/lens/server/auth/DelegationTokenAuthenticationFilter.java @@ -35,6 +35,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import org.apache.lens.server.LensServerConf; +import org.apache.lens.server.api.LensConfConstants; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -62,7 +63,8 @@ import lombok.extern.slf4j.Slf4j; public class DelegationTokenAuthenticationFilter implements ContainerRequestFilter { private static final String AUTH_SCHEME = "Digest-MD5"; private static final org.apache.hadoop.conf.Configuration CONF = LensServerConf.getHiveConf(); - private static final Path PATH_TO_CHECK = new Path(CONF.get("DELEGATION_TOKEN_AUTH_HDFS_PATH_TO_CHECK")); + private static final Path PATH_TO_CHECK = new Path( + CONF.get(LensConfConstants.DELEGATION_TOKEN_AUTH_HDFS_PATH_TO_CHECK)); private ResourceInfo resourceInfo;; http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index bef8251..dd81f62 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -978,4 +978,22 @@ <description>lens server principal name, must be in format lens/_HOST@KDC_REALM</description> </property> + <property> + <name>lens.server.ssl.enabled</name> + <value>false</value> + <description>flag to enable https communication between lens server and client.</description> + </property> + + <property> + <name>lens.server.ssl.file.path</name> + <value>/tmp/certs</value> + <description>local path for cert file if ssl ie enabled.</description> + </property> + + <property> + <name>lens.server.ssl.password</name> + <value>password</value> + <description>password for cert file</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/lens-server/src/test/resources/lens-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml index f5a0237..bc1e2bc 100644 --- a/lens-server/src/test/resources/lens-site.xml +++ b/lens-server/src/test/resources/lens-site.xml @@ -219,4 +219,22 @@ <description>lens server principal name, must be in format lens/_HOST@KDC_REALM</description> </property> + <property> + <name>lens.server.ssl.enabled</name> + <value>false</value> + <description>flag to enable https communication between lens server and client.</description> + </property> + + <property> + <name>lens.server.ssl.file.path</name> + <value>/tmp/certs</value> + <description>local path for cert file if ssl ie enabled.</description> + </property> + + <property> + <name>lens.server.ssl.password</name> + <value>password</value> + <description>password for cert file</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index 8720c82..9a1572b 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -115,182 +115,192 @@ Lens server configuration *--+--+---+--+ |44|lens.server.inmemory.resultset.ttl.secs|300|This property defines the TTL(time to live) in seconds for all result sets of type InMemoryResultSet beyond which they are eligible for purging irrespective of whether the result set has been read or not. The default value is 300 seconds (5 minutes).| *--+--+---+--+ -|45|lens.server.launcher.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the query launcher thread pool. If there are no query launches for this period,then cached threads will be released from the pool.| +|45|lens.server.kdc.login.service.interval.minutes|360|interval in minutes to refresh auth token when kerberos is enabled on hdfs and/or hive, metastore| *--+--+---+--+ -|46|lens.server.launcher.pool.max.threads|100|Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on all drivers together. This value should be greater than the max concurrent queries allowed on all drivers.| +|46|lens.server.launcher.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the query launcher thread pool. If there are no query launches for this period,then cached threads will be released from the pool.| *--+--+---+--+ -|47|lens.server.launcher.pool.min.threads|3|Minimum number of threads in the query launcher thread pool| +|47|lens.server.launcher.pool.max.threads|100|Maximum number of threads in the query launcher thread pool. Keeping the default to hundred, we may never grow till there, it would go to max for concurrrent queries allowed on all drivers together. This value should be greater than the max concurrent queries allowed on all drivers.| *--+--+---+--+ -|48|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| +|48|lens.server.launcher.pool.min.threads|3|Minimum number of threads in the query launcher thread pool| *--+--+---+--+ -|49|lens.server.mail.from.address|b...@company.com|The from field in the notifier mail to the submitter.| +|49|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource| *--+--+---+--+ -|50|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| +|50|lens.server.mail.from.address|b...@company.com|The from field in the notifier mail to the submitter.| *--+--+---+--+ -|51|lens.server.mail.port|25|SMTP Port| +|51|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail| *--+--+---+--+ -|52|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| +|52|lens.server.mail.port|25|SMTP Port| *--+--+---+--+ -|53|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| +|53|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.| *--+--+---+--+ -|54|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| +|54|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.| *--+--+---+--+ -|55|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| +|55|lens.server.max.sessions.per.user|10|Number of sessions can be allowed for each user. User has to close one of the active sessions to open a new session once limit is reached. Otherwise Server throws an exception by saying that opened session limit has been already reached for user.| *--+--+---+--+ -|56|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| +|56|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service| *--+--+---+--+ -|57|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| +|57|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource| *--+--+---+--+ -|58|lens.server.metrics.ganglia.host| |The ganglia host name| +|58|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.| *--+--+---+--+ -|59|lens.server.metrics.ganglia.port| |The ganglia port| +|59|lens.server.metrics.ganglia.host| |The ganglia host name| *--+--+---+--+ -|60|lens.server.metrics.graphite.host| |The graphite host name| +|60|lens.server.metrics.ganglia.port| |The ganglia port| *--+--+---+--+ -|61|lens.server.metrics.graphite.port| |The graphite port| +|61|lens.server.metrics.graphite.host| |The graphite host name| *--+--+---+--+ -|62|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| +|62|lens.server.metrics.graphite.port| |The graphite port| *--+--+---+--+ -|63|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| +|63|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds| *--+--+---+--+ -|64|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| +|64|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.| *--+--+---+--+ -|65|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| +|65|lens.server.moxyjson.ws.feature.impl|org.glassfish.jersey.moxy.json.MoxyJsonFeature|Enable Moxy json feature| *--+--+---+--+ -|66|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| +|66|lens.server.moxyjsonconfigresovler.ws.feature.impl|org.apache.lens.api.util.MoxyJsonConfigurationContextResolver|Moxy json configuration resolver| *--+--+---+--+ -|67|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| +|67|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource| *--+--+---+--+ -|68|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| +|68|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location| *--+--+---+--+ -|69|lens.server.query.comparator.classes|org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator|The Query cost comparator chain. Queries are compared in this order. To compare queries q1 and q2, first number of retries are considered. The one with more retries is placed first in the queue. If those are same, then their priorities are considered, with higher priorities coming before lower ones. If those are also same, then their submission times are considered. The query that was submitted first is placed first.| +|69|lens.server.principal|lens/_h...@apache.com|lens server principal name, must be in format lens/_HOST@KDC_REALM| *--+--+---+--+ -|70|lens.server.query.cost.parser.class|org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser|The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost| +|70|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| *--+--+---+--+ -|71|lens.server.query.expiry.check.interval.millis|60000|The interval(milliseconds) with which query expiry will run periodically. Default is 1 minute. The value needs to be much lower than lens.query.timeout.millis. If the final deployment values of query timeout can be smaller, then reduce this value to be much lower.| +|71|lens.server.query.comparator.classes|org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator|The Query cost comparator chain. Queries are compared in this order. To compare queries q1 and q2, first number of retries are considered. The one with more retries is placed first in the queue. If those are same, then their priorities are considered, with higher priorities coming before lower ones. If those are also same, then their submission times are considered. The query that was submitted first is placed first.| *--+--+---+--+ -|72|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +|72|lens.server.query.cost.parser.class|org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser|The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost| *--+--+---+--+ -|73|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | +|73|lens.server.query.expiry.check.interval.millis|60000|The interval(milliseconds) with which query expiry will run periodically. Default is 1 minute. The value needs to be much lower than lens.query.timeout.millis. If the final deployment values of query timeout can be smaller, then reduce this value to be much lower.| +*--+--+---+--+ +|74|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +*--+--+---+--+ +|75|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | | | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ | | | | |Or define shortcuts for certain frequently used queries :) | *--+--+---+--+ -|74|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| +|76|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| +*--+--+---+--+ +|77|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +*--+--+---+--+ +|78|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +*--+--+---+--+ +|79|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| *--+--+---+--+ -|75|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +|80|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| *--+--+---+--+ -|76|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +|81|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| *--+--+---+--+ -|77|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| +|82|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| *--+--+---+--+ -|78|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| +|83|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| *--+--+---+--+ -|79|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| +|84|lens.server.resultset.purge.enabled|false|Whether to purge the query results| *--+--+---+--+ -|80|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| +|85|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| *--+--+---+--+ -|81|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| +|86|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| *--+--+---+--+ -|82|lens.server.resultset.purge.enabled|false|Whether to purge the query results| +|87|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| *--+--+---+--+ -|83|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| +|88|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| *--+--+---+--+ -|84|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| +|89|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| *--+--+---+--+ -|85|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| +|90|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| *--+--+---+--+ -|86|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| +|91|lens.server.scheduler.instance.waiting.thread.interval.millis|300000|Thread interval for checking the waiting instances in milliseconds| *--+--+---+--+ -|87|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| +|92|lens.server.scheduler.max.job.per.user|-1|Maximum number of jobs that can be scheduled by a single user. If the number is less than zero, then there is no restriction on the number of jobs scheduled.| *--+--+---+--+ -|88|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| +|93|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| *--+--+---+--+ -|89|lens.server.scheduler.instance.waiting.thread.interval.millis|300000|Thread interval for checking the waiting instances in milliseconds| +|94|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| *--+--+---+--+ -|90|lens.server.scheduler.max.job.per.user|-1|Maximum number of jobs that can be scheduled by a single user. If the number is less than zero, then there is no restriction on the number of jobs scheduled.| +|95|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| *--+--+---+--+ -|91|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| +|96|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| *--+--+---+--+ -|92|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| +|97|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| *--+--+---+--+ -|93|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| +|98|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|94|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| +|99|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ -|95|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| +|100|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ -|96|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| +|101|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| *--+--+---+--+ -|97|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| +|102|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| *--+--+---+--+ -|98|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| +|103|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| *--+--+---+--+ -|99|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| +|104|lens.server.ssl.enabled|false|flag to enable https communication between lens server| *--+--+---+--+ -|100|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| +|105|lens.server.ssl.file.path|/usr/local/lens/server/certs|local path for cert file if ssl ie enabled.| *--+--+---+--+ -|101|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| +|106|lens.server.ssl.password|password|password for cert file| *--+--+---+--+ -|102|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| +|107|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| *--+--+---+--+ -|103|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| +|108|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| *--+--+---+--+ -|104|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| +|109|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| *--+--+---+--+ -|105|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| +|110|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| *--+--+---+--+ -|106|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| +|111|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| *--+--+---+--+ -|107|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| +|112|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| *--+--+---+--+ -|108|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| +|113|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|109|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|114|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|110|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|115|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ -|111|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| +|116|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ -|112|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| +|117|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|113|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|118|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|114|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|119|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|115|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|120|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|116|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|121|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|117|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|122|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|118|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|123|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|119|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|124|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|120|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|125|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|121|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|126|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|122|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|127|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|123|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|128|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|124|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|129|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|125|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|130|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|126|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|131|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|127|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|132|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|128|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|133|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|129|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|134|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|130|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|135|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| *--+--+---+--+ -|131|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +|136|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|132|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|137|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/src/site/apt/admin/hivedriver-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt index 221c166..71208bb 100644 --- a/src/site/apt/admin/hivedriver-config.apt +++ b/src/site/apt/admin/hivedriver-config.apt @@ -52,9 +52,9 @@ Hive driver configuration *--+--+---+--+ |14|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.| *--+--+---+--+ -|15|lens.driver.hive.driver.kerberos.principal|hive/_h...@apache.com|Set principal name to be used for hive server.| +|15|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2| *--+--+---+--+ -|16|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2| +|16|lens.driver.hive.kerberos.principal|hive/_h...@apache.com|Set principal name to be used for hive server.| *--+--+---+--+ |17|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ | | | | |The cost is calculated based on partition weights and fact weights. The interpretation of the default config is: \ | @@ -78,6 +78,4 @@ Hive driver configuration *--+--+---+--+ |20|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|21|lens.driver.hive.kerberos.principal| |Set principal name to be used for hive server.| -*--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/7a698725/src/site/apt/user/olap-query-conf.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/olap-query-conf.apt b/src/site/apt/user/olap-query-conf.apt index 90d635b..c671f74 100644 --- a/src/site/apt/user/olap-query-conf.apt +++ b/src/site/apt/user/olap-query-conf.apt @@ -56,32 +56,34 @@ OLAP query configuration *--+--+---+--+ |14|lens.cube.query.lookahead.ptparts.forinterval.${interval}|1|The value of number of lookahead process time partitions for interval specified. Interval can be any Update period.| *--+--+---+--+ -|15|lens.cube.query.max.interval| |Maximum value of the update period that the query timed dimensions can take values of. For example, if query involves month ranges, user can say query maximum interval is daily, then no monthly partitions will be picked.| +|15|lens.cube.query.lookahead.timeparts.forinterval.${interval}|1|The number of time partitions for interval specified where look ahead is applied. Interval can be any Update period.| *--+--+---+--+ -|16|lens.cube.query.nonexisting.partitions| |The list of comma separated non existing partitions, if query can run with partial data. The value will be set by the cube query rewriter| +|16|lens.cube.query.max.interval| |Maximum value of the update period that the query timed dimensions can take values of. For example, if query involves month ranges, user can say query maximum interval is daily, then no monthly partitions will be picked.| *--+--+---+--+ -|17|lens.cube.query.partition.where.clause.format| |The simple date format of how the queried partition should be put in where clause. If nothing is specified, it will use the format from org.apache.lens.cube.metadata.UpdatePeriod for each type of partition| +|17|lens.cube.query.nonexisting.partitions| |The list of comma separated non existing partitions, if query can run with partial data. The value will be set by the cube query rewriter| *--+--+---+--+ -|18|lens.cube.query.pick.lightest.fact.first|false|If set to true, lightest fact will be resolved first than resolving storages. Otherwise, storages will be resolved to check all partitions exist and then pick lightest fact among candidates| +|18|lens.cube.query.partition.where.clause.format| |The simple date format of how the queried partition should be put in where clause. If nothing is specified, it will use the format from org.apache.lens.cube.metadata.UpdatePeriod for each type of partition| *--+--+---+--+ -|19|lens.cube.query.process.time.partition.column| |The column name which is a process time column. If process time column is specified, query rewriter will look ahead the partitions of other timed dimensions inside this column.| +|19|lens.cube.query.pick.lightest.fact.first|false|If set to true, lightest fact will be resolved first than resolving storages. Otherwise, storages will be resolved to check all partitions exist and then pick lightest fact among candidates| *--+--+---+--+ -|20|lens.cube.query.promote.groupby.toselect|false|Tells whether to promote group by clauses to be promoted to select expressions if they are already not projected. To enable automatic promotion, this value should be true.| +|20|lens.cube.query.process.time.partition.column| |The column name which is a process time column. If process time column is specified, query rewriter will look ahead the partitions of other timed dimensions inside this column.| *--+--+---+--+ -|21|lens.cube.query.promote.select.togroupby|false|Tells whether to promote select expressions which is not inside any aggregate, to be promoted to groupby clauses, if they are already not part of groupby clauses. To enable automatic promotion, this value should be true.| +|21|lens.cube.query.promote.groupby.toselect|false|Tells whether to promote group by clauses to be promoted to select expressions if they are already not projected. To enable automatic promotion, this value should be true.| *--+--+---+--+ -|22|lens.cube.query.replace.timedim|true|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.| +|22|lens.cube.query.promote.select.togroupby|false|Tells whether to promote select expressions which is not inside any aggregate, to be promoted to groupby clauses, if they are already not part of groupby clauses. To enable automatic promotion, this value should be true.| *--+--+---+--+ -|23|lens.cube.query.rewrite.dim.filter.to.fact.filter|false|Flag specifies if dimension filter has to be rewritten as fact filter. for eg. where dim.name in ('x', 'y') will become where fact.dimid in (select dim.id from dim where dim.name in ('x','y'))| +|23|lens.cube.query.replace.timedim|true|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.| *--+--+---+--+ -|24|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.ORTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter| +|24|lens.cube.query.rewrite.dim.filter.to.fact.filter|false|Flag specifies if dimension filter has to be rewritten as fact filter. for eg. where dim.name in ('x', 'y') will become where fact.dimid in (select dim.id from dim where dim.name in ('x','y'))| *--+--+---+--+ -|25|lens.cube.query.valid.${cubename}.facttables| |List of comma separated fact tables that are valid for cube. If no value is specified, all fact tables are valid| +|25|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.ORTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter| *--+--+---+--+ -|26|lens.cube.query.valid.dim.storgaetables| |List of comma separated dimension storage tables that are valid. If no value is specified, all tables are valid| +|26|lens.cube.query.valid.${cubename}.facttables| |List of comma separated fact tables that are valid for cube. If no value is specified, all fact tables are valid| *--+--+---+--+ -|27|lens.cube.query.valid.fact.${facttable}.storage.${storagename}.updateperiods| |List of comma separated update periods that are valid for a fact on a storage. If no value is specified, all update periods are valid| +|27|lens.cube.query.valid.dim.storgaetables| |List of comma separated dimension storage tables that are valid. If no value is specified, all tables are valid| *--+--+---+--+ -|28|lens.cube.query.valid.fact.${facttable}.storagetables| |List of comma separated storage tables that are valid for a fact. If no value is specified, all storage tables are valid| +|28|lens.cube.query.valid.fact.${facttable}.storage.${storagename}.updateperiods| |List of comma separated update periods that are valid for a fact on a storage. If no value is specified, all update periods are valid| +*--+--+---+--+ +|29|lens.cube.query.valid.fact.${facttable}.storagetables| |List of comma separated storage tables that are valid for a fact. If no value is specified, all storage tables are valid| *--+--+---+--+ The configuration parameters and their default values
