Repository: lens Updated Branches: refs/heads/master fa2df7cb0 -> 138d11353
LENS-1519 :Result download url to be made configurable Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d94bcb11 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d94bcb11 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d94bcb11 Branch: refs/heads/master Commit: d94bcb11f94d3f06a5edd4a739f9db4380512a8d Parents: c6423ae Author: Rajitha R <rajit...@apache.org> Authored: Mon Jun 11 21:19:52 2018 +0530 Committer: Rajitha.R <rajit...@im0318-l0.corp.inmobi.com> Committed: Mon Jun 11 21:19:52 2018 +0530 ---------------------------------------------------------------------- .../lens/server/api/LensConfConstants.java | 15 +++++++- .../query/DefaultDownloadResultUrlProvider.java | 36 ++++++++++++++++++++ .../api/query/DownloadResultUrlProvider.java | 29 ++++++++++++++++ .../lens/server/query/LensPersistentResult.java | 11 ++++-- .../server/query/QueryExecutionServiceImpl.java | 3 ++ src/site/apt/user/cli.apt | 2 +- src/site/apt/user/client-config.apt | 12 ++++--- 7 files changed, 100 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 5a4e3dd..2147f08 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -23,6 +23,8 @@ import javax.ws.rs.core.MediaType; import org.apache.lens.api.parse.Parser; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.*; +import org.apache.lens.server.api.query.DefaultDownloadResultUrlProvider; +import org.apache.lens.server.api.query.DownloadResultUrlProvider; import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; import org.apache.lens.server.api.query.cost.QueryCost; @@ -733,7 +735,6 @@ public final class LensConfConstants { public static final int DEFAULT_KDC_LOGIN_SERVICE_INTERVAL_IN_MINUTES = 360; - /** * Lens principal for kerberos authentication */ @@ -1265,6 +1266,18 @@ public final class LensConfConstants { */ public static final String COMPLETENESS_CHECKER_CLASS = "lens.cube.metastore.completeness.checker.class"; + /*The class that implements the result download url provider interface */ + public static final String RESULT_DOWNLOAD_URL_PROVIDER_CLASS = SERVER_PFX + "result.download.url.provider.class"; + + /*The default result download url provider class */ + public static final Class<? extends DownloadResultUrlProvider> DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER = + DefaultDownloadResultUrlProvider.class.asSubclass(DownloadResultUrlProvider.class); + + /** + * The download url content config + */ + public static final String QUERY_EMAIL_DOWNLOAD_URL = QUERY_PFX + "email.download.url"; + /** * The default implementation of DataCompletenessChecker */ http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java new file mode 100644 index 0000000..cf8a066 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.query; + +import org.apache.lens.server.api.LensConfConstants; + +import org.apache.hadoop.conf.Configuration; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DefaultDownloadResultUrlProvider implements DownloadResultUrlProvider { + + @Override + public String getResultUrl(Configuration conf, String queryHandle) { + log.debug("Returning Default result set url "); + return conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL) + + "queryapi/queries/" + queryHandle + "/httpresultset"; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java new file mode 100644 index 0000000..136bf60 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.query; + +import org.apache.hadoop.conf.Configuration; + +/* +* The interface to implement for returning download url. This is embedded in the query competion email sent to the user +* */ +public interface DownloadResultUrlProvider { + + String getResultUrl(Configuration conf, String queryHandle); +} http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java index 3f24343..91bb701 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java @@ -25,12 +25,14 @@ import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.lens.server.api.driver.PersistentResultSet; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.DownloadResultUrlProvider; import org.apache.lens.server.api.query.FinishedLensQuery; import org.apache.lens.server.api.query.QueryContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.util.ReflectionUtils; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -54,6 +56,8 @@ public class LensPersistentResult extends PersistentResultSet { @Getter private String httpResultUrl = null; + private DownloadResultUrlProvider resultUrlSetter = null; + /** * Instantiates a new lens persistent result. * @param queryHandle the query handle @@ -71,8 +75,11 @@ public class LensPersistentResult extends PersistentResultSet { this.fileSize = fileSize; this.conf = conf; if (isHttpResultAvailable()) { - this.httpResultUrl = conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL) - + "queryapi/queries/" + queryHandle + "/httpresultset"; + resultUrlSetter = ReflectionUtils.newInstance(this.conf.getClass(LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS, + LensConfConstants.DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER, DownloadResultUrlProvider.class)); + this.httpResultUrl = resultUrlSetter.getResultUrl(this.conf, queryHandle.toString()); + log.info("Config : " + this.conf.get(LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS) + " Result url set as : " + + this.httpResultUrl); } } http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 4d141e5..77fce0b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -1932,11 +1932,13 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE */ private LensPersistentResult getResultsetFromDAO(QueryHandle queryHandle) throws LensException { FinishedLensQuery query = lensServerDao.getQuery(queryHandle.toString()); + QueryContext ctx = allQueries.get(queryHandle); if (query != null) { if (query.getResult() == null) { throw new NotFoundException("InMemory Query result purged " + queryHandle); } try { + conf.addResource(ctx.getConf()); return new LensPersistentResult(query, conf); } catch (Exception e) { throw new LensException(e); @@ -1966,6 +1968,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE LensResultSet resultSet = resultSets.get(queryHandle); if (resultSet == null) { if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) { + conf.addResource(ctx.getConf()); resultSets.put(queryHandle, new LensPersistentResult(ctx, conf)); } else if (ctx.isResultAvailableInDriver() && !ctx.isQueryClosedOnDriver()) { //InMemory result can not be returned for a closed query http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/src/site/apt/user/cli.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt index 12bc88c..aca2498 100644 --- a/src/site/apt/user/cli.apt +++ b/src/site/apt/user/cli.apt @@ -238,7 +238,7 @@ User CLI Commands *--+--+ |fact drop all storages [--fact_name] \<fact_name\>|drop all storages associated to fact <<<fact_name>>>\ | *--+--+ -|fact drop partitions [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [[--filter] \<partition-filter\>]|drop all partitions associated with fact <<<fact_name>>>, storage <<<storage_name>>> filtered by <<<partition-filter>>>\ | +|fact drop partitions [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--update_period] \<update_period\> [[--filter] \<partition-filter\>]|drop all partitions associated with fact <<<fact_name>>>, storage <<<storage_name>>> filtered by <<<partition-filter>>>\ | *--+--+ |fact drop storage [--fact_name] \<fact_name\> [--storage_name] \<storage_name\>|drop storage <<<storage_name>>> from fact <<<fact_name>>>\ | *--+--+ http://git-wip-us.apache.org/repos/asf/lens/blob/d94bcb11/src/site/apt/user/client-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/client-config.apt b/src/site/apt/user/client-config.apt index 4f0b2b4..0da93f1 100644 --- a/src/site/apt/user/client-config.apt +++ b/src/site/apt/user/client-config.apt @@ -38,12 +38,16 @@ Lens client configuration *--+--+---+--+ |7|lens.client.requestfilter.ws.filter.impl|org.apache.lens.client.RequestFilter|Implementation class for Request Filter| *--+--+---+--+ -|8|lens.client.user.name|anonymous|Lens client user name| +|8|lens.client.ssl.enabled|false|Specifies whether https is enabled for lens server or not.| *--+--+---+--+ -|9|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +|9|lens.client.ssl.ignore.server.cert|true|Specifies whether https is enabled for lens server or not.| *--+--+---+--+ -|10|lens.query.cancel.on.timeout|false|Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout value specified while submitting the query for execution.| +|10|lens.client.user.name|anonymous|Lens client user name| *--+--+---+--+ -|11|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| +|11|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| +*--+--+---+--+ +|12|lens.query.cancel.on.timeout|false|Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout value specified while submitting the query for execution.| +*--+--+---+--+ +|13|lens.server.base.url|http://0.0.0.0:9999/lensapi|The base url for the lens server| *--+--+---+--+ The configuration parameters and their default values
