Repository: lens Updated Branches: refs/heads/master 11cc1859f -> 3076b26c8
LENS-1521 : Download url config needs to be specific to Query and not Service Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3076b26c Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3076b26c Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3076b26c Branch: refs/heads/master Commit: 3076b26c81d3d0d3511e9ee6bf00033a5d22149c Parents: 11cc185 Author: Amareshwari Sriramadasu <[email protected]> Authored: Fri Jun 22 17:16:15 2018 +0530 Committer: Rajitha.R <[email protected]> Committed: Fri Jun 22 17:16:15 2018 +0530 ---------------------------------------------------------------------- .../main/java/org/apache/lens/api/LensConf.java | 4 ++++ .../query/DefaultDownloadResultUrlProvider.java | 5 +++-- .../api/query/DownloadResultUrlProvider.java | 10 +++++++++- .../lens/server/api/query/QueryContext.java | 1 - .../lens/server/query/LensPersistentResult.java | 19 +++++++++++-------- .../server/query/QueryExecutionServiceImpl.java | 12 +++++------- 6 files changed, 32 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-api/src/main/java/org/apache/lens/api/LensConf.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/LensConf.java b/lens-api/src/main/java/org/apache/lens/api/LensConf.java index 86504c7..d48e4c2 100644 --- a/lens-api/src/main/java/org/apache/lens/api/LensConf.java +++ b/lens-api/src/main/java/org/apache/lens/api/LensConf.java @@ -79,4 +79,8 @@ public class LensConf extends ToYAMLString implements Serializable { public String getProperty(Object key) { return properties.get(key); } + + public String getProperty(Object key, String defaultValue) { + return properties.getOrDefault(key, defaultValue); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java index cf8a066..786a23e 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.api.query; +import org.apache.lens.api.LensConf; import org.apache.lens.server.api.LensConfConstants; import org.apache.hadoop.conf.Configuration; @@ -28,8 +29,8 @@ import lombok.extern.slf4j.Slf4j; public class DefaultDownloadResultUrlProvider implements DownloadResultUrlProvider { @Override - public String getResultUrl(Configuration conf, String queryHandle) { - log.debug("Returning Default result set url "); + public String getResultUrl(Configuration conf, LensConf qconf, String queryHandle) { + log.debug("Returning Default result set url " + conf.get(LensConfConstants.SERVER_BASE_URL)); return conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL) + "queryapi/queries/" + queryHandle + "/httpresultset"; } http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java index 136bf60..55fd127 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java @@ -18,6 +18,8 @@ */ package org.apache.lens.server.api.query; +import org.apache.lens.api.LensConf; + import org.apache.hadoop.conf.Configuration; /* @@ -25,5 +27,11 @@ import org.apache.hadoop.conf.Configuration; * */ public interface DownloadResultUrlProvider { - String getResultUrl(Configuration conf, String queryHandle); + /* + @param : conf : server conf + @param : qconf : query conf + @param : queryHandle : the query handle + @return : download url sent to user via email + * */ + String getResultUrl(Configuration conf, LensConf qconf, String queryHandle); } http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 8176f03..a0d8e0b 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -280,7 +280,6 @@ public class QueryContext extends AbstractQueryContext implements FailureContext if (selectedDriver != null) { this.setSelectedDriver(selectedDriver); } - this.lensConf = qconf; this.driverStatus = new DriverQueryStatus(); } http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java index 4aa4578..443ad9d 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java @@ -20,6 +20,7 @@ package org.apache.lens.server.query; import java.io.IOException; +import org.apache.lens.api.LensConf; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.LensResultSetMetadata; @@ -32,7 +33,7 @@ import org.apache.lens.server.api.query.QueryContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.util.ReflectionUtils; +import org.apache.hadoop.util.ReflectionUtils; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -53,6 +54,7 @@ public class LensPersistentResult extends PersistentResultSet { /** The file size. */ private final Long fileSize; private final Configuration conf; + private final LensConf queryConf; @Getter private String httpResultUrl = null; @@ -68,35 +70,36 @@ public class LensPersistentResult extends PersistentResultSet { */ public LensPersistentResult(QueryHandle queryHandle, LensResultSetMetadata metadata, String outputPath, Integer numRows, Long fileSize, - Configuration conf) { + Configuration conf, LensConf qconf) { this.metadata = metadata; this.outputPath = outputPath; this.numRows = numRows; this.fileSize = fileSize; this.conf = conf; + this.queryConf = qconf; if (isHttpResultAvailable()) { resultUrlSetter = ReflectionUtils.newInstance(this.conf.getClass( LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS, - LensConfConstants.DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER, DownloadResultUrlProvider.class)); - this.httpResultUrl = resultUrlSetter.getResultUrl(this.conf, queryHandle.toString()); + LensConfConstants.DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER, DownloadResultUrlProvider.class), this.conf); + this.httpResultUrl = resultUrlSetter.getResultUrl(this.conf, this.queryConf, queryHandle.toString()); log.info("Config : " + this.conf.get(LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS) + " Result url set as : " + this.httpResultUrl); } } - public LensPersistentResult(QueryContext ctx, Configuration conf) { + LensPersistentResult(QueryContext ctx, Configuration conf, LensConf qconf) { this(ctx.getQueryHandle(), ctx.getQueryOutputFormatter().getMetadata(), ctx.getQueryOutputFormatter().getFinalOutputPath(), ctx.getQueryOutputFormatter().getNumRows(), - ctx.getQueryOutputFormatter().getFileSize(), conf); + ctx.getQueryOutputFormatter().getFileSize(), conf, qconf); } - public LensPersistentResult(FinishedLensQuery query, Configuration conf) throws + LensPersistentResult(FinishedLensQuery query, Configuration conf, LensConf qconf) throws ClassNotFoundException, IOException { this(QueryHandle.fromString(query.getHandle()), LensResultSetMetadata.fromJson(query.getMetadata()), - query.getResult(), query.getRows(), query.getFileSize(), conf); + query.getResult(), query.getRows(), query.getFileSize(), conf, qconf); } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index d925b79..3f1731e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -1938,11 +1938,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE throw new NotFoundException("InMemory Query result purged " + queryHandle); } try { - Configuration queryConf = conf; - if (ctx != null && ctx.getConf() != null) { - queryConf.addResource(ctx.getConf()); + LensConf qConf = null; + if (ctx != null && ctx.getLensConf() != null) { + qConf = ctx.getLensConf(); } - return new LensPersistentResult(query, queryConf); + return new LensPersistentResult(query, conf, qConf); } catch (Exception e) { throw new LensException(e); } @@ -1971,9 +1971,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE LensResultSet resultSet = resultSets.get(queryHandle); if (resultSet == null) { if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) { - Configuration queryConf = conf; - queryConf.addResource(ctx.getConf()); - resultSets.put(queryHandle, new LensPersistentResult(ctx, queryConf)); + resultSets.put(queryHandle, new LensPersistentResult(ctx, conf, ctx.getLensConf())); } else if (ctx.isResultAvailableInDriver() && !ctx.isQueryClosedOnDriver()) { //InMemory result can not be returned for a closed query resultSet = getDriverResultset(queryHandle);
