LENS-514: Addition of an identifier in every log line for segregating logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4498c448 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4498c448 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4498c448 Branch: refs/heads/current-release-line Commit: 4498c4488021afce6ddaf9cdd5e16fca27b2d708 Parents: 2e90c29 Author: Himanshu Gahlaut <[email protected]> Authored: Thu May 28 12:31:50 2015 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Thu May 28 12:31:50 2015 +0530 ---------------------------------------------------------------------- .../org/apache/lens/api/query/QueryHandle.java | 11 +++- .../lens/api/query/QueryPrepareHandle.java | 4 ++ .../apache/lens/api/response/LensResponse.java | 14 +++-- .../apache/lens/driver/hive/TestHiveDriver.java | 3 +- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 32 ++++++++--- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 8 +-- .../server/api/query/ExplainQueryContext.java | 11 +++- .../server/api/query/PreparedQueryContext.java | 6 +- .../lens/server/api/query/QueryContext.java | 12 ++-- .../server/api/query/QueryExecutionService.java | 8 ++- .../server/model/LogSegregationContext.java | 35 ++++++++++++ .../MappedDiagnosticLogSegregationContext.java | 36 ++++++++++++ .../lens/server/AuthenticationFilter.java | 3 - .../org/apache/lens/server/LensApplication.java | 4 ++ .../server/LensRequestContextInitFilter.java | 59 ++++++++++++++++++++ .../java/org/apache/lens/server/LensServer.java | 10 ++-- .../org/apache/lens/server/LensServices.java | 14 ++++- .../org/apache/lens/server/query/QueryApp.java | 2 + .../lens/server/query/QueryEndNotifier.java | 14 +++-- .../server/query/QueryExecutionServiceImpl.java | 36 +++++++++--- .../lens/server/query/QueryServiceResource.java | 16 ++++-- .../lens/server/query/ResultFormatter.java | 9 ++- .../stats/store/log/LogStatisticsStore.java | 7 ++- .../store/log/StatisticsLogFileScannerTask.java | 12 ++++ .../store/log/StatisticsLogRollupHandler.java | 10 +++- .../org/apache/lens/server/LensJerseyTest.java | 6 +- .../lens/server/TestStartupOnMetastoreDown.java | 7 ++- .../server/query/QueryAPIErrorResponseTest.java | 5 +- .../stats/TestStatisticsLogFileScannerTask.java | 6 +- lens-server/src/test/resources/log4j.properties | 4 +- tools/conf-pseudo-distr/server/log4j.properties | 10 ++-- tools/conf/server/log4j.properties | 10 ++-- 32 files changed, 350 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java index 7b615e6..9f20184 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java @@ -27,6 +27,8 @@ import java.util.UUID; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.commons.lang.StringUtils; + import lombok.*; /** @@ -74,6 +76,13 @@ public class QueryHandle extends QuerySubmitResult implements Serializable { return new QueryHandle(UUID.fromString(handle)); } + public String getHandleIdString() { + if (handleId == null) { + return StringUtils.EMPTY; + } + return handleId.toString(); + } + /* * (non-Javadoc) * @@ -81,6 +90,6 @@ public class QueryHandle extends QuerySubmitResult implements Serializable { */ @Override public String toString() { - return handleId.toString(); + return getHandleIdString(); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java index efa04f1..756287e 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java @@ -75,6 +75,10 @@ public class QueryPrepareHandle extends QuerySubmitResult { */ @Override public String toString() { + return getQueryHandleString(); + } + + public String getQueryHandleString() { return prepareHandleId.toString(); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java index f6c3593..818ae40 100644 --- a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java +++ b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java @@ -18,11 +18,15 @@ */ package org.apache.lens.api.response; +import static com.google.common.base.Preconditions.checkArgument; + import javax.ws.rs.core.Response.Status; import javax.xml.bind.annotation.*; import org.apache.lens.api.query.QuerySubmitResult; +import org.apache.commons.lang.StringUtils; + import lombok.*; /** @@ -78,12 +82,12 @@ public class LensResponse<DATA, PAYLOAD> { private LensResponse(final String apiVersion, final String id, final DATA data, final LensErrorTO lensErrorTO, @NonNull final Status httpStatusCode) { - /* The checks commented below should be enabled in future, once story of apiVersion and id to be used for log - tracing is clear. Right now there could be REST APIs throwing LensException without initializing apiVersion - and id. + /* The check commented below should be enabled in future, once story of apiVersion is clear. Right now there could + be REST APIs throwing LensException without initializing apiVersion + + checkArgument(StringUtils.isNotBlank(apiVersion)); */ - checkArgument(StringUtils.isNotBlank(apiVersion)); - checkArgument(StringUtils.isNotBlank(id)); */ + checkArgument(StringUtils.isNotBlank(id)); this.apiVersion = apiVersion; this.id = id; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index 71fbc74..b21258a 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -160,7 +160,8 @@ public class TestHiveDriver { } protected ExplainQueryContext createExplainContext(final String query, Configuration conf) { - ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers); + ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf, + drivers); ectx.setLensSessionIdentifier(sessionid); return ectx; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 87e4638..70deaa4 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -47,6 +47,8 @@ import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryRewriter; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.log4j.Logger; import lombok.Getter; +import lombok.NonNull; import lombok.Setter; /** @@ -89,6 +92,8 @@ public class JDBCDriver implements LensDriver { /** Estimate connection provider */ private ConnectionProvider estimateConnectionProvider; + private LogSegregationContext logSegregationContext; + /** * Data related to a query submitted to JDBCDriver. */ @@ -142,12 +147,15 @@ public class JDBCDriver implements LensDriver { @Setter private long endTime; + private final LogSegregationContext logSegregationContext; + /** * Instantiates a new jdbc query context. * * @param context the context */ - public JdbcQueryContext(QueryContext context) { + public JdbcQueryContext(QueryContext context, @NonNull final LogSegregationContext logSegregationContext) { + this.logSegregationContext = logSegregationContext; this.lensContext = context; } @@ -182,6 +190,10 @@ public class JDBCDriver implements LensDriver { } isClosed = true; } + + public String getQueryHandleString() { + return this.lensContext.getQueryHandleString(); + } } /** @@ -261,14 +273,16 @@ public class JDBCDriver implements LensDriver { /** The query context. */ private final JdbcQueryContext queryContext; + private final LogSegregationContext logSegregationContext; /** * Instantiates a new query callable. * * @param queryContext the query context */ - public QueryCallable(JdbcQueryContext queryContext) { + public QueryCallable(JdbcQueryContext queryContext, @NonNull LogSegregationContext logSegregationContext) { this.queryContext = queryContext; + this.logSegregationContext = logSegregationContext; queryContext.setStartTime(System.currentTimeMillis()); } @@ -279,6 +293,9 @@ public class JDBCDriver implements LensDriver { */ @Override public QueryResult call() { + + logSegregationContext.set(this.queryContext.getQueryHandleString()); + Statement stmt = null; Connection conn = null; QueryResult result = new QueryResult(); @@ -432,6 +449,7 @@ public class JDBCDriver implements LensDriver { LOG.error("Error initializing connection provider: " + e.getMessage(), e); throw new LensException(e); } + this.logSegregationContext = new MappedDiagnosticLogSegregationContext(); } /** @@ -494,8 +512,6 @@ public class JDBCDriver implements LensDriver { /** * Rewrite query. * - * @param query the query - * @param conf the conf * @return the string * @throws LensException the lens exception */ @@ -850,10 +866,10 @@ public class JDBCDriver implements LensDriver { */ private QueryResult executeInternal(QueryContext context, String rewrittenQuery) throws LensException { - JdbcQueryContext queryContext = new JdbcQueryContext(context); + JdbcQueryContext queryContext = new JdbcQueryContext(context, logSegregationContext); queryContext.setPrepared(false); queryContext.setRewrittenQuery(rewrittenQuery); - QueryResult result = new QueryCallable(queryContext).call(); + QueryResult result = new QueryCallable(queryContext, logSegregationContext).call(); return result; // LOG.info("Execute " + context.getQueryHandle()); } @@ -870,10 +886,10 @@ public class JDBCDriver implements LensDriver { // Always use the driver rewritten query not user query. Since the // conf we are passing here is query context conf, we need to add jdbc xml in resource path String rewrittenQuery = rewriteQuery(context); - JdbcQueryContext jdbcCtx = new JdbcQueryContext(context); + JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext); jdbcCtx.setRewrittenQuery(rewrittenQuery); try { - Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx)); + Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext)); jdbcCtx.setResultFuture(future); } catch (RejectedExecutionException e) { LOG.error("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index 4f03f23..2e3dcc7 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -24,10 +24,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -131,7 +128,8 @@ public class TestJdbcDriver { } protected ExplainQueryContext createExplainContext(final String query, Configuration conf) { - ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers); + ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf, + drivers); return ectx; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java index da44955..c7b9887 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java @@ -31,14 +31,21 @@ import org.apache.hadoop.conf.Configuration; public class ExplainQueryContext extends AbstractQueryContext { private static final long serialVersionUID = 1L; + private final String requestId; /** * Constructor. Only needs user query and conf. * * @param query * @param qconf */ - public ExplainQueryContext(String query, final String user, LensConf conf, Configuration qconf, - Collection<LensDriver> drivers) { + public ExplainQueryContext(final String requestId, String query, final String user, LensConf conf, + Configuration qconf, Collection<LensDriver> drivers) { super(query, user, conf, qconf, drivers, true); + this.requestId = requestId; + } + + @Override + public String getLogHandle() { + return this.requestId; } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java index 1ce89ac..b6f669b 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java @@ -153,12 +153,16 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye lensConf); } + public String getQueryHandleString() { + return prepareHandle.getQueryHandleString(); + } + /** * Get prepared query handle string * @return */ @Override public String getLogHandle() { - return prepareHandle.getPrepareHandleId().toString(); + return getQueryHandleString(); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 169ac8d..3e0c26c 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -368,12 +368,12 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que public boolean getCompressOutput() { return conf.getBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION, - LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION); + LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION); } public long getMaxResultSplitRows() { return conf.getLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, - LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS); + LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS); } /** @@ -383,7 +383,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que */ public boolean splitResultIntoMultipleFiles() { return conf.getBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, - LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE); + LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE); } public String getClusterUser() { @@ -397,6 +397,10 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que */ @Override public String getLogHandle() { - return queryHandle.getHandleId().toString(); + return getQueryHandleString(); + } + + public String getQueryHandleString() { + return queryHandle.getHandleIdString(); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java index 8b6d131..3c0f787 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java @@ -40,6 +40,7 @@ public interface QueryExecutionService { /** * Estimate the cost of given query. * + * @param requestId the request Id of request used to start estimate operation * @param sessionHandle the session handle * @param query The query should be in HiveQL(SQL like) * @param conf The query configuration @@ -48,18 +49,21 @@ public interface QueryExecutionService { * * @throws LensException thrown in case of failure */ - QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException; + QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf) + throws LensException; /** * Explain the given query. * + * @param requestId the request Id of request used to start explain operation * @param sessionHandle the session handle * @param query The query should be in HiveQL(SQL like) * @param conf The query configuration * @return The query plan; * @throws LensException the lens exception */ - QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException; + QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf) + throws LensException; /** * Prepare the query. http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java new file mode 100644 index 0000000..8691a05 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.model; + +public interface LogSegregationContext { + + /** + * Sets an id to be used by current thread in every log line for log segregation + * + * @param id the id to be added to every log line of current thread + */ + void set(final String id); + + /** + * + * @return the id being used by the current thread for log segregation + */ + String get(); +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java new file mode 100644 index 0000000..989cb9d --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.model; + +import org.slf4j.MDC; + +public class MappedDiagnosticLogSegregationContext implements LogSegregationContext { + + private static final String LOG_SEGREGATION_ID = "logSegregationId"; + + @Override + public void set(String id) { + MDC.put(LOG_SEGREGATION_ID, id); + } + + @Override + public String get() { + return MDC.get(LOG_SEGREGATION_ID); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java index b64d822..e39c7a2 100644 --- a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java +++ b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java @@ -19,7 +19,6 @@ package org.apache.lens.server; import java.io.IOException; -import java.util.UUID; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerRequestFilter; @@ -45,9 +44,7 @@ public class AuthenticationFilter implements ContainerRequestFilter { public void filter(ContainerRequestContext requestContext) throws IOException { final SecurityContext securityContext = requestContext.getSecurityContext(); - String requestId = UUID.randomUUID().toString(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : null; - requestContext.getHeaders().add("requestId", requestId); LOG.info("Request from user: " + user + ", path=" + requestContext.getUriInfo().getPath()); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensApplication.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java index cb452e8..a3079cc 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java @@ -73,6 +73,10 @@ public class LensApplication extends Application { classes.add(wsFilterClass); LOG.info("Added filter " + wsFilterClass); } + + LOG.debug("LensRequestContextInitFilter added..."); + classes.add(LensRequestContextInitFilter.class); + return classes; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java new file mode 100644 index 0000000..c9658fc --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server; + +import java.io.IOException; +import java.util.UUID; + +import javax.annotation.Priority; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; + +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; + +import lombok.extern.slf4j.Slf4j; + +/** + * LensRequestContextInitFilter is expected to be called before all other request filters. + * Priority value of 1 is to ensure the same. + * + */ +@Slf4j +@Priority(1) +public class LensRequestContextInitFilter implements ContainerRequestFilter { + + private static final String REQUEST_ID = "requestId"; + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + + log.debug("Entering {}", getClass().getName()); + + /* Create a unique identifier for request */ + String uniqueRequesId = UUID.randomUUID().toString(); + + /* Add request id for appearing in every log line */ + new MappedDiagnosticLogSegregationContext().set(uniqueRequesId); + + /* Add request id to headers */ + requestContext.getHeaders().add(REQUEST_ID, uniqueRequesId); + + log.debug("Leaving {}", getClass().getName()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServer.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java index c6d7ea1..e200bdf 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java @@ -20,10 +20,7 @@ package org.apache.lens.server; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.logging.Logger; import javax.ws.rs.core.UriBuilder; @@ -33,6 +30,7 @@ import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.error.LensExceptionMapper; import org.apache.lens.server.metrics.MetricsServiceImpl; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.ui.UIApp; import org.apache.commons.logging.Log; @@ -113,6 +111,7 @@ public class LensServer { } private ResourceConfig getApp() { + ResourceConfig app = ResourceConfig.forApplicationClass(LensApplication.class); app.register(new LoggingFilter(Logger.getLogger(LensServer.class.getName() + ".request"), true)); app.register(LensExceptionMapper.class); @@ -185,6 +184,9 @@ public class LensServer { */ public static void main(String[] args) throws Exception { + final String runId = UUID.randomUUID().toString(); + new MappedDiagnosticLogSegregationContext().set(runId); + printStartupMessage(); try { final LensServer thisServer = LensServer.createLensServer(LensServerConf.getHiveConf()); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index f769619..ab197ab 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -33,6 +33,8 @@ import org.apache.lens.server.api.ServiceProvider; import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.metrics.MetricsServiceImpl; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.stats.StatisticsService; import org.apache.lens.server.user.UserConfigLoaderFactory; @@ -49,6 +51,7 @@ import org.apache.hive.service.Service; import org.apache.hive.service.cli.CLIService; import lombok.Getter; +import lombok.NonNull; import lombok.Setter; @@ -68,7 +71,8 @@ public class LensServices extends CompositeService implements ServiceProvider { private static final String FS_IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; /** The instance. */ - private static LensServices instance = new LensServices(LENS_SERVICES_NAME); + private static LensServices instance = new LensServices(LENS_SERVICES_NAME, + new MappedDiagnosticLogSegregationContext()); /** The conf. */ private HiveConf conf; @@ -108,6 +112,9 @@ public class LensServices extends CompositeService implements ServiceProvider { @Getter private ErrorCollection errorCollection; + @Getter + private final LogSegregationContext logSegregationContext; + /** * The Enum SERVICE_MODE. */ @@ -129,8 +136,9 @@ public class LensServices extends CompositeService implements ServiceProvider { * * @param name the name */ - public LensServices(String name) { + public LensServices(String name, @NonNull final LogSegregationContext logSegregationContext) { super(name); + this.logSegregationContext = logSegregationContext; } // This is only for test, to simulate a restart of the server @@ -244,6 +252,8 @@ public class LensServices extends CompositeService implements ServiceProvider { @Override public void run() { try { + final String runId = UUID.randomUUID().toString(); + logSegregationContext.set(runId); persistLensServiceState(); LOG.info("SnapShot of Lens Services created"); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java index 0d23726..9db034a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java @@ -25,6 +25,7 @@ import javax.ws.rs.ApplicationPath; import javax.ws.rs.core.Application; import org.apache.lens.server.LensApplicationListener; +import org.apache.lens.server.LensRequestContextInitFilter; import org.glassfish.jersey.filter.LoggingFilter; import org.glassfish.jersey.media.multipart.MultiPartFeature; @@ -40,6 +41,7 @@ public class QueryApp extends Application { // register root resource classes.add(QueryServiceResource.class); classes.add(MultiPartFeature.class); + classes.add(LensRequestContextInitFilter.class); classes.add(LoggingFilter.class); classes.add(LensApplicationListener.class); return classes; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java index 1c6f186..df36a21 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java @@ -37,6 +37,7 @@ import org.apache.lens.server.api.events.AsyncEventListener; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryEnded; +import org.apache.lens.server.model.LogSegregationContext; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; @@ -44,6 +45,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import lombok.Data; +import lombok.NonNull; /** * The Class QueryEndNotifier. @@ -77,23 +79,26 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> { /** The mail smtp connection timeout. */ private final int mailSmtpConnectionTimeout; + private final LogSegregationContext logSegregationContext; + /** * Instantiates a new query end notifier. * * @param queryService the query service * @param hiveConf the hive conf */ - public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf) { + public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf, + @NonNull final LogSegregationContext logSegregationContext) { this.queryService = queryService; this.conf = hiveConf; from = conf.get(LensConfConstants.MAIL_FROM_ADDRESS); host = conf.get(LensConfConstants.MAIL_HOST); port = conf.get(LensConfConstants.MAIL_PORT); - mailSmtpTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT, - LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT)); + mailSmtpTimeout = Integer.parseInt( + conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT, LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT)); mailSmtpConnectionTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_CONNECTIONTIMEOUT, LensConfConstants.MAIL_DEFAULT_SMTP_CONNECTIONTIMEOUT)); - + this.logSegregationContext = logSegregationContext; } /* @@ -112,6 +117,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> { + ". No email generated"); return; } + this.logSegregationContext.set(queryContext.getQueryHandleString()); boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(LensConfConstants.QUERY_MAIL_NOTIFY, LensConfConstants.WHETHER_MAIL_NOTIFY_DEFAULT)); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 7c2da3a..dbc52e9 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -51,6 +51,8 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.query.*; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.stats.StatisticsService; import org.apache.lens.server.util.UtilityMethods; @@ -229,6 +231,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu */ private ExecutorService estimatePool; + private final LogSegregationContext logSegregationContext; + /** * The driver event listener. */ @@ -250,8 +254,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu * @param cliService the cli service * @throws LensException the lens exception */ - public QueryExecutionServiceImpl(CLIService cliService) throws LensException { + public QueryExecutionServiceImpl(CLIService cliService) + throws LensException { super(NAME, cliService); + this.logSegregationContext = new MappedDiagnosticLogSegregationContext(); } /** @@ -280,10 +286,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu log.info("Registered query state logger"); } // Add result formatter - getEventService().addListenerForType(new ResultFormatter(this), QueryExecuted.class); + getEventService().addListenerForType(new ResultFormatter(this, this.logSegregationContext), QueryExecuted.class); getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(this, getEventService()), QueryEnded.class); - getEventService().addListenerForType(new QueryEndNotifier(this, getCliService().getHiveConf()), QueryEnded.class); + getEventService().addListenerForType( + new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class); log.info("Registered query result formatter"); } @@ -456,6 +463,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu public QueryContext getCtx() { return ctx; } + + public String getQueryHandleString() { + return ctx.getQueryHandleString(); + } } /** @@ -479,6 +490,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) { try { QueryContext ctx = queuedQueries.take(); + + /* Setting log segregation id */ + logSegregationContext.set(ctx.getQueryHandleString()); + synchronized (ctx) { if (ctx.getStatus().getStatus().equals(Status.QUEUED)) { log.info("Launching query:" + ctx.getUserQuery()); @@ -553,6 +568,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu if (stopped || statusPoller.isInterrupted()) { return; } + + logSegregationContext.set(ctx.getQueryHandleString()); log.info("Polling status for " + ctx.getQueryHandle()); try { // session is not required to update status of the query @@ -762,6 +779,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu FinishedQuery finished = null; try { finished = finishedQueries.take(); + logSegregationContext.set(finished.getQueryHandleString()); } catch (InterruptedException e) { log.info("QueryPurger has been interrupted, exiting"); return; @@ -835,6 +853,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu while (!stopped && !prepareQueryPurger.isInterrupted()) { try { PreparedQueryContext prepared = preparedQueryQueue.take(); + logSegregationContext.set(prepared.getQueryHandleString()); destroyPreparedQuery(prepared); log.info("Purged prepared query: " + prepared.getPrepareHandle()); } catch (LensException e) { @@ -1005,6 +1024,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu } }; + LOG.debug("starting estimate pool"); ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), threadFactory); estimatePool.allowCoreThreadTimeOut(true); @@ -1163,6 +1183,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu @Override public void run() { try { + logSegregationContext.set(ctx.getLogHandle()); acquire(ctx.getLensSessionIdentifier()); MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_GAUGE); @@ -2021,13 +2042,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu * java.lang.String, org.apache.lens.api.LensConf) */ @Override - public QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf lensConf) + public QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException { try { log.info("Estimate: " + sessionHandle.toString() + " query:" + query); acquire(sessionHandle); Configuration qconf = getLensConf(sessionHandle, lensConf); - ExplainQueryContext estimateQueryContext = new ExplainQueryContext(query, + ExplainQueryContext estimateQueryContext = new ExplainQueryContext(requestId, query, getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values()); estimateQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString()); accept(query, qconf, SubmitOp.ESTIMATE); @@ -2045,12 +2066,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu * java.lang.String, org.apache.lens.api.LensConf) */ @Override - public QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException { + public QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf) + throws LensException { try { log.info("Explain: " + sessionHandle.toString() + " query:" + query); acquire(sessionHandle); Configuration qconf = getLensConf(sessionHandle, lensConf); - ExplainQueryContext explainQueryContext = new ExplainQueryContext(query, + ExplainQueryContext explainQueryContext = new ExplainQueryContext(requestId, query, getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values()); explainQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString()); accept(query, qconf, SubmitOp.EXPLAIN); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index 34e14b6..badde8c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -40,6 +40,7 @@ import org.apache.lens.server.api.annotations.MultiPurposeResource; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.error.UnSupportedQuerySubmitOpException; +import org.apache.lens.server.model.LogSegregationContext; import org.apache.commons.lang.StringUtils; @@ -61,6 +62,8 @@ public class QueryServiceResource { private final ErrorCollection errorCollection; + private final LogSegregationContext logSegregationContext; + /** * Check session id. * @@ -123,6 +126,7 @@ public class QueryServiceResource { public QueryServiceResource() throws LensException { queryServer = (QueryExecutionService) LensServices.get().getService("query"); errorCollection = LensServices.get().getErrorCollection(); + logSegregationContext = LensServices.get().getLogSegregationContext(); } QueryExecutionService getQueryServer() { @@ -201,8 +205,9 @@ public class QueryServiceResource { @FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis, @DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException { - try { + final String requestId = this.logSegregationContext.get(); + try { validateSessionId(sessionid); SubmitOp sop = checkAndGetQuerySubmitOperation(operation); validateQuery(query); @@ -210,13 +215,13 @@ public class QueryServiceResource { QuerySubmitResult result; switch (sop) { case ESTIMATE: - result = queryServer.estimate(sessionid, query, conf); + result = queryServer.estimate(requestId, sessionid, query, conf); break; case EXECUTE: result = queryServer.executeAsync(sessionid, query, conf, queryName); break; case EXPLAIN: - result = queryServer.explain(sessionid, query, conf); + result = queryServer.explain(requestId, sessionid, query, conf); break; case EXECUTE_WITH_TIMEOUT: result = queryServer.execute(sessionid, query, timeoutmillis, conf, queryName); @@ -224,9 +229,10 @@ public class QueryServiceResource { default: throw new UnSupportedQuerySubmitOpException(); } - return LensResponse.composedOf(null, null, result); + + return LensResponse.composedOf(null, requestId, result); } catch (LensException e) { - e.buildLensErrorResponse(errorCollection, null, null); + e.buildLensErrorResponse(errorCollection, null, requestId); throw e; } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java index 7f5ed65..de7e120 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java @@ -28,6 +28,7 @@ import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.AsyncEventListener; import org.apache.lens.server.api.metrics.MetricsService; import org.apache.lens.server.api.query.*; +import org.apache.lens.server.model.LogSegregationContext; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.ReflectionUtils; +import lombok.NonNull; + /** * The Class ResultFormatter. */ @@ -46,13 +49,16 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> { /** The query service. */ QueryExecutionServiceImpl queryService; + private final LogSegregationContext logSegregationContext; + /** * Instantiates a new result formatter. * * @param queryService the query service */ - public ResultFormatter(QueryExecutionServiceImpl queryService) { + public ResultFormatter(QueryExecutionServiceImpl queryService, @NonNull LogSegregationContext logSegregationContext) { this.queryService = queryService; + this.logSegregationContext = logSegregationContext; } /* @@ -73,6 +79,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> { private void formatOutput(QueryExecuted event) { QueryHandle queryHandle = event.getQueryHandle(); QueryContext ctx = queryService.getQueryContext(queryHandle); + this.logSegregationContext.set(ctx.getQueryHandleString()); try { if (!ctx.isPersistent()) { LOG.info("No result formatting required for query " + queryHandle); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java index 131a375..81ad260 100644 --- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java +++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java @@ -21,6 +21,8 @@ package org.apache.lens.server.stats.store.log; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.metrics.MetricsService; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.stats.event.LoggableLensStatistics; import org.apache.lens.server.stats.store.StatisticsStore; @@ -53,11 +55,14 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics> /** The rollup handler. */ private StatisticsLogRollupHandler rollupHandler; + private final LogSegregationContext logSegregationContext; + /** * Instantiates a new log statistics store. */ public LogStatisticsStore() { this.mapper = new ObjectMapper(); + this.logSegregationContext = new MappedDiagnosticLogSegregationContext(); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); } @@ -71,7 +76,7 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics> handler = new StatisticsLogPartitionHandler(); handler.initialize(conf); LOG.info("Creating new rollup handler"); - rollupHandler = new StatisticsLogRollupHandler(); + rollupHandler = new StatisticsLogRollupHandler(this.logSegregationContext); rollupHandler.initialize(conf); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java index 254632b..7e9c74d 100644 --- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java +++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java @@ -23,18 +23,21 @@ import java.io.FilenameFilter; import java.util.HashMap; import java.util.Map; import java.util.TimerTask; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEventService; import org.apache.lens.server.api.metrics.MetricsService; +import org.apache.lens.server.model.LogSegregationContext; import org.apache.log4j.FileAppender; import org.apache.log4j.Logger; import org.slf4j.LoggerFactory; +import lombok.NonNull; import lombok.Setter; /** @@ -58,6 +61,11 @@ public class StatisticsLogFileScannerTask extends TimerTask { /** The class set. */ private Map<String, String> classSet = new ConcurrentHashMap<String, String>(); + private final LogSegregationContext logSegregationContext; + + public StatisticsLogFileScannerTask(@NonNull final LogSegregationContext logSegregationContext) { + this.logSegregationContext = logSegregationContext; + } /* * (non-Javadoc) * @@ -66,6 +74,10 @@ public class StatisticsLogFileScannerTask extends TimerTask { @Override public void run() { try { + + final String runId = UUID.randomUUID().toString(); + this.logSegregationContext.set(runId); + for (Map.Entry<String, String> entry : scanSet.entrySet()) { File f = new File(entry.getValue()).getAbsoluteFile(); String fileName = f.getAbsolutePath(); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java index 78e4518..9ff878c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java +++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java @@ -22,11 +22,14 @@ import java.util.Timer; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.events.LensEventService; +import org.apache.lens.server.model.LogSegregationContext; import org.apache.hadoop.conf.Configuration; import org.eclipse.jetty.util.ConcurrentHashSet; +import lombok.NonNull; + /** * Class which handles the log4j rolled file. */ @@ -44,13 +47,18 @@ public class StatisticsLogRollupHandler { /** The scan set. */ private final ConcurrentHashSet<String> scanSet = new ConcurrentHashSet<String>(); + private final LogSegregationContext logSegregationContext; + + public StatisticsLogRollupHandler(@NonNull final LogSegregationContext logSegregationContext) { + this.logSegregationContext = logSegregationContext; + } /** * Initalize the handler. * * @param conf configuration to be used while initialization. */ public void initialize(Configuration conf) { - task = new StatisticsLogFileScannerTask(); + task = new StatisticsLogFileScannerTask(this.logSegregationContext); timer = new Timer(); rate = conf.getLong(LensConfConstants.STATS_ROLLUP_SCAN_RATE, LensConfConstants.DEFAULT_STATS_ROLLUP_SCAN_RATE); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index 7913845..fb3e67b 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -30,6 +30,8 @@ import org.apache.lens.driver.hive.TestRemoteHiveDriver; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metrics.LensMetricsUtil; import org.apache.lens.server.api.metrics.MetricsService; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,6 +52,8 @@ public abstract class LensJerseyTest extends JerseyTest { private int port = -1; + private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext(); + protected URI getUri() { return UriBuilder.fromUri("http://localhost/";).port(getTestPort()).build(); } @@ -195,7 +199,7 @@ public abstract class LensJerseyTest extends JerseyTest { LensServices.get().stop(); LensMetricsUtil.clearRegistry(); System.out.println("Lens services stopped!"); - LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME)); + LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME, this.logSegregationContext)); LensServices.get().init(conf); LensServices.get().start(); System.out.println("Lens services restarted!"); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java index decd6ef..3fb123e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java @@ -18,6 +18,9 @@ */ package org.apache.lens.server; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -28,9 +31,11 @@ import org.testng.Assert; public class TestStartupOnMetastoreDown { private static final Log LOG = LogFactory.getLog(TestStartupOnMetastoreDown.class); + private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext(); + // @Test public void testServicesStartOnMetastoreDown() throws Exception { - LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME); + LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME, logSegregationContext); HiveConf hiveConf = new HiveConf(); // Set metastore uri to an invalid location http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java index 5e135be..4c2a7a4 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java @@ -45,6 +45,7 @@ import org.apache.lens.api.response.LensErrorTO; import org.apache.lens.api.response.LensJAXBContextResolver; import org.apache.lens.cube.error.ColUnAvailableInTimeRange; import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensRequestContextInitFilter; import org.apache.lens.server.common.ErrorResponseExpectedData; import org.apache.lens.server.error.LensExceptionMapper; import org.apache.lens.server.metastore.MetastoreResource; @@ -89,8 +90,8 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest { enable(TestProperties.LOG_TRAFFIC); enable(TestProperties.DUMP_ENTITY); - return new ResourceConfig(SessionResource.class, MetastoreResource.class, QueryServiceResource.class, - MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class); + return new ResourceConfig(LensRequestContextInitFilter.class, SessionResource.class, MetastoreResource.class, + QueryServiceResource.class, MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class); } @Override http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java index d5e60c4..819c62a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java +++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.lens.server.api.events.LensEventService; +import org.apache.lens.server.model.LogSegregationContext; +import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.stats.store.log.PartitionEvent; import org.apache.lens.server.stats.store.log.StatisticsLogFileScannerTask; @@ -52,6 +54,8 @@ public class TestStatisticsLogFileScannerTask { /** The hidden. */ private File hidden; + private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext(); + /** * Creates the test log file. * @@ -90,7 +94,7 @@ public class TestStatisticsLogFileScannerTask { appender.setName(TestStatisticsLogFileScannerTask.class.getSimpleName()); l.addAppender(appender); - StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask(); + StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask(this.logSegregationContext); task.addLogFile(TestStatisticsLogFileScannerTask.class.getName()); LensEventService service = Mockito.mock(LensEventService.class); final List<PartitionEvent> events = new ArrayList<PartitionEvent>(); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties index 65d065f..f3ccea2 100644 --- a/lens-server/src/test/resources/log4j.properties +++ b/lens-server/src/test/resources/log4j.properties @@ -31,11 +31,11 @@ log4j.additivity.org.apache.lens.server.stats.TestLogStatisticsStore$MyLoggableL # Configuration for appenders log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout -log4j.appender.STDOUT.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.STDOUT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.STDOUT.Threshold=ERROR log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender log4j.appender.TEST_LOG_FILE.File=lens-server/target/test.log log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout -log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n log4j.appender.TEST_LOG_FILE.Threshold=ERROR http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf-pseudo-distr/server/log4j.properties ---------------------------------------------------------------------- diff --git a/tools/conf-pseudo-distr/server/log4j.properties b/tools/conf-pseudo-distr/server/log4j.properties index afadc2f..2b04ea5 100644 --- a/tools/conf-pseudo-distr/server/log4j.properties +++ b/tools/conf-pseudo-distr/server/log4j.properties @@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender # CONSOLE uses PatternLayout. log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.ROOT=org.apache.log4j.RollingFileAppender log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout -log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.ROOT.MaxFileSize=100000KB # Keep 20 backup files @@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20 log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.AUDIT.MaxFileSize=100000KB # Keep 20 backup files @@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20 log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout -log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.REQUEST.MaxFileSize=100000KB # Keep 20 backup files @@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20 log4j.appender.STATUS=org.apache.log4j.RollingFileAppender log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout -log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.STATUS.MaxFileSize=100000KB # Keep 20 backup files http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf/server/log4j.properties ---------------------------------------------------------------------- diff --git a/tools/conf/server/log4j.properties b/tools/conf/server/log4j.properties index afadc2f..2b04ea5 100644 --- a/tools/conf/server/log4j.properties +++ b/tools/conf/server/log4j.properties @@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender # CONSOLE uses PatternLayout. log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.ROOT=org.apache.log4j.RollingFileAppender log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout -log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.ROOT.MaxFileSize=100000KB # Keep 20 backup files @@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20 log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.AUDIT.MaxFileSize=100000KB # Keep 20 backup files @@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20 log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout -log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.REQUEST.MaxFileSize=100000KB # Keep 20 backup files @@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20 log4j.appender.STATUS=org.apache.log4j.RollingFileAppender log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout -log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n +log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n log4j.appender.STATUS.MaxFileSize=100000KB # Keep 20 backup files
