STORM-1280 port backtype.storm.daemon.logviewer to java * address checkstyle violations * also addressed this from DRPC so far, but needs to decide for DRPC vs Drpc
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6a041a98 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6a041a98 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6a041a98 Branch: refs/heads/master Commit: 6a041a98fc7fe04db3f520fa283192d96ffb0a56 Parents: 7f84691 Author: Jungtaek Lim <[email protected]> Authored: Mon Jul 17 13:35:51 2017 +0900 Committer: Jungtaek Lim <[email protected]> Committed: Mon Jul 17 13:35:51 2017 +0900 ---------------------------------------------------------------------- pom.xml | 6 + storm-buildtools/storm_checkstyle.xml | 6 + storm-webapp/pom.xml | 2 +- .../daemon/common/JsonResponseBuilder.java | 14 +- .../daemon/drpc/webapp/DRPCApplication.java | 2 +- .../daemon/logviewer/LogviewerConstant.java | 8 +- .../storm/daemon/logviewer/LogviewerServer.java | 20 ++- .../handler/LogviewerLogDownloadHandler.java | 30 +++- .../handler/LogviewerLogPageHandler.java | 116 ++++++++++---- .../handler/LogviewerLogSearchHandler.java | 144 +++++++++++------ .../handler/LogviewerProfileHandler.java | 83 ++++++---- .../logviewer/utils/DirectoryCleaner.java | 51 ++++--- .../daemon/logviewer/utils/LogCleaner.java | 64 ++++---- .../logviewer/utils/LogFileDownloader.java | 18 ++- .../utils/LogviewerResponseBuilder.java | 59 +++++-- .../logviewer/utils/ResourceAuthorizer.java | 48 ++++-- .../daemon/logviewer/utils/WorkerLogs.java | 82 +++++++--- .../logviewer/webapp/LogviewerApplication.java | 28 ++-- .../logviewer/webapp/LogviewerResource.java | 77 ++++++++-- .../daemon/utils/ListFunctionalSupport.java | 53 ++++++- .../apache/storm/daemon/utils/StreamUtil.java | 9 ++ .../apache/storm/daemon/utils/URLBuilder.java | 43 ------ .../apache/storm/daemon/utils/UrlBuilder.java | 53 +++++++ .../storm/daemon/drpc/DRPCServerTest.java | 13 +- .../storm/daemon/logviewer/LogviewerTest.java | 51 ------- .../handler/LogviewerLogPageHandlerTest.java | 32 ++-- .../handler/LogviewerLogSearchHandlerTest.java | 153 ++++++++++--------- .../testsupport/ArgumentsVerifier.java | 15 +- .../testsupport/MockDirectoryBuilder.java | 12 +- .../logviewer/testsupport/MockFileBuilder.java | 12 +- .../daemon/logviewer/utils/LogCleanerTest.java | 61 ++++---- .../logviewer/utils/ResourceAuthorizerTest.java | 24 +-- .../daemon/logviewer/utils/WorkerLogsTest.java | 18 +-- 33 files changed, 912 insertions(+), 495 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ffee0cd..9d890d9 100644 --- a/pom.xml +++ b/pom.xml @@ -486,6 +486,12 @@ <exclude>**/derby.log</exclude> <exclude>**/src/dev/**</exclude> + <!-- webapp test resources --> + <exclude>**/src/test/resources/test-3072.log.test</exclude> + <exclude>**/src/test/resources/small-worker.log.test</exclude> + <exclude>**/src/test/resources/test-worker.log.test</exclude> + <exclude>**/src/test/resources/logviewer-search-context-tests.log.test</exclude> + <!-- StormSQL --> <exclude>**/src/codegen/config.fmpp</exclude> <exclude>**/src/codegen/data/Parser.tdd</exclude> http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-buildtools/storm_checkstyle.xml ---------------------------------------------------------------------- diff --git a/storm-buildtools/storm_checkstyle.xml b/storm-buildtools/storm_checkstyle.xml index 248e3ce..3b91be9 100644 --- a/storm-buildtools/storm_checkstyle.xml +++ b/storm-buildtools/storm_checkstyle.xml @@ -46,6 +46,9 @@ --> <module name = "Checker"> + <!-- Filter out Checkstyle warnings that have been suppressed with the @SuppressWarnings annotation --> + <module name="SuppressWarningsFilter" /> + <property name="charset" value="UTF-8"/> <property name="severity" value="warning"/> @@ -58,6 +61,9 @@ </module> <module name="TreeWalker"> + <!-- Make the @SuppressWarnings annotations available to Checkstyle --> + <module name="SuppressWarningsHolder" /> + <module name="OuterTypeFilename"/> <module name="IllegalTokenText"> <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/> http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/pom.xml ---------------------------------------------------------------------- diff --git a/storm-webapp/pom.xml b/storm-webapp/pom.xml index eeb6ee6..57ccb88 100644 --- a/storm-webapp/pom.xml +++ b/storm-webapp/pom.xml @@ -143,7 +143,7 @@ <artifactId>maven-checkstyle-plugin</artifactId> <!--Note - the version would be inherited--> <configuration> - <maxAllowedViolations>500</maxAllowedViolations> + <maxAllowedViolations>4</maxAllowedViolations> </configuration> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java index 2048d81..dc39bc6 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java @@ -18,12 +18,15 @@ package org.apache.storm.daemon.common; -import org.apache.storm.ui.UIHelpers; - -import javax.ws.rs.core.Response; import java.util.Collections; import java.util.Map; +import javax.ws.rs.core.Response; +import org.apache.storm.ui.UIHelpers; + +/** + * Response builder for JSON. It utilizes {@link UIHelpers} to construct JSON body and headers. + */ public class JsonResponseBuilder { private Object data; private String callback; @@ -56,6 +59,11 @@ public class JsonResponseBuilder { return this; } + /** + * Build a Response based on given parameters. + * + * @return A Response. + */ public Response build() { String body = UIHelpers.getJsonResponseBody(data, callback, needSerialize); Map<String, Object> respHeaders = UIHelpers.getJsonResponseHeaders(callback, headers); http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java index db42310..15cc4d7 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java @@ -24,8 +24,8 @@ import java.util.Set; import javax.ws.rs.ApplicationPath; import javax.ws.rs.core.Application; -import org.apache.storm.daemon.drpc.DRPC; import org.apache.storm.daemon.common.AuthorizationExceptionMapper; +import org.apache.storm.daemon.drpc.DRPC; @ApplicationPath("") public class DRPCApplication extends Application { http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java index 612e0e2..d1a15be 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java @@ -18,6 +18,12 @@ package org.apache.storm.daemon.logviewer; -public class LogviewerConstant { +/** + * Constants which are used across logviewer related classes. + */ +public final class LogviewerConstant { + private LogviewerConstant() { + } + public static final int DEFAULT_BYTES_PER_PAGE = 51200; } http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java index 1b53809..4f9834d 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java @@ -18,8 +18,17 @@ package org.apache.storm.daemon.logviewer; +import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES; + import com.codahale.metrics.Meter; import com.google.common.annotations.VisibleForTesting; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; import org.apache.storm.daemon.logviewer.utils.LogCleaner; @@ -40,14 +49,9 @@ import org.glassfish.jersey.servlet.ServletContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES; - +/** + * The main entry of Logviewer. + */ public class LogviewerServer implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class); private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls"); http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java index 7529297..7b2fbcb 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java @@ -18,28 +18,52 @@ package org.apache.storm.daemon.logviewer.handler; +import java.io.IOException; +import javax.ws.rs.core.Response; + import org.apache.storm.daemon.logviewer.utils.LogFileDownloader; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; import org.apache.storm.daemon.logviewer.utils.WorkerLogs; -import javax.ws.rs.core.Response; -import java.io.IOException; - public class LogviewerLogDownloadHandler { private WorkerLogs workerLogs; private final LogFileDownloader logFileDownloadHelper; + /** + * Constructor. + * + * @param logRoot root worker log directory + * @param daemonLogRoot root daemon log directory + * @param workerLogs {@link WorkerLogs} + * @param resourceAuthorizer {@link ResourceAuthorizer} + */ public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer) { this.workerLogs = workerLogs; this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer); } + /** + * Download an worker log. + * + * @param fileName file to download + * @param user username + * @return a Response which lets browsers download that file. + * @see {@link LogFileDownloader#downloadFile(String, String, boolean)} + */ public Response downloadLogFile(String fileName, String user) throws IOException { workerLogs.setLogFilePermission(fileName); return logFileDownloadHelper.downloadFile(fileName, user, false); } + /** + * Download a daemon log. + * + * @param fileName file to download + * @param user username + * @return a Response which lets browsers download that file. + * @see {@link LogFileDownloader#downloadFile(String, String, boolean)} + */ public Response downloadDaemonLogFile(String fileName, String user) throws IOException { return logFileDownloadHelper.downloadFile(fileName, user, true); } http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java index d2488f6..04b1a13 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java @@ -18,22 +18,26 @@ package org.apache.storm.daemon.logviewer.handler; +import static j2html.TagCreator.a; +import static j2html.TagCreator.body; +import static j2html.TagCreator.div; +import static j2html.TagCreator.form; +import static j2html.TagCreator.h3; +import static j2html.TagCreator.head; +import static j2html.TagCreator.html; +import static j2html.TagCreator.input; +import static j2html.TagCreator.link; +import static j2html.TagCreator.p; +import static j2html.TagCreator.pre; +import static j2html.TagCreator.select; +import static j2html.TagCreator.text; +import static j2html.TagCreator.title; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; + import j2html.TagCreator; import j2html.tags.DomContent; -import org.apache.commons.lang.StringUtils; -import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; -import org.apache.storm.daemon.logviewer.LogviewerConstant; -import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; -import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; -import org.apache.storm.daemon.logviewer.utils.WorkerLogs; -import org.apache.storm.daemon.utils.StreamUtil; -import org.apache.storm.daemon.utils.URLBuilder; -import org.apache.storm.ui.InvalidRequestException; -import org.apache.storm.ui.UIHelpers; -import org.apache.storm.utils.ConfigUtils; -import org.apache.storm.utils.ServerUtils; -import org.apache.storm.utils.Utils; -import org.jooq.lambda.Unchecked; import java.io.ByteArrayOutputStream; import java.io.File; @@ -54,10 +58,20 @@ import java.util.zip.GZIPInputStream; import javax.ws.rs.core.Response; -import static j2html.TagCreator.*; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toList; -import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; +import org.apache.commons.lang.StringUtils; +import org.apache.storm.daemon.logviewer.LogviewerConstant; +import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; +import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; +import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; +import org.apache.storm.daemon.logviewer.utils.WorkerLogs; +import org.apache.storm.daemon.utils.StreamUtil; +import org.apache.storm.daemon.utils.UrlBuilder; +import org.apache.storm.ui.InvalidRequestException; +import org.apache.storm.ui.UIHelpers; +import org.apache.storm.utils.ConfigUtils; +import org.apache.storm.utils.ServerUtils; +import org.apache.storm.utils.Utils; +import org.jooq.lambda.Unchecked; public class LogviewerLogPageHandler { private final String logRoot; @@ -65,6 +79,14 @@ public class LogviewerLogPageHandler { private final WorkerLogs workerLogs; private final ResourceAuthorizer resourceAuthorizer; + /** + * Constructor. + * + * @param logRoot root worker log directory + * @param daemonLogRoot root daemon log directory + * @param workerLogs {@link WorkerLogs} + * @param resourceAuthorizer {@link ResourceAuthorizer} + */ public LogviewerLogPageHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer) { @@ -74,6 +96,16 @@ public class LogviewerLogPageHandler { this.resourceAuthorizer = resourceAuthorizer; } + /** + * Enumerate worker log files for given criteria. + * + * @param user username + * @param port worker's port, null for all workers + * @param topologyId topology ID, null for all topologies + * @param callback callback for JSONP + * @param origin origin + * @return list of worker logs for given criteria + */ public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException { List<File> fileResults = null; if (topologyId == null) { @@ -130,7 +162,18 @@ public class LogviewerLogPageHandler { return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin); } - public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException { + /** + * Provides a worker log file to view. + * + * @param fileName file to view + * @param start start offset, can be null + * @param length length to read in this page, can be null + * @param grep search string if request is a result of the search, can be null + * @param user username + * @return HTML view page of worker log + */ + public Response logPage(String fileName, Integer start, Integer length, String grep, String user) + throws IOException, InvalidRequestException { String rootDir = logRoot; if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) { workerLogs.setLogFilePermission(fileName); @@ -141,8 +184,6 @@ public class LogviewerLogPageHandler { File topoDir = file.getParentFile().getParentFile(); if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) { - long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length(); - SortedSet<File> logFiles; try { logFiles = Arrays.stream(topoDir.listFiles()) @@ -169,6 +210,7 @@ public class LogviewerLogPageHandler { logString = escapeHtml("This is a binary file and cannot display! You may download the full file."); } + long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length(); start = start != null ? start : Long.valueOf(fileLength - length).intValue(); List<DomContent> bodyContents = new ArrayList<>(); @@ -209,15 +251,24 @@ public class LogviewerLogPageHandler { } } - public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException { + /** + * Provides a daemon log file to view. + * + * @param fileName file to view + * @param start start offset, can be null + * @param length length to read in this page, can be null + * @param grep search string if request is a result of the search, can be null + * @param user username + * @return HTML view page of daemon log + */ + public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) + throws IOException, InvalidRequestException { String rootDir = daemonLogRoot; File file = new File(rootDir, fileName).getCanonicalFile(); String path = file.getCanonicalPath(); boolean isZipFile = path.endsWith(".gz"); if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) { - long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length(); - // all types of files included List<File> logFiles = Arrays.stream(new File(rootDir).listFiles()) .filter(File::isFile) @@ -239,6 +290,7 @@ public class LogviewerLogPageHandler { logString = escapeHtml("This is a binary file and cannot display! You may download the full file."); } + long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length(); start = start != null ? start : Long.valueOf(fileLength - length).intValue(); List<DomContent> bodyContents = new ArrayList<>(); @@ -331,36 +383,36 @@ public class LogviewerLogPageHandler { } private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) { - int prevStart = Math.max(0, start - length); - int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length; - List<DomContent> btnLinks = new ArrayList<>(); - Map<String, Object> urlQueryParams = new HashMap<>(); urlQueryParams.put("file", fileName); urlQueryParams.put("start", Math.max(0, start - length)); urlQueryParams.put("length", length); - btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start)); + List<DomContent> btnLinks = new ArrayList<>(); + + int prevStart = Math.max(0, start - length); + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start)); urlQueryParams.clear(); urlQueryParams.put("file", fileName); urlQueryParams.put("start", 0); urlQueryParams.put("length", length); - btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First")); + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "First")); urlQueryParams.clear(); urlQueryParams.put("file", fileName); urlQueryParams.put("length", length); - btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last")); + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Last")); urlQueryParams.clear(); urlQueryParams.put("file", fileName); urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length)); urlQueryParams.put("length", length); - btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start)); + int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length; + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start)); return div(btnLinks.toArray(new DomContent[]{})); } http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java index 304c2d6..76f1683 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java @@ -18,29 +18,17 @@ package org.apache.storm.daemon.logviewer.handler; +import static java.util.stream.Collectors.toList; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.first; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.last; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.storm.DaemonConfig; -import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; -import org.apache.storm.daemon.common.JsonResponseBuilder; -import org.apache.storm.daemon.logviewer.LogviewerConstant; -import org.apache.storm.daemon.logviewer.utils.WorkerLogs; -import org.apache.storm.daemon.utils.StreamUtil; -import org.apache.storm.daemon.utils.URLBuilder; -import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; -import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; -import org.apache.storm.ui.InvalidRequestException; -import org.apache.storm.utils.ObjectReader; -import org.apache.storm.utils.ServerUtils; -import org.apache.storm.utils.Utils; -import org.json.simple.JSONAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.ws.rs.core.Response; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -58,15 +46,29 @@ import java.util.Map; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import static java.util.stream.Collectors.toList; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.first; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.last; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast; +import javax.ws.rs.core.Response; + +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.storm.DaemonConfig; +import org.apache.storm.daemon.common.JsonResponseBuilder; +import org.apache.storm.daemon.logviewer.LogviewerConstant; +import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; +import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; +import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; +import org.apache.storm.daemon.logviewer.utils.WorkerLogs; +import org.apache.storm.daemon.utils.StreamUtil; +import org.apache.storm.daemon.utils.UrlBuilder; +import org.apache.storm.ui.InvalidRequestException; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.ServerUtils; +import org.apache.storm.utils.Utils; +import org.json.simple.JSONAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LogviewerLogSearchHandler { - private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class); + private static final Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class); public static final int GREP_MAX_SEARCH_SIZE = 1024; public static final int GREP_BUF_SIZE = 2048; @@ -79,6 +81,14 @@ public class LogviewerLogSearchHandler { private final ResourceAuthorizer resourceAuthorizer; private final Integer logviewerPort; + /** + * Constructor. + * + * @param stormConf storm configuration + * @param logRoot log root directory + * @param daemonLogRoot daemon log root directory + * @param resourceAuthorizer {@link ResourceAuthorizer} + */ public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) { this.stormConf = stormConf; @@ -89,6 +99,19 @@ public class LogviewerLogSearchHandler { this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT)); } + /** + * Search from a worker log file. + * + * @param fileName log file + * @param user username + * @param isDaemon whether the log file is regarding worker or daemon + * @param search search string + * @param numMatchesStr the count of maximum matches + * @param offsetStr start offset for log file + * @param callback callback for JSONP + * @param origin origin + * @return Response containing JSON content representing search result + */ public Response searchLogFile(String fileName, String user, boolean isDaemon, String search, String numMatchesStr, String offsetStr, String callback, String origin) throws IOException, InvalidRequestException { @@ -130,6 +153,21 @@ public class LogviewerLogSearchHandler { return response; } + /** + * Deep search across worker log files in a topology. + * + * @param topologyId topology ID + * @param user username + * @param search search string + * @param numMatchesStr the count of maximum matches + * @param portStr worker port, null or '*' if the request wants to search from all worker logs + * @param fileOffsetStr index (offset) of the log files + * @param offsetStr start offset for log file + * @param searchArchived true if the request wants to search also archived files, false if not + * @param callback callback for JSONP + * @param origin origin + * @return Response containing JSON content representing search result + */ public Response deepSearchLogsForTopology(String topologyId, String user, String search, String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr, Boolean searchArchived, String callback, String origin) { @@ -178,8 +216,8 @@ public class LogviewerLogSearchHandler { if (!containsPort) { returnValue = new ArrayList<>(); } else { - File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId + - Utils.FILE_PATH_SEPARATOR + port); + File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId + + Utils.FILE_PATH_SEPARATOR + port); if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) { returnValue = new ArrayList<>(); @@ -218,22 +256,19 @@ public class LogviewerLogSearchHandler { } @VisibleForTesting - Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException { - return substringSearch(file, searchString, true, 10, 0); - } - - @VisibleForTesting Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException { return substringSearch(file, searchString, false, numMatches, startByteOffset); } - private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException { + private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, + Integer startByteOffset) throws InvalidRequestException { try { if (StringUtils.isEmpty(searchString)) { throw new IllegalArgumentException("Precondition fails: search string should not be empty."); } if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) { - throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE); + throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + + GREP_MAX_SEARCH_SIZE); } boolean isZipFile = file.getName().endsWith(".gz"); @@ -255,9 +290,8 @@ public class LogviewerLogSearchHandler { } ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE); - byte[] bufArray = buf.array(); - int totalBytesRead = 0; - byte[] searchBytes = searchString.getBytes("UTF-8"); + final byte[] bufArray = buf.array(); + final byte[] searchBytes = searchString.getBytes("UTF-8"); numMatches = numMatches != null ? numMatches : 10; startByteOffset = startByteOffset != null ? startByteOffset : 0; @@ -273,6 +307,7 @@ public class LogviewerLogSearchHandler { Arrays.fill(bufArray, (byte) 0); + int totalBytesRead = 0; int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE)); buf.limit(bytesRead); totalBytesRead += bytesRead; @@ -295,7 +330,7 @@ public class LogviewerLogSearchHandler { // The start index is positioned to find any possible // occurrence search string that did not quite fit in the // buffer on the previous read. - int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length; + final int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length; totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength); if (totalBytesRead < 0) { @@ -326,6 +361,10 @@ public class LogviewerLogSearchHandler { } } + @VisibleForTesting + Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException { + return substringSearch(file, searchString, true, 10, 0); + } /** * Get the filtered, authorized, sorted log files for a port. @@ -370,7 +409,7 @@ public class LogviewerLogSearchHandler { String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog); - List<Map<String, Object>> newMatches = new ArrayList<>(matches); + final List<Map<String, Object>> newMatches = new ArrayList<>(matches); Map<String, Object> currentFileMatch = new HashMap<>(theseMatches); currentFileMatch.put("fileName", fileName); List<String> splitPath; @@ -412,7 +451,8 @@ public class LogviewerLogSearchHandler { private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf, int initBufOffset, BufferedInputStream stream, Integer bytesSkipped, int bytesRead, ByteBuffer haystack, byte[] needle, - List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException { + List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) + throws IOException { int bufOffset = initBufOffset; List<Map<String, Object>> matches = initialMatches; @@ -422,8 +462,8 @@ public class LogviewerLogSearchHandler { while (true) { int offset = offsetOfBytes(haystack.array(), needle, bufOffset); if (matches.size() < numMatches && offset >= 0) { - int fileOffset = offsetToBuf + offset; - int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length; + final int fileOffset = offsetToBuf + offset; + final int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length; byte[] beforeArg = null; byte[] afterArg = null; @@ -541,7 +581,8 @@ public class LogviewerLogSearchHandler { * Tries once to read ahead in the stream to fill the context and * resets the stream to its position before the call. */ - private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException { + private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) + throws IOException { int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE); byte[] afterBytes = new byte[numExpected]; stream.mark(numExpected); @@ -617,7 +658,7 @@ public class LogviewerLogSearchHandler { @VisibleForTesting String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException { - String host = Utils.hostname(); + final String host = Utils.hostname(); String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR, takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3)); @@ -626,12 +667,12 @@ public class LogviewerLogSearchHandler { parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE); - return URLBuilder.build(String.format("http://%s:%d/api/v1/log";, host, port), parameters); + return UrlBuilder.build(String.format("http://%s:%d/api/v1/log";, host, port), parameters); } @VisibleForTesting String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException { - String host = Utils.hostname(); + final String host = Utils.hostname(); String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR, takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1)); @@ -640,7 +681,7 @@ public class LogviewerLogSearchHandler { parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE); - return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog";, host, port), parameters); + return UrlBuilder.build(String.format("http://%s:%d/api/v1/daemonlog";, host, port), parameters); } @VisibleForTesting @@ -651,6 +692,13 @@ public class LogviewerLogSearchHandler { private String searchString; private List<Map<String, Object>> matches; + /** + * Constructor. + * + * @param fileOffset offset (index) of the files + * @param searchString search string + * @param matches map representing matched search result + */ public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) { this.fileOffset = fileOffset; this.searchString = searchString; http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java index 39608da..d7487e7 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java @@ -18,18 +18,6 @@ package org.apache.storm.daemon.logviewer.handler; -import j2html.tags.DomContent; -import org.apache.commons.lang.StringUtils; -import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; -import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; -import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; -import org.apache.storm.utils.ServerUtils; - -import javax.ws.rs.core.Response; -import java.io.File; -import java.io.IOException; -import java.util.List; - import static j2html.TagCreator.a; import static j2html.TagCreator.body; import static j2html.TagCreator.head; @@ -40,17 +28,44 @@ import static j2html.TagCreator.title; import static j2html.TagCreator.ul; import static java.util.stream.Collectors.toList; +import j2html.tags.DomContent; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import javax.ws.rs.core.Response; + +import org.apache.commons.lang.StringUtils; +import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; +import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; +import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; +import org.apache.storm.utils.ServerUtils; + public class LogviewerProfileHandler { public static final String WORKER_LOG_FILENAME = "worker.log"; private final String logRoot; private final ResourceAuthorizer resourceAuthorizer; + /** + * Constructor. + * + * @param logRoot worker log root directory + * @param resourceAuthorizer {@link ResourceAuthorizer} + */ public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) { this.logRoot = logRoot; this.resourceAuthorizer = resourceAuthorizer; } + /** + * Enumerate dump (profile) files for given worker. + * + * @param topologyId topology ID + * @param hostPort host and port of worker + * @param user username + * @return The HTML page representing list page of dump files + */ public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException { String portStr = hostPort.split(":")[1]; File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr)); @@ -68,6 +83,33 @@ public class LogviewerProfileHandler { } } + /** + * Download a dump file. + * + * @param topologyId topology ID + * @param hostPort host and port of worker + * @param fileName dump file name + * @param user username + * @return a Response which lets browsers download that file. + * @see {@link org.apache.storm.daemon.logviewer.utils.LogFileDownloader#downloadFile(String, String, boolean)} + */ + public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException { + String portStr = hostPort.split(":")[1]; + File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr)); + File file = new File(dir, fileName); + + if (dir.exists() && file.exists()) { + String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME); + if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) { + return LogviewerResponseBuilder.buildDownloadFile(file); + } else { + return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user); + } + } else { + return LogviewerResponseBuilder.buildResponsePageNotFound(); + } + } + private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException { List<DomContent> liTags = getProfilerDumpFiles(dir).stream() .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file))) @@ -86,23 +128,6 @@ public class LogviewerProfileHandler { ).render(); } - public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException { - String portStr = hostPort.split(":")[1]; - File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr)); - File file = new File(dir, fileName); - - if (dir.exists() && file.exists()) { - String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME); - if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) { - return LogviewerResponseBuilder.buildDownloadFile(file); - } else { - return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user); - } - } else { - return LogviewerResponseBuilder.buildResponsePageNotFound(); - } - } - private List<String> getProfilerDumpFiles(File dir) throws IOException { List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir); return filesForDir.stream().filter(file -> { http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java index c8a4f69..7bbb8d7 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java @@ -15,19 +15,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.daemon.logviewer.utils; -import java.io.IOException; import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.DirectoryStream; -import java.util.Stack; import java.util.ArrayList; -import java.util.List; -import java.util.Set; import java.util.Comparator; +import java.util.List; import java.util.PriorityQueue; +import java.util.Set; +import java.util.Stack; import java.util.regex.Pattern; import org.slf4j.Logger; @@ -37,25 +38,35 @@ import org.slf4j.LoggerFactory; * Provide methods to help Logviewer to clean up * files in directories and to get a list of files without * worrying about excessive memory usage. - * */ public class DirectoryCleaner { private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class); // used to recognize the pattern of active log files, we may remove the "current" from this list private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid)$"); // used to recognize the pattern of some meta files in a worker log directory - private static final Pattern META_LOG_PATTERN= Pattern.compile(".*\\.(yaml|pid)$"); + private static final Pattern META_LOG_PATTERN = Pattern.compile(".*\\.(yaml|pid)$");// max number of files to delete for every round + + private static final int PQ_SIZE = 1024; + private static final int MAX_ROUNDS = 512; // max rounds of scanning the dirs + public static final int MAX_NUMBER_OF_FILES_FOR_DIR = 1024; // not defining this as static is to allow for mocking in tests + + /** + * Creates DirectoryStream for give directory. + * + * @param dir File instance representing specific directory + * @return DirectoryStream + */ public DirectoryStream<Path> getStreamForDirectory(File dir) throws IOException { - DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath()); - return stream; + return Files.newDirectoryStream(dir.toPath()); } /** * If totalSize of files exceeds the either the per-worker quota or global quota, * Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs. * We use the parameter forPerDir to switch between the two deletion modes. + * * @param dirs the list of directories to be scanned for deletion * @param quota the per-dir quota or the total quota for the all directories * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally @@ -64,8 +75,6 @@ public class DirectoryCleaner { */ public int deleteOldestWhileTooLarge(List<File> dirs, long quota, boolean forPerDir, Set<String> activeDirs) throws IOException { - final int PQ_SIZE = 1024; // max number of files to delete for every round - final int MAX_ROUNDS = 512; // max rounds of scanning the dirs long totalSize = 0; int deletedFiles = 0; @@ -133,12 +142,12 @@ public class DirectoryCleaner { round++; if (round >= MAX_ROUNDS) { if (forPerDir) { - LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " + - "a single directory : {}, will delete the rest files in next interval.", + LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " + + "a single directory : {}, will delete the rest files in next interval.", MAX_ROUNDS, dirs.get(0).getCanonicalPath()); } else { - LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " + - "will delete the rest files in next interval.", MAX_ROUNDS); + LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " + + "will delete the rest files in next interval.", MAX_ROUNDS); } break; } @@ -165,15 +174,19 @@ public class DirectoryCleaner { return false; } - // Note that to avoid memory problem, we only return the first 1024 files in a directory + /** + * Lists files in directory. + * Note that to avoid memory problem, we only return the first 1024 files in a directory. + * + * @param dir directory to get file list + * @return files in directory + */ public static List<File> getFilesForDir(File dir) throws IOException { List<File> files = new ArrayList<File>(); - final int MAX_NUM = 1024; - try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) { for (Path path : stream) { files.add(path.toFile()); - if (files.size() >= MAX_NUM) { + if (files.size() >= MAX_NUMBER_OF_FILES_FOR_DIR) { break; } } http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java index 6e4e073..bfd80a3 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java @@ -18,16 +18,16 @@ package org.apache.storm.daemon.logviewer.utils; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toList; + +import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS; +import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS; +import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB; +import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB; + import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.io.IOUtils; -import org.apache.storm.StormTimer; -import org.apache.storm.daemon.supervisor.SupervisorUtils; -import org.apache.storm.utils.ObjectReader; -import org.apache.storm.utils.Time; -import org.apache.storm.utils.Utils; -import org.jooq.lambda.Unchecked; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.File; @@ -41,23 +41,24 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.BinaryOperator; import java.util.stream.StreamSupport; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS; -import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS; -import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB; -import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB; +import org.apache.commons.io.IOUtils; +import org.apache.storm.StormTimer; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.Time; +import org.apache.storm.utils.Utils; +import org.jooq.lambda.Unchecked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * Cleans dead workers logs and directories. + */ public class LogCleaner implements Runnable, Closeable { private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class); @@ -71,6 +72,14 @@ public class LogCleaner implements Runnable, Closeable { private final long maxSumWorkerLogsSizeMb; private long maxPerWorkerLogsSizeMb; + /** + * Constuctor. + * + * @param stormConf configuration map for Storm cluster + * @param workerLogs {@link WorkerLogs} instance + * @param directoryCleaner {@link DirectoryCleaner} instance + * @param logRootDir root log directory + */ public LogCleaner(Map<String, Object> stormConf, WorkerLogs workerLogs, DirectoryCleaner directoryCleaner, File logRootDir) { this.stormConf = stormConf; @@ -87,6 +96,9 @@ public class LogCleaner implements Runnable, Closeable { maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb); } + /** + * Start log cleanup thread. + */ public void start() { if (intervalSecs != null) { LOG.debug("starting log cleanup thread at interval: {}", intervalSecs); @@ -100,9 +112,9 @@ public class LogCleaner implements Runnable, Closeable { } else { LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread."); } - } + @Override public void close() { if (logviewerCleanupTimer != null) { try { @@ -116,6 +128,7 @@ public class LogCleaner implements Runnable, Closeable { /** * Delete old log dirs for which the workers are no longer alive. */ + @Override public void run() { try { int nowSecs = Time.currentTimeSecs(); @@ -152,7 +165,8 @@ public class LogCleaner implements Runnable, Closeable { @VisibleForTesting List<Integer> perWorkerDirCleanup(long size) { return workerLogs.getAllWorkerDirs().stream() - .map(Unchecked.function(dir -> directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null))) + .map(Unchecked.function(dir -> + directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null))) .collect(toList()); } @@ -167,14 +181,6 @@ public class LogCleaner implements Runnable, Closeable { return directoryCleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs); } - private Set<String> getAliveIds(int nowSecs) throws Exception { - return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream() - .filter(entry -> Objects.nonNull(entry.getValue()) - && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf)) - .map(Map.Entry::getKey) - .collect(toCollection(TreeSet::new)); - } - /** * Delete the topo dir if it contains zero port dirs. */ http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java index 7d9cb50..62c08da 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java @@ -18,22 +18,38 @@ package org.apache.storm.daemon.logviewer.utils; -import javax.ws.rs.core.Response; import java.io.File; import java.io.IOException; +import javax.ws.rs.core.Response; + public class LogFileDownloader { private final String logRoot; private final String daemonLogRoot; private final ResourceAuthorizer resourceAuthorizer; + /** + * Constructor. + * + * @param logRoot root worker log directory + * @param daemonLogRoot root daemon log directory + * @param resourceAuthorizer {@link ResourceAuthorizer} + */ public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) { this.logRoot = logRoot; this.daemonLogRoot = daemonLogRoot; this.resourceAuthorizer = resourceAuthorizer; } + /** + * Checks authorization for the log file and download + * + * @param fileName file to download + * @param user username + * @param isDaemon true if the file is a daemon log, false if the file is an worker log + * @return a Response which lets browsers download that file. + */ public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException { String rootDir = isDaemon ? daemonLogRoot : logRoot; File file = new File(rootDir, fileName).getCanonicalFile(); http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java index d0f2857..59570ee 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java @@ -18,13 +18,12 @@ package org.apache.storm.daemon.logviewer.utils; -import com.google.common.io.ByteStreams; -import org.apache.storm.daemon.common.JsonResponseBuilder; -import org.apache.storm.ui.UIHelpers; +import static j2html.TagCreator.body; +import static j2html.TagCreator.h2; +import static javax.ws.rs.core.Response.Status.OK; +import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; +import com.google.common.io.ByteStreams; import java.io.BufferedOutputStream; import java.io.File; @@ -35,26 +34,46 @@ import java.io.OutputStream; import java.util.HashMap; import java.util.Map; -import static j2html.TagCreator.body; -import static j2html.TagCreator.h2; -import static javax.ws.rs.core.Response.Status.OK; -import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; + +import org.apache.storm.daemon.common.JsonResponseBuilder; +import org.apache.storm.ui.UIHelpers; public class LogviewerResponseBuilder { private LogviewerResponseBuilder() { } + /** + * Build a Response object representing success response with HTML entity. + * + * @param content HTML entity content, String type + */ public static Response buildSuccessHtmlResponse(String content) { return Response.status(OK).entity(content) .type(MediaType.TEXT_HTML_TYPE).build(); } + /** + * Build a Response object representing success response with JSON entity. + * + * @param entity entity object to represent it as JSON + * @param callback callback for JSONP + * @param origin origin + * @see {@link JsonResponseBuilder} + */ public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) { return new JsonResponseBuilder().setData(entity).setCallback(callback) .setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build(); } + /** + * Build a Response object representing download a file. + * + * @param file file to download + */ public static Response buildDownloadFile(File file) throws IOException { // do not close this InputStream in method: it will be used from jetty server InputStream is = new FileInputStream(file); @@ -65,6 +84,11 @@ public class LogviewerResponseBuilder { .build(); } + /** + * Build a Response object representing unauthorized user, with HTML response. + * + * @param user username + */ public static Response buildResponseUnautohrizedUser(String user) { String entity = buildUnauthorizedUserHtml(user); return Response.status(OK) @@ -73,6 +97,9 @@ public class LogviewerResponseBuilder { .build(); } + /** + * Build a Response object representing page not found. + */ public static Response buildResponsePageNotFound() { return Response.status(404) .entity("Page not found") @@ -80,11 +107,23 @@ public class LogviewerResponseBuilder { .build(); } + /** + * Build a Response object representing unauthorized user, with JSON response. + * + * @param user username + * @param callback callback for JSONP + */ public static Response buildUnauthorizedUserJsonResponse(String user, String callback) { return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user)) .setCallback(callback).setStatus(401).build(); } + /** + * Build a Response object representing exception, with JSON response. + * + * @param ex Exception object + * @param callback callback for JSONP + */ public static Response buildExceptionJsonResponse(Exception ex, String callback) { return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex)) .setCallback(callback).setStatus(500).build(); http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java index 55691b2..16f408a 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java @@ -18,16 +18,10 @@ package org.apache.storm.daemon.logviewer.utils; +import static org.apache.storm.DaemonConfig.UI_FILTER; + import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; -import org.apache.storm.Config; -import org.apache.storm.DaemonConfig; -import org.apache.storm.security.auth.AuthUtils; -import org.apache.storm.security.auth.IGroupMappingServiceProvider; -import org.apache.storm.utils.ObjectReader; -import org.apache.storm.utils.ServerConfigUtils; -import org.apache.storm.utils.Utils; import java.io.File; import java.io.IOException; @@ -37,28 +31,51 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.apache.storm.DaemonConfig.UI_FILTER; +import org.apache.commons.lang.StringUtils; +import org.apache.storm.Config; +import org.apache.storm.DaemonConfig; +import org.apache.storm.security.auth.AuthUtils; +import org.apache.storm.security.auth.IGroupMappingServiceProvider; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.ServerConfigUtils; +import org.apache.storm.utils.Utils; public class ResourceAuthorizer { private final Map<String, Object> stormConf; private final IGroupMappingServiceProvider groupMappingServiceProvider; + /** + * Constuctor. + * + * @param stormConf storm configuration + */ public ResourceAuthorizer(Map<String, Object> stormConf) { this.stormConf = stormConf; this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf); } + /** + * Checks whether user is allowed to access file via UI. Always true when UI filter is not set. + * + * @param fileName file name to access + * @param user username + */ public boolean isUserAllowedToAccessFile(String fileName, String user) { return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName); } + /** + * Checks whether user is authorized to access file. Checks regardless of UI filter. + * + * @param user username + * @param fileName file name to access + */ public boolean isAuthorizedLogUser(String user, String fileName) { if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName) || getLogUserGroupWhitelist(fileName) == null) { return false; } else { - Set<String> groups = getUserGroups(user); LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName); List<String> logsUsers = new ArrayList<>(); @@ -70,11 +87,18 @@ public class ResourceAuthorizer { logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS))); logsGroups.addAll(whitelist.getGroupWhitelist()); - return logsUsers.stream().anyMatch(u -> u.equals(user)) || - Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0; + Set<String> groups = getUserGroups(user); + + return logsUsers.stream().anyMatch(u -> u.equals(user)) + || Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0; } } + /** + * Get the whitelist of users and groups for given file. + * + * @param fileName file name to get the whitelist + */ public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) { File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName); Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java index 6018b79..eba6404 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java @@ -18,16 +18,13 @@ package org.apache.storm.daemon.logviewer.utils; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toMap; +import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER; +import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER; +import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast; + import com.google.common.collect.Lists; -import org.apache.storm.daemon.supervisor.ClientSupervisorUtils; -import org.apache.storm.daemon.supervisor.SupervisorUtils; -import org.apache.storm.utils.ObjectReader; -import org.apache.storm.utils.Time; -import org.apache.storm.utils.Utils; -import org.jooq.lambda.Unchecked; -import org.jooq.lambda.tuple.Tuple2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -43,12 +40,19 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.stream.Stream; -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toMap; -import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER; -import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER; -import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast; +import org.apache.storm.daemon.supervisor.ClientSupervisorUtils; +import org.apache.storm.daemon.supervisor.SupervisorUtils; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.Time; +import org.apache.storm.utils.Utils; +import org.jooq.lambda.Unchecked; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * A class that knows about how to operate with worker log directory. + */ public class WorkerLogs { private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class); @@ -56,19 +60,30 @@ public class WorkerLogs { private final Map<String, Object> stormConf; private final File logRootDir; + /** + * Constructor. + * + * @param stormConf storm configuration + * @param logRootDir the log root directory + */ public WorkerLogs(Map<String, Object> stormConf, File logRootDir) { this.stormConf = stormConf; this.logRootDir = logRootDir; } + /** + * Set permission of log file so that logviewer can serve the file. + * + * @param fileName log file + */ public void setLogFilePermission(String fileName) throws IOException { File file = new File(logRootDir, fileName).getCanonicalFile(); boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), false); File parent = new File(logRootDir, fileName).getParentFile(); Optional<File> mdFile = (parent == null) ? Optional.empty() : getMetadataFileForWorkerLogDir(parent); - Optional<String> topoOwner = mdFile.isPresent() ? - Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath())) : - Optional.empty(); + Optional<String> topoOwner = mdFile.isPresent() + ? Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath())) + : Optional.empty(); if (runAsUser && topoOwner.isPresent() && file.exists() && !Files.isReadable(file.toPath())) { LOG.debug("Setting permissions on file {} with topo-owner {}", fileName, topoOwner); @@ -78,6 +93,9 @@ public class WorkerLogs { } } + /** + * Return a list of all log files from worker directories in root log directory. + */ public List<File> getAllLogsForRootDir() throws IOException { List<File> files = new ArrayList<>(); Set<File> topoDirFiles = getAllWorkerDirs(); @@ -90,6 +108,9 @@ public class WorkerLogs { return files; } + /** + * Return a set of all worker directories in root log directory. + */ public Set<File> getAllWorkerDirs() { File[] rootDirFiles = logRootDir.listFiles(); if (rootDirFiles != null) { @@ -116,6 +137,10 @@ public class WorkerLogs { .collect(toCollection(TreeSet::new)); } + /** + * Return a metadata file (worker.yaml) for given worker log directory. + * @param logDir worker log directory + */ public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException { File metaFile = new File(logDir, WORKER_YAML); if (metaFile.exists()) { @@ -126,16 +151,31 @@ public class WorkerLogs { } } + /** + * Return worker id from worker meta file. + * + * @param metaFile metadata file + */ public String getWorkerIdFromMetadataFile(String metaFile) { Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile); return ObjectReader.getString(map.get("worker-id"), null); } + /** + * Return topology owner from worker meta file. + * + * @param metaFile metadata file + */ public String getTopologyOwnerFromMetadataFile(String metaFile) { Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile); return ObjectReader.getString(map.get(TOPOLOGY_SUBMITTER_USER), null); } + /** + * Retrieve the set of alive worker IDs. + * + * @param nowSecs current time in seconds + */ public Set<String> getAliveIds(int nowSecs) throws Exception { return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream() .filter(entry -> Objects.nonNull(entry.getValue()) @@ -144,6 +184,12 @@ public class WorkerLogs { .collect(toCollection(TreeSet::new)); } + /** + * Finds a worker ID for each directory in set and return it as map. + * + * @param logDirs directories to check whether they're worker directories or not + * @return the pair of worker ID, directory. worker ID will be an empty string if the directory is not a worker directory. + */ public Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) { // we could also make this static, but not to do it due to mock return logDirs.stream().map(Unchecked.function(logDir -> { @@ -156,6 +202,8 @@ public class WorkerLogs { /** * Return the path of the worker log with the format of topoId/port/worker.log.* + * + * @param file worker log */ public static String getTopologyPortWorkerLog(File file) { try { http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java index 32a723e..c44e8cd 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java @@ -18,31 +18,32 @@ package org.apache.storm.daemon.logviewer.webapp; +import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME; + +import java.io.File; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.ws.rs.ApplicationPath; +import javax.ws.rs.core.Application; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.storm.daemon.common.AuthorizationExceptionMapper; +import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler; +import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler; import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler; import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; -import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler; -import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler; import org.apache.storm.daemon.logviewer.utils.WorkerLogs; import org.apache.storm.security.auth.AuthUtils; import org.apache.storm.security.auth.IHttpCredentialsPlugin; import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.ObjectReader; -import javax.ws.rs.ApplicationPath; -import javax.ws.rs.core.Application; -import java.io.File; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME; - @ApplicationPath("") public class LogviewerApplication extends Application { private static Map<String, Object> stormConf; @@ -75,6 +76,11 @@ public class LogviewerApplication extends Application { return singletons; } + /** + * Spot to inject storm configuration before initializing LogviewerApplication instance. + * + * @param stormConf storm configuration + */ public static void setup(Map<String, Object> stormConf) { LogviewerApplication.stormConf = stormConf; } http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java ---------------------------------------------------------------------- diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java index bc6529c..478001d 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java @@ -19,13 +19,25 @@ package org.apache.storm.daemon.logviewer.webapp; import com.codahale.metrics.Meter; + +import java.io.IOException; +import java.net.URLDecoder; +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; + import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.apache.storm.daemon.common.JsonResponseBuilder; -import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler; -import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler; import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler; import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler; +import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler; +import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.security.auth.IHttpCredentialsPlugin; import org.apache.storm.ui.InvalidRequestException; @@ -33,24 +45,20 @@ import org.apache.storm.ui.UIHelpers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.net.URLDecoder; -import java.util.Map; - +/** + * Handles HTTP requests for Logviewer. + */ @Path("/") public class LogviewerResource { private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class); private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests"); - private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests"); - private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests"); - private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests"); + private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter( + "logviewer:num-daemonlog-page-http-requests"); + private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter( + "logviewer:num-download-log-file-http-requests"); + private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter( + "logviewer:num-download-log-daemon-file-http-requests"); private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests"); private final LogviewerLogPageHandler logviewer; @@ -59,6 +67,15 @@ public class LogviewerResource { private final LogviewerLogSearchHandler logSearchHandler; private final IHttpCredentialsPlugin httpCredsHandler; + /** + * Constructor. + * + * @param logviewerParam {@link LogviewerLogPageHandler} + * @param profileHandler {@link LogviewerProfileHandler} + * @param logDownloadHandler {@link LogviewerLogDownloadHandler} + * @param logSearchHandler {@link LogviewerLogSearchHandler} + * @param httpCredsHandler {@link IHttpCredentialsPlugin} + */ public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler, LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler, IHttpCredentialsPlugin httpCredsHandler) { @@ -69,6 +86,9 @@ public class LogviewerResource { this.httpCredsHandler = httpCredsHandler; } + /** + * Handles '/log' request. + */ @GET @Path("/log") public Response log(@Context HttpServletRequest request) throws IOException { @@ -87,6 +107,9 @@ public class LogviewerResource { } } + /** + * Handles '/daemonlog' request. + */ @GET @Path("/daemonlog") public Response daemonLog(@Context HttpServletRequest request) throws IOException { @@ -105,6 +128,9 @@ public class LogviewerResource { } } + /** + * Handles '/searchLogs' request. + */ @GET @Path("/searchLogs") public Response searchLogs(@Context HttpServletRequest request) throws IOException { @@ -117,6 +143,9 @@ public class LogviewerResource { return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin); } + /** + * Handles '/listLogs' request. + */ @GET @Path("/listLogs") public Response listLogs(@Context HttpServletRequest request) throws IOException { @@ -131,6 +160,9 @@ public class LogviewerResource { return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin); } + /** + * Handles '/dumps' (listing dump files) request. + */ @GET @Path("/dumps/{topo-id}/{host-port}") public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort, @@ -139,6 +171,9 @@ public class LogviewerResource { return profileHandler.listDumpFiles(topologyId, hostPort, user); } + /** + * Handles '/dumps' (downloading specific dump file) request. + */ @GET @Path("/dumps/{topo-id}/{host-port}/{filename}") public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort, @@ -147,6 +182,9 @@ public class LogviewerResource { return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user); } + /** + * Handles '/download' (downloading specific log file) request. + */ @GET @Path("/download") public Response downloadLogFile(@Context HttpServletRequest request) throws IOException { @@ -158,6 +196,9 @@ public class LogviewerResource { return logDownloadHandler.downloadLogFile(decodedFileName, user); } + /** + * Handles '/daemondownload' (downloading specific daemon log file) request. + */ @GET @Path("/daemondownload") public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException { @@ -169,6 +210,9 @@ public class LogviewerResource { return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user); } + /** + * Handles '/search' (searching from specific worker or daemon log file) request. + */ @GET @Path("/search/{file}") public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException { @@ -191,6 +235,9 @@ public class LogviewerResource { } } + /** + * Handles '/deepSearch' request. + */ @GET @Path("/deepSearch/{topoId}") public Response deepSearch(@PathParam("topoId") String topologyId,
