Repository: hadoop Updated Branches: refs/heads/trunk 0a3aa40fe -> 871dc420f
YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong. (cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/871dc420 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/871dc420 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/871dc420 Branch: refs/heads/trunk Commit: 871dc420f8a4f151189c0925e062c64859a8f275 Parents: 0a3aa40 Author: Junping Du <junping...@apache.org> Authored: Tue Mar 14 12:56:54 2017 -0700 Committer: Junping Du <junping...@apache.org> Committed: Tue Mar 14 12:58:12 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 172 +++++++++++++------ .../hadoop/yarn/client/cli/TestLogsCLI.java | 31 ++++ .../yarn/logaggregation/LogCLIHelpers.java | 11 +- 3 files changed, 160 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 3cb1c7d..8407b19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -44,6 +44,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool { return false; } - private List<PerContainerLogFileInfo> getContainerLogFiles( + private List<Pair<PerContainerLogFileInfo, String>> getContainerLogFiles( Configuration conf, String containerIdStr, String nodeHttpAddress) throws IOException { - List<PerContainerLogFileInfo> logFileInfos = new ArrayList<>(); + List<Pair<PerContainerLogFileInfo, String>> logFileInfos + = new ArrayList<>(); Client webServiceClient = Client.create(); try { WebResource webResource = webServiceClient @@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool { } for (int i = 0; i < array.length(); i++) { JSONObject log = array.getJSONObject(i); + String aggregateType = log.has("logAggregationType") ? + log.getString("logAggregationType") : "N/A"; Object ob = log.get("containerLogInfo"); if (ob instanceof JSONArray) { JSONArray obArray = (JSONArray)ob; for (int j = 0; j < obArray.length(); j++) { - logFileInfos.add(generatePerContainerLogFileInfoFromJSON( - obArray.getJSONObject(j))); + logFileInfos.add(new Pair<PerContainerLogFileInfo, String>( + generatePerContainerLogFileInfoFromJSON( + obArray.getJSONObject(j)), aggregateType)); } } else if (ob instanceof JSONObject) { - logFileInfos.add(generatePerContainerLogFileInfoFromJSON( - (JSONObject)ob)); + logFileInfos.add(new Pair<PerContainerLogFileInfo, String>( + generatePerContainerLogFileInfoFromJSON( + (JSONObject)ob), aggregateType)); } } } catch (Exception e) { @@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool { IOUtils.closeQuietly(is); } } - // for the case, we have already uploaded partial logs in HDFS - int result = logCliHelper.dumpAContainerLogsForLogType( - newOptions, false); - if (result == 0 || foundAnyLogs) { + + if (foundAnyLogs) { return 0; } else { return -1; @@ -586,6 +591,19 @@ public class LogsCLI extends Configured implements Tool { newOptions); } + private int printAggregatedContainerLogs(ContainerLogsRequest request, + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { + return printContainerLogsForFinishedApplication(request, + logCliHelper, useRegex); + } + + private int printAggregatedContainerLogsWithoutNodeId( + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { + return printContainerLogsForFinishedApplicationWithoutNodeId(request, + logCliHelper, useRegex); + } + @Private @VisibleForTesting public ContainerReport getContainerReport(String containerIdStr) @@ -723,9 +741,10 @@ public class LogsCLI extends Configured implements Tool { } private int showContainerLogInfo(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException, YarnException { + LogCLIHelpers logCliHelper) throws IOException, YarnException, + ClientHandlerException, UniformInterfaceException, JSONException { if (!request.isAppFinished()) { - return printContainerInfoFromRunningApplication(request); + return printContainerInfoFromRunningApplication(request, logCliHelper); } else { return logCliHelper.printAContainerLogMetadata( request, System.out, System.err); @@ -900,7 +919,8 @@ public class LogsCLI extends Configured implements Tool { } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, + ClientHandlerException, UniformInterfaceException, JSONException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -941,14 +961,30 @@ public class LogsCLI extends Configured implements Tool { return printContainerLogsForFinishedApplicationWithoutNodeId( request, logCliHelper, useRegex); } else { - System.err.println("Unable to get logs for this container:" - + containerIdStr + "for the application:" + appIdStr - + " with the appOwner: " + appOwner); - System.err.println("The application: " + appIdStr - + " is still running, and we can not get Container report " - + "for the container: " + containerIdStr +". Please try later " - + "or after the application finishes."); - return -1; + nodeHttpAddress = getNodeHttpAddressFromRMWebString(request); + if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { + request.setNodeHttpAddress(nodeHttpAddress); + } else { + // for the case, we have already uploaded partial logs in HDFS + int result = -1; + if (nodeAddress != null && !nodeAddress.isEmpty()) { + result = printAggregatedContainerLogs( + request, logCliHelper, useRegex); + } else { + result = printAggregatedContainerLogsWithoutNodeId( + request, logCliHelper, useRegex); + } + if (result == -1) { + System.err.println("Unable to get logs for this container:" + + containerIdStr + " for the application:" + appIdStr + + " with the appOwner: " + appOwner); + System.err.println("The application: " + appIdStr + + " is still running, and we can not get Container report " + + "for the container: " + containerIdStr +". Please try later " + + "or after the application finishes."); + } + return result; + } } } // If the application is not in the final state, @@ -1144,7 +1180,9 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerInfoFromRunningApplication( - ContainerLogsRequest options) throws YarnException, IOException { + ContainerLogsRequest options, LogCLIHelpers logCliHelper) + throws YarnException, IOException, ClientHandlerException, + UniformInterfaceException, JSONException { String containerIdStr = options.getContainerId(); String nodeIdStr = options.getNodeId(); List<ContainerReport> reports = @@ -1152,54 +1190,75 @@ public class LogsCLI extends Configured implements Tool { List<ContainerReport> filteredReports = filterContainersInfo( options, reports); if (filteredReports.isEmpty()) { - StringBuilder sb = new StringBuilder(); - if (containerIdStr != null && !containerIdStr.isEmpty()) { - sb.append("Trying to get container with ContainerId: " - + containerIdStr + "\n"); + // if we specify the containerId as well as NodeAddress + String nodeHttpAddress = null; + if (options.getContainerId() != null + && !options.getContainerId().isEmpty()) { + nodeHttpAddress = getNodeHttpAddressFromRMWebString(options); } - if (nodeIdStr != null && !nodeIdStr.isEmpty()) { - sb.append("Trying to get container from NodeManager: " - + nodeIdStr + "\n"); + if (nodeHttpAddress != null) { + outputContainerLogMeta(options.getContainerId(), options.getNodeId(), + nodeHttpAddress); + return 0; + } else { + int result = logCliHelper.printAContainerLogMetadata( + options, System.out, System.err); + if (result == -1) { + StringBuilder sb = new StringBuilder(); + if (containerIdStr != null && !containerIdStr.isEmpty()) { + sb.append("Trying to get container with ContainerId: " + + containerIdStr + "\n"); + } + if (nodeIdStr != null && !nodeIdStr.isEmpty()) { + sb.append("Trying to get container from NodeManager: " + + nodeIdStr + "\n"); + } + sb.append("Can not find any matched containers for the application: " + + options.getAppId()); + System.err.println(sb.toString()); + } + return result; } - sb.append("Can not find any matched containers for the application: " - + options.getAppId()); - System.err.println(sb.toString()); - return -1; } for (ContainerReport report : filteredReports) { String nodeId = report.getAssignedNode().toString(); String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst( WebAppUtils.getHttpSchemePrefix(getConf()), ""); String containerId = report.getContainerId().toString(); - String containerString = String.format( - LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); - outStream.println(containerString); - outStream.println(StringUtils.repeat("=", containerString.length())); - outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, - "LogFile", "LogLength", "LastModificationTime"); - outStream.println(StringUtils.repeat("=", containerString.length())); - List<PerContainerLogFileInfo> infos = getContainerLogFiles( - getConf(), containerId, nodeHttpAddress); - for (PerContainerLogFileInfo info : infos) { - outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, - info.getFileName(), info.getFileSize(), - info.getLastModifiedTime()); - } + outputContainerLogMeta(containerId, nodeId, nodeHttpAddress); } return 0; } + private void outputContainerLogMeta(String containerId, String nodeId, + String nodeHttpAddress) throws IOException { + String containerString = String.format( + LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); + outStream.println(containerString); + outStream.println(StringUtils.repeat("=", containerString.length())); + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + "LogFile", "LogLength", "LastModificationTime", "LogAggregationType"); + outStream.println(StringUtils.repeat("=", containerString.length() * 2)); + List<Pair<PerContainerLogFileInfo, String>> infos = getContainerLogFiles( + getConf(), containerId, nodeHttpAddress); + for (Pair<PerContainerLogFileInfo, String> info : infos) { + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + info.getKey().getFileName(), info.getKey().getFileSize(), + info.getKey().getLastModifiedTime(), info.getValue()); + } + } + @VisibleForTesting public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request, boolean useRegex) throws IOException { // fetch all the log files for the container // filter the log files based on the given -log_files pattern - List<PerContainerLogFileInfo> allLogFileInfos= + List<Pair<PerContainerLogFileInfo, String>> allLogFileInfos= getContainerLogFiles(getConf(), request.getContainerId(), request.getNodeHttpAddress()); List<String> fileNames = new ArrayList<String>(); - for (PerContainerLogFileInfo fileInfo : allLogFileInfos) { - fileNames.add(fileInfo.getFileName()); + for (Pair<PerContainerLogFileInfo, String> fileInfo : allLogFileInfos) { + fileNames.add(fileInfo.getKey().getFileName()); } return getMatchedLogFiles(request, fileNames, useRegex); @@ -1217,4 +1276,17 @@ public class LogsCLI extends Configured implements Tool { .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); } + + @VisibleForTesting + public String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request) + throws ClientHandlerException, UniformInterfaceException, JSONException { + if (request.getNodeId() == null || request.getNodeId().isEmpty()) { + return null; + } + JSONObject nodeInfo = YarnWebServiceUtils + .getNodeInfoFromRMWebService(getConf(), request.getNodeId()) + .getJSONObject("node"); + return nodeInfo.has("nodeHTTPAddress") ? + nodeInfo.getString("nodeHTTPAddress") : null; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index ef164a5..05993d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -755,6 +755,23 @@ public class TestLogsCLI { Set<String> logTypes1 = capturedRequests.get(1).getLogTypes(); Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1)); Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1)); + + mockYarnClient = createMockYarnClientWithException( + YarnApplicationState.RUNNING, ugi.getShortUserName()); + LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient)); + doReturn(0).when(cli2).printContainerLogsFromRunningApplication( + any(Configuration.class), any(ContainerLogsRequest.class), + any(LogCLIHelpers.class), anyBoolean()); + doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString( + any(ContainerLogsRequest.class)); + cli2.setConf(new YarnConfiguration()); + ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); + exitCode = cli2.run(new String[] {"-applicationId", appId.toString(), + "-containerId", containerId100.toString(), "-nodeAddress", "NM:1234"}); + assertTrue(exitCode == 0); + verify(cli2, times(1)).printContainerLogsFromRunningApplication( + any(Configuration.class), logsRequestCaptor.capture(), + any(LogCLIHelpers.class), anyBoolean()); } @Test (timeout = 15000) @@ -1391,6 +1408,20 @@ public class TestLogsCLI { return mockClient; } + private YarnClient createMockYarnClientWithException( + YarnApplicationState appState, String user) + throws YarnException, IOException { + YarnClient mockClient = mock(YarnClient.class); + ApplicationReport mockAppReport = mock(ApplicationReport.class); + doReturn(user).when(mockAppReport).getUser(); + doReturn(appState).when(mockAppReport).getYarnApplicationState(); + doReturn(mockAppReport).when(mockClient).getApplicationReport( + any(ApplicationId.class)); + doThrow(new YarnException()).when(mockClient).getContainerReport( + any(ContainerId.class)); + return mockClient; + } + private YarnClient createMockYarnClientWithException() throws YarnException, IOException { YarnClient mockClient = mock(YarnClient.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 90b64ed..cf34a1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -51,7 +51,7 @@ import com.google.common.annotations.VisibleForTesting; public class LogCLIHelpers implements Configurable { public static final String PER_LOG_FILE_INFO_PATTERN = - "%30s\t%30s\t%30s" + System.getProperty("line.separator"); + "%30s\t%30s\t%30s\t%30s" + System.getProperty("line.separator"); public static final String CONTAINER_ON_NODE_PATTERN = "Container: %s on %s"; @@ -164,6 +164,7 @@ public class LogCLIHelpers implements Configurable { String containerString = String.format(CONTAINER_ON_NODE_PATTERN, containerId, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); // We have to re-create reader object to reset the stream index // after calling getContainerLogsStream which would move the stream @@ -238,6 +239,7 @@ public class LogCLIHelpers implements Configurable { String containerString = String.format(CONTAINER_ON_NODE_PATTERN, containerId, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, @@ -377,6 +379,7 @@ public class LogCLIHelpers implements Configurable { CONTAINER_ON_NODE_PATTERN, key, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { @@ -454,12 +457,12 @@ public class LogCLIHelpers implements Configurable { out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); out.printf(PER_LOG_FILE_INFO_PATTERN, "LogFile", "LogLength", - "LastModificationTime"); - out.println(StringUtils.repeat("=", containerString.length())); + "LastModificationTime", "LogAggregationType"); + out.println(StringUtils.repeat("=", containerString.length() * 2)); for (PerContainerLogFileInfo logMeta : containerLogMeta .getContainerLogMeta()) { out.printf(PER_LOG_FILE_INFO_PATTERN, logMeta.getFileName(), - logMeta.getFileSize(), logMeta.getLastModifiedTime()); + logMeta.getFileSize(), logMeta.getLastModifiedTime(), "AGGREGATED"); } } return 0; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
