Repository: hadoop Updated Branches: refs/heads/branch-2 694b4abbc -> 4a6cef8c9
YARN-5200. Enhanced "yarn logs" to be able to get a list of containers whose logs are aggregated via a "show_container_log_info" option. Contributed by Xuan Gong. (cherry picked from commit eb471632349deac4b62f8dec853c8ceb64c9617a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a6cef8c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a6cef8c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a6cef8c Branch: refs/heads/branch-2 Commit: 4a6cef8c910548251bec336147ee4be65736102a Parents: 694b4ab Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Wed Jul 13 10:47:43 2016 -0700 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Wed Jul 13 10:55:18 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 321 ++++++++++++++----- .../hadoop/yarn/client/cli/TestLogsCLI.java | 85 +++-- .../logaggregation/AggregatedLogFormat.java | 13 +- .../yarn/logaggregation/LogCLIHelpers.java | 97 +++++- 4 files changed, 392 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6cef8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 7194cd0..ce26bd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -78,14 +78,30 @@ public class LogsCLI extends Configured implements Tool { private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String CONTAINER_LOG_FILES = "logFiles"; - private static final String SHOW_META_INFO = "show_meta_info"; private static final String LIST_NODES_OPTION = "list_nodes"; + private static final String SHOW_APPLICATION_LOG_INFO + = "show_application_log_info"; + private static final String SHOW_CONTAINER_LOG_INFO + = "show_container_log_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; public static final String HELP_CMD = "help"; + private PrintStream outStream = System.out; + private YarnClient yarnClient = null; @Override public int run(String[] args) throws Exception { + try { + yarnClient = createYarnClient(); + return runCommand(args); + } finally { + if (yarnClient != null) { + yarnClient.close(); + } + } + } + + private int runCommand(String[] args) throws Exception { Options opts = createCommandOpts(); Options printOpts = createPrintOpts(opts); if (args.length < 1) { @@ -102,8 +118,9 @@ public class LogsCLI extends Configured implements Tool { String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; - boolean showMetaInfo = false; boolean nodesList = false; + boolean showApplicationLogInfo = false; + boolean showContainerLogInfo = false; String[] logFiles = null; List<String> amContainersList = new ArrayList<String>(); String localDir = null; @@ -115,9 +132,11 @@ public class LogsCLI extends Configured implements Tool { nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); - showMetaInfo = commandLine.hasOption(SHOW_META_INFO); nodesList = commandLine.hasOption(LIST_NODES_OPTION); localDir = commandLine.getOptionValue(OUT_OPTION); + showApplicationLogInfo = commandLine.hasOption( + SHOW_APPLICATION_LOG_INFO); + showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -172,6 +191,12 @@ public class LogsCLI extends Configured implements Tool { } } + if (showApplicationLogInfo && showContainerLogInfo) { + System.err.println("Invalid options. Can only accept one of " + + "show_application_log_info/show_container_log_info."); + return -1; + } + LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); @@ -215,14 +240,17 @@ public class LogsCLI extends Configured implements Tool { isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes); - if (showMetaInfo) { - return showMetaInfo(request, logCliHelper); + if (showContainerLogInfo) { + return showContainerLogInfo(request, logCliHelper); } if (nodesList) { return showNodeLists(request, logCliHelper); } + if (showApplicationLogInfo) { + return showApplicationLogInfo(request, logCliHelper); + } // To get am logs if (getAMContainerLogs) { return fetchAMContainerLogs(request, amContainersList, @@ -246,21 +274,15 @@ public class LogsCLI extends Configured implements Tool { private ApplicationReport getApplicationReport(ApplicationId appId) throws IOException, YarnException { - YarnClient yarnClient = createYarnClient(); - - try { - return yarnClient.getApplicationReport(appId); - } finally { - yarnClient.close(); - } + return yarnClient.getApplicationReport(appId); } @VisibleForTesting protected YarnClient createYarnClient() { - YarnClient yarnClient = YarnClient.createYarnClient(); - yarnClient.init(getConf()); - yarnClient.start(); - return yarnClient; + YarnClient client = YarnClient.createYarnClient(); + client.init(getConf()); + client.start(); + return client; } public static void main(String[] args) throws Exception { @@ -272,7 +294,7 @@ public class LogsCLI extends Configured implements Tool { } private void printHelpMessage(Options options) { - System.out.println("Retrieve logs for completed YARN applications."); + outStream.println("Retrieve logs for YARN applications."); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]", new Options()); @@ -336,9 +358,9 @@ public class LogsCLI extends Configured implements Tool { return false; } - private List<String> getContainerLogFiles(Configuration conf, + private List<PerLogFileInfo> getContainerLogFiles(Configuration conf, String containerIdStr, String nodeHttpAddress) throws IOException { - List<String> logFiles = new ArrayList<>(); + List<PerLogFileInfo> logFileInfos = new ArrayList<>(); Client webServiceClient = Client.create(); try { WebResource webResource = webServiceClient @@ -355,7 +377,9 @@ public class LogsCLI extends Configured implements Tool { response.getEntity(JSONObject.class); JSONArray array = json.getJSONArray("containerLogInfo"); for (int i = 0; i < array.length(); i++) { - logFiles.add(array.getJSONObject(i).getString("fileName")); + String fileName = array.getJSONObject(i).getString("fileName"); + String fileSize = array.getJSONObject(i).getString("fileSize"); + logFileInfos.add(new PerLogFileInfo(fileName, fileSize)); } } catch (Exception e) { System.err.println("Unable to parse json from webservice. Error:"); @@ -368,7 +392,7 @@ public class LogsCLI extends Configured implements Tool { System.err.println("Unable to fetch log files list"); throw new IOException(ex); } - return logFiles; + return logFileInfos; } @Private @@ -379,15 +403,26 @@ public class LogsCLI extends Configured implements Tool { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); String nodeHttpAddress = request.getNodeHttpAddress(); + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + System.err.println("Can not get the logs for the container: " + + containerIdStr); + System.err.println("The node http address is required to get container " + + "logs for the Running application."); + return -1; + } String nodeId = request.getNodeId(); PrintStream out = logCliHelper.createPrintStream(localDir, nodeId, containerIdStr); try { // fetch all the log files for the container // filter the log files based on the given --logFiles pattern - List<String> allLogs= + List<PerLogFileInfo> allLogFileInfos= getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress); - List<String> matchedFiles = getMatchedLogFiles(request, allLogs); + List<String> fileNames = new ArrayList<String>(); + for (PerLogFileInfo fileInfo : allLogFileInfos) { + fileNames.add(fileInfo.getFileName()); + } + List<String> matchedFiles = getMatchedLogFiles(request, fileNames); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + containerIdStr @@ -398,8 +433,8 @@ public class LogsCLI extends Configured implements Tool { newOptions.setLogTypes(matchedFiles); Client webServiceClient = Client.create(); - String containerString = "\n\nContainer: " + containerIdStr + " on " - + nodeId; + String containerString = String.format( + LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerIdStr, nodeId); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); boolean foundAnyLogs = false; @@ -478,13 +513,8 @@ public class LogsCLI extends Configured implements Tool { @VisibleForTesting public ContainerReport getContainerReport(String containerIdStr) throws YarnException, IOException { - YarnClient yarnClient = createYarnClient(); - try { - return yarnClient.getContainerReport( - ContainerId.fromString(containerIdStr)); - } finally { - yarnClient.close(); - } + return yarnClient.getContainerReport( + ContainerId.fromString(containerIdStr)); } private boolean isApplicationFinished(YarnApplicationState appState) { @@ -509,8 +539,10 @@ public class LogsCLI extends Configured implements Tool { for (JSONObject amContainer : amContainersList) { ContainerLogsRequest amRequest = new ContainerLogsRequest(request); amRequest.setContainerId(amContainer.getString("containerId")); - amRequest.setNodeHttpAddress( - amContainer.getString("nodeHttpAddress")); + String httpAddress = amContainer.getString("nodeHttpAddress"); + if (httpAddress != null && !httpAddress.isEmpty()) { + amRequest.setNodeHttpAddress(httpAddress); + } amRequest.setNodeId(amContainer.getString("nodeId")); requests.add(amRequest); } @@ -546,8 +578,8 @@ public class LogsCLI extends Configured implements Tool { for (ContainerLogsRequest amRequest : requests) { outputAMContainerLogs(amRequest, conf, logCliHelper); } - System.out.println(); - System.out.println("Specified ALL for -am option. " + outStream.println(); + outStream.println("Specified ALL for -am option. " + "Printed logs for all am containers."); } else { for (String amContainer : amContainers) { @@ -603,15 +635,13 @@ public class LogsCLI extends Configured implements Tool { } } - private int showMetaInfo(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException { + private int showContainerLogInfo(ContainerLogsRequest request, + LogCLIHelpers logCliHelper) throws IOException, YarnException { if (!request.isAppFinished()) { - System.err.println("The -show_meta_info command can be only used " - + "with finished applications"); - return -1; + return printContainerInfoFromRunningApplication(request); } else { - logCliHelper.printLogMetadata(request, System.out, System.err); - return 0; + return logCliHelper.printAContainerLogMetadata( + request, System.out, System.err); } } @@ -627,6 +657,33 @@ public class LogsCLI extends Configured implements Tool { } } + private int showApplicationLogInfo(ContainerLogsRequest request, + LogCLIHelpers logCliHelper) throws IOException, YarnException { + String appState = "Application State: " + + (request.isAppFinished() ? "Completed." : "Running."); + if (!request.isAppFinished()) { + List<ContainerReport> reports = + getContainerReportsFromRunningApplication(request); + List<ContainerReport> filterReports = filterContainersInfo( + request, reports); + if (filterReports.isEmpty()) { + System.err.println("Can not find any containers for the application:" + + request.getAppId() + "."); + return -1; + } + outStream.println(appState); + for (ContainerReport report : filterReports) { + outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, + report.getContainerId(), report.getAssignedNode())); + } + return 0; + } else { + outStream.println(appState); + logCliHelper.printContainersList(request, System.out, System.err); + return 0; + } + } + private Options createCommandOpts() { Options opts = new Options(); opts.addOption(HELP_CMD, false, "Displays help for all commands."); @@ -662,13 +719,16 @@ public class LogsCLI extends Configured implements Tool { logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); - opts.addOption(SHOW_META_INFO, false, "Show the log metadata, " + opts.addOption(SHOW_CONTAINER_LOG_INFO, false, + "Show the container log metadata, " + "including log-file names, the size of the log files. " + "You can combine this with --containerId to get log metadata for " + "the specific container, or with --nodeAddress to get log metadata " - + "for all the containers on the specific NodeManager. " - + "Currently, this option can only be used for finished " - + "applications."); + + "for all the containers on the specific NodeManager."); + opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the " + + "containerIds which belong to the specific Application. " + + "You can combine this with --nodeAddress to get containerIds " + + "for all the containers on the specific NodeManager."); opts.addOption(LIST_NODES_OPTION, false, "Show the list of nodes that successfully aggregated logs. " + "This option can only be used with finished applications."); @@ -696,8 +756,9 @@ public class LogsCLI extends Configured implements Tool { printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION)); printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES)); - printOpts.addOption(commandOpts.getOption(SHOW_META_INFO)); printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION)); + printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO)); + printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); return printOpts; @@ -802,12 +863,14 @@ public class LogsCLI extends Configured implements Tool { // the ContainerReport. In the containerReport, we could get // nodeAddress and nodeHttpAddress ContainerReport report = getContainerReport(containerIdStr); - nodeHttpAddress = - report.getNodeHttpAddress().replaceFirst( - WebAppUtils.getHttpSchemePrefix(getConf()), ""); + nodeHttpAddress = report.getNodeHttpAddress(); + if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { + nodeHttpAddress = nodeHttpAddress.replaceFirst( + WebAppUtils.getHttpSchemePrefix(getConf()), ""); + request.setNodeHttpAddress(nodeHttpAddress); + } nodeId = report.getAssignedNode().toString(); request.setNodeId(nodeId); - request.setNodeHttpAddress(nodeHttpAddress); } catch (IOException | YarnException ex) { if (isAppFinished) { return printContainerLogsForFinishedApplicationWithoutNodeId( @@ -943,32 +1006,140 @@ public class LogsCLI extends Configured implements Tool { ContainerLogsRequest options) throws YarnException, IOException { List<ContainerLogsRequest> newOptionsList = new ArrayList<ContainerLogsRequest>(); - YarnClient yarnClient = createYarnClient(); - try { - List<ApplicationAttemptReport> attempts = - yarnClient.getApplicationAttempts(options.getAppId()); - for (ApplicationAttemptReport attempt : attempts) { - List<ContainerReport> containers = yarnClient.getContainers( - attempt.getApplicationAttemptId()); - for (ContainerReport container : containers) { - ContainerLogsRequest newOptions = new ContainerLogsRequest(options); - newOptions.setContainerId(container.getContainerId().toString()); - newOptions.setNodeId(container.getAssignedNode().toString()); - newOptions.setNodeHttpAddress(container.getNodeHttpAddress() - .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); - // if we do not specify the value for CONTAINER_LOG_FILES option, - // we will only output syslog - List<String> logFiles = newOptions.getLogTypes(); - if (logFiles == null || logFiles.isEmpty()) { - logFiles = Arrays.asList("syslog"); - newOptions.setLogTypes(logFiles); + List<ContainerReport> reports = + getContainerReportsFromRunningApplication(options); + for (ContainerReport container : reports) { + ContainerLogsRequest newOptions = new ContainerLogsRequest(options); + newOptions.setContainerId(container.getContainerId().toString()); + newOptions.setNodeId(container.getAssignedNode().toString()); + String httpAddress = container.getNodeHttpAddress(); + if (httpAddress != null && !httpAddress.isEmpty()) { + newOptions.setNodeHttpAddress(httpAddress + .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); + } + // if we do not specify the value for CONTAINER_LOG_FILES option, + // we will only output syslog + List<String> logFiles = newOptions.getLogTypes(); + if (logFiles == null || logFiles.isEmpty()) { + logFiles = Arrays.asList("syslog"); + newOptions.setLogTypes(logFiles); + } + newOptionsList.add(newOptions); + } + return newOptionsList; + } + + private List<ContainerReport> getContainerReportsFromRunningApplication( + ContainerLogsRequest options) throws YarnException, IOException { + List<ContainerReport> reports = new ArrayList<ContainerReport>(); + List<ApplicationAttemptReport> attempts = + yarnClient.getApplicationAttempts(options.getAppId()); + for (ApplicationAttemptReport attempt : attempts) { + List<ContainerReport> containers = yarnClient.getContainers( + attempt.getApplicationAttemptId()); + reports.addAll(containers); + } + return reports; + } + + // filter the containerReports based on the nodeId and ContainerId + private List<ContainerReport> filterContainersInfo( + ContainerLogsRequest options, List<ContainerReport> containers) { + List<ContainerReport> filterReports = new ArrayList<ContainerReport>( + containers); + String nodeId = options.getNodeId(); + boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty()); + String containerId = options.getContainerId(); + boolean filterBasedOnContainerId = (containerId != null + && !containerId.isEmpty()); + + if (filterBasedOnNodeId || filterBasedOnContainerId) { + // filter the reports based on the containerId and.or nodeId + for(ContainerReport report : containers) { + if (filterBasedOnContainerId) { + if (!report.getContainerId().toString() + .equalsIgnoreCase(containerId)) { + filterReports.remove(report); + } + } + + if (filterBasedOnNodeId) { + if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) { + filterReports.remove(report); } - newOptionsList.add(newOptions); } } - return newOptionsList; - } finally { - yarnClient.close(); + } + return filterReports; + } + + private int printContainerInfoFromRunningApplication( + ContainerLogsRequest options) throws YarnException, IOException { + String containerIdStr = options.getContainerId(); + String nodeIdStr = options.getNodeId(); + List<ContainerReport> reports = + getContainerReportsFromRunningApplication(options); + List<ContainerReport> filteredReports = filterContainersInfo( + options, reports); + if (filteredReports.isEmpty()) { + StringBuilder sb = new StringBuilder(); + if (containerIdStr != null && !containerIdStr.isEmpty()) { + sb.append("Trying to get container with ContainerId: " + + containerIdStr + "\n"); + } + if (nodeIdStr != null && !nodeIdStr.isEmpty()) { + sb.append("Trying to get container from NodeManager: " + + nodeIdStr + "\n"); + } + sb.append("Can not find any matched containers for the application: " + + options.getAppId()); + System.err.println(sb.toString()); + return -1; + } + for (ContainerReport report : filteredReports) { + String nodeId = report.getAssignedNode().toString(); + String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst( + WebAppUtils.getHttpSchemePrefix(getConf()), ""); + String containerId = report.getContainerId().toString(); + String containerString = String.format( + LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); + outStream.println(containerString); + outStream.println(StringUtils.repeat("=", containerString.length())); + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + "LogType", "LogLength"); + outStream.println(StringUtils.repeat("=", containerString.length())); + List<PerLogFileInfo> infos = getContainerLogFiles( + getConf(), containerId, nodeHttpAddress); + for (PerLogFileInfo info : infos) { + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + info.getFileName(), info.getFileLength()); + } + } + return 0; + } + + private static class PerLogFileInfo { + private String fileName; + private String fileLength; + public PerLogFileInfo(String fileName, String fileLength) { + setFileName(fileName); + setFileLength(fileLength); + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getFileLength() { + return fileLength; + } + + public void setFileLength(String fileLength) { + this.fileLength = fileLength; } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6cef8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 34369ef..cb9a663 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -182,7 +182,7 @@ public class TestLogsCLI { assertTrue(exitCode == -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.println("Retrieve logs for completed YARN applications."); + pw.println("Retrieve logs for YARN applications."); pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]"); pw.println(); pw.println("general options are:"); @@ -217,14 +217,18 @@ public class TestLogsCLI { pw.println(" container logs. The container logs will"); pw.println(" be stored based on the node the container"); pw.println(" ran on."); - pw.println(" -show_meta_info Show the log metadata, including log-file"); - pw.println(" names, the size of the log files. You can"); - pw.println(" combine this with --containerId to get"); - pw.println(" log metadata for the specific container,"); - pw.println(" or with --nodeAddress to get log metadata"); - pw.println(" for all the containers on the specific"); - pw.println(" NodeManager. Currently, this option can"); - pw.println(" only be used for finished applications."); + pw.println(" -show_application_log_info Show the containerIds which belong to the"); + pw.println(" specific Application. You can combine"); + pw.println(" this with --nodeAddress to get"); + pw.println(" containerIds for all the containers on"); + pw.println(" the specific NodeManager."); + pw.println(" -show_container_log_info Show the container log metadata,"); + pw.println(" including log-file names, the size of the"); + pw.println(" log files. You can combine this with"); + pw.println(" --containerId to get log metadata for the"); + pw.println(" specific container, or with --nodeAddress"); + pw.println(" to get log metadata for all the"); + pw.println(" containers on the specific NodeManager."); pw.println(" -size <size> Prints the log file's first 'n' bytes or"); pw.println(" the last 'n' bytes. Use negative values"); pw.println(" as bytes to read from the end and"); @@ -698,9 +702,8 @@ public class TestLogsCLI { "-applicationId", appTest.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Guessed logs' owner is " + priorityUser + " and current user " - + UserGroupInformation.getCurrentUser().getUserName() - + " does not have permission to access")); + "Can not find the logs for the application: " + + appTest.toString())); sysErrStream.reset(); } finally { fs.delete(new Path(remoteLogRootDir), true); @@ -842,49 +845,77 @@ public class TestLogsCLI { LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); - cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info" }); + int result = cli.run(new String[] {"-applicationId", appId.toString(), + "-show_container_log_info", "-show_application_log_info"}); + assertTrue(result == -1); + assertTrue(sysErrStream.toString().contains("Invalid options. " + + "Can only accept one of show_application_log_info/" + + "show_container_log_info.")); + + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_container_log_info"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( - "LogType:syslog")); + "syslog")); assertTrue(sysOutStream.toString().contains( - "LogLength:43")); + "43")); sysOutStream.reset(); - cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-containerId", "container_0_0001_01_000001" }); + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_container_log_info", "-containerId", + "container_0_0001_01_000001"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertFalse(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( - "LogType:syslog")); + "syslog")); assertTrue(sysOutStream.toString().contains( - "LogLength:43")); + "43")); sysOutStream.reset(); - cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-nodeAddress", "localhost" }); + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_container_log_info", "-nodeAddress", "localhost"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( - "LogType:syslog")); + "syslog")); assertTrue(sysOutStream.toString().contains( - "LogLength:43")); + "43")); sysOutStream.reset(); - cli.run(new String[] { "-applicationId", appId.toString(), - "-show_meta_info", "-nodeAddress", "localhost", "-containerId", - "container_1234" }); + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_container_log_info", "-nodeAddress", "localhost", + "-containerId", "container_1234"}); assertTrue(sysErrStream.toString().contains( "Invalid ContainerId specified")); sysErrStream.reset(); + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_application_log_info"}); + assertTrue(sysOutStream.toString().contains( + "Application State: Completed.")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000001 on localhost")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000002 on localhost")); + sysOutStream.reset(); + + cli.run(new String[] {"-applicationId", appId.toString(), + "-show_application_log_info", "-nodeAddress", "localhost"}); + assertTrue(sysOutStream.toString().contains( + "Application State: Completed.")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000001 on localhost")); + assertTrue(sysOutStream.toString().contains( + "container_0_0001_01_000002 on localhost")); + sysOutStream.reset(); + fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6cef8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 7508dd5..3c4f835 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -49,6 +49,7 @@ import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.output.WriterOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.math3.util.Pair; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -959,25 +960,21 @@ public class AggregatedLogFormat { } @Private - public static String readContainerMetaDataAndSkipData( + public static Pair<String, String> readContainerMetaDataAndSkipData( DataInputStream valueStream, PrintStream out) throws IOException { String fileType = valueStream.readUTF(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); - if (out != null) { - out.print("LogType:"); - out.println(fileType); - out.print("LogLength:"); - out.println(fileLengthStr); - } + Pair<String, String> logMeta = new Pair<String, String>( + fileType, fileLengthStr); long totalSkipped = 0; long currSkipped = 0; while (currSkipped != -1 && totalSkipped < fileLength) { currSkipped = valueStream.skip(fileLength - totalSkipped); totalSkipped += currSkipped; } - return fileType; + return logMeta; } public void close() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6cef8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 4de11a5..ef56504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -50,6 +51,11 @@ import com.google.common.annotations.VisibleForTesting; public class LogCLIHelpers implements Configurable { + public static final String PER_LOG_FILE_INFO_PATTERN = + "%20s\t%20s" + System.getProperty("line.separator"); + public static final String CONTAINER_ON_NODE_PATTERN = + "Container: %s on %s"; + private Configuration conf; @Private @@ -151,8 +157,8 @@ public class LogCLIHelpers implements Configurable { AggregatedLogFormat.LogReader reader = null; PrintStream out = createPrintStream(localDir, fileName, containerId); try { - String containerString = "\n\nContainer: " + containerId + " on " - + thisNodeFile.getPath().getName(); + String containerString = String.format(CONTAINER_ON_NODE_PATTERN, + containerId, thisNodeFile.getPath().getName()); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); reader = @@ -219,8 +225,10 @@ public class LogCLIHelpers implements Configurable { thisNodeFile.getPath()); out = createPrintStream(localDir, thisNodeFile.getPath().getName(), containerId); - out.println(containerId + " on " + thisNodeFile.getPath().getName()); - out.println(StringUtils.repeat("=", containerId.length())); + String containerString = String.format(CONTAINER_ON_NODE_PATTERN, + containerId, thisNodeFile.getPath().getName()); + out.println(containerString); + out.println(StringUtils.repeat("=", containerString.length())); if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, thisNodeFile.getModificationTime(), options.getBytes()) > -1) { @@ -355,9 +363,9 @@ public class LogCLIHelpers implements Configurable { PrintStream out = createPrintStream(localDir, thisNodeFile.getPath().getName(), key.toString()); try { - String containerString = - "\n\nContainer: " + key + " on " - + thisNodeFile.getPath().getName(); + String containerString = String.format( + CONTAINER_ON_NODE_PATTERN, key, + thisNodeFile.getPath().getName()); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); while (true) { @@ -400,7 +408,7 @@ public class LogCLIHelpers implements Configurable { } @Private - public void printLogMetadata(ContainerLogsRequest options, + public int printAContainerLogMetadata(ContainerLogsRequest options, PrintStream out, PrintStream err) throws IOException { ApplicationId appId = options.getAppId(); @@ -413,7 +421,7 @@ public class LogCLIHelpers implements Configurable { RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir( appId, appOwner); if (nodeFiles == null) { - return; + return -1; } boolean foundAnyLogs = false; while (nodeFiles.hasNext()) { @@ -434,16 +442,21 @@ public class LogCLIHelpers implements Configurable { valueStream = reader.next(key); while (valueStream != null) { if (getAllContainers || (key.toString().equals(containerIdStr))) { - String containerString = - "\n\nContainer: " + key + " on " - + thisNodeFile.getPath().getName(); + String containerString = String.format(CONTAINER_ON_NODE_PATTERN, + key, thisNodeFile.getPath().getName()); out.println(containerString); out.println("Log Upload Time:" + thisNodeFile.getModificationTime()); out.println(StringUtils.repeat("=", containerString.length())); + out.printf(PER_LOG_FILE_INFO_PATTERN, "LogType", "LogLength"); + out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { - LogReader.readContainerMetaDataAndSkipData(valueStream, out); + Pair<String, String> logMeta = + LogReader.readContainerMetaDataAndSkipData( + valueStream, out); + out.printf(PER_LOG_FILE_INFO_PATTERN, + logMeta.getFirst(), logMeta.getSecond()); } catch (EOFException eof) { break; } @@ -473,7 +486,9 @@ public class LogCLIHelpers implements Configurable { err.println("Can not find log metadata for container: " + containerIdStr); } + return -1; } + return 0; } @Private @@ -501,6 +516,60 @@ public class LogCLIHelpers implements Configurable { } } + @Private + public void printContainersList(ContainerLogsRequest options, + PrintStream out, PrintStream err) throws IOException { + ApplicationId appId = options.getAppId(); + String appOwner = options.getAppOwner(); + String nodeId = options.getNodeId(); + String nodeIdStr = (nodeId == null) ? null + : LogAggregationUtils.getNodeString(nodeId); + RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir( + appId, appOwner); + if (nodeFiles == null) { + return; + } + boolean foundAnyLogs = false; + while (nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + if (nodeIdStr != null) { + if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) { + continue; + } + } + if (!thisNodeFile.getPath().getName() + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + out.println(String.format(CONTAINER_ON_NODE_PATTERN, key, + thisNodeFile.getPath().getName())); + foundAnyLogs = true; + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + } + } + if (!foundAnyLogs) { + if (nodeId != null) { + err.println("Can not find information for any containers on " + + nodeId); + } else { + err.println("Can not find any container information for " + + "the application: " + appId); + } + } + } + private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId, String appOwner) throws IOException { Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner); @@ -621,7 +690,7 @@ public class LogCLIHelpers implements Configurable { while (true) { try { String logFile = LogReader.readContainerMetaDataAndSkipData( - valueStream, null); + valueStream, null).getFirst(); logTypes.add(logFile); } catch (EOFException eof) { break; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
