Repository: hadoop Updated Branches: refs/heads/trunk a52eb4bc5 -> f81dc3f99
YARN-2703. Added logUploadedTime into LogValue for better display. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f81dc3f9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f81dc3f9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f81dc3f9 Branch: refs/heads/trunk Commit: f81dc3f995579c1b94b11d60e9fc6da56c8a9496 Parents: a52eb4b Author: Zhijie Shen <[email protected]> Authored: Fri Oct 24 14:10:46 2014 -0700 Committer: Zhijie Shen <[email protected]> Committed: Fri Oct 24 14:10:46 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/client/cli/TestLogsCLI.java | 1 + .../yarn/logaggregation/AggregatedLogFormat.java | 19 +++++++++++++++++++ .../yarn/webapp/log/AggregatedLogsBlock.java | 4 +++- .../logaggregation/TestAggregatedLogFormat.java | 4 +++- .../TestLogAggregationService.java | 15 +++++++++------ 6 files changed, 38 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cfa0727..c30499f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -391,6 +391,9 @@ Release 2.6.0 - UNRELEASED YARN-2209. Replaced AM resync/shutdown command with corresponding exceptions and made related MR changes. (Jian He via zjshen) + YARN-2703. Added logUploadedTime into LogValue for better display. (Xuan Gong + via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 132dca2..2e9e92d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -227,6 +227,7 @@ public class TestLogsCLI { assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001!")); + assertTrue(sysOutStream.toString().contains("LogUploadTime")); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 46e39cc..035d61d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; @@ -226,6 +227,9 @@ public class AggregatedLogFormat { // Write the logFile Type out.writeUTF(logFile.getName()); + // Write the uploaded TimeStamp + out.writeLong(System.currentTimeMillis()); + // Write the log length as UTF so that it is printable out.writeUTF(String.valueOf(fileLength)); @@ -636,6 +640,7 @@ public class AggregatedLogFormat { int bufferSize = 65536; char[] cbuf = new char[bufferSize]; String fileType; + long uploadTime; String fileLengthStr; long fileLength; @@ -646,10 +651,13 @@ public class AggregatedLogFormat { // EndOfFile return; } + uploadTime = valueStream.readLong(); fileLengthStr = valueStream.readUTF(); fileLength = Long.parseLong(fileLengthStr); writer.write("\n\nLogType:"); writer.write(fileType); + writer.write("\nLogUploadTime:"); + writer.write(String.valueOf(uploadTime)); writer.write("\nLogLength:"); writer.write(fileLengthStr); writer.write("\nLog Contents:\n"); @@ -681,10 +689,13 @@ public class AggregatedLogFormat { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); + long uploadTime = valueStream.readLong(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); out.print("LogType: "); out.println(fileType); + out.print("LogUploadTime: "); + out.println(Times.format(uploadTime)); out.print("LogLength: "); out.println(fileLengthStr); out.println("Log Contents:"); @@ -715,6 +726,7 @@ public class AggregatedLogFormat { public static class ContainerLogsReader { private DataInputStream valueStream; private String currentLogType = null; + private long currentLogUpLoadTime = 0; private long currentLogLength = 0; private BoundedInputStream currentLogData = null; private InputStreamReader currentLogISR; @@ -735,12 +747,14 @@ public class AggregatedLogFormat { } currentLogType = null; + currentLogUpLoadTime = 0; currentLogLength = 0; currentLogData = null; currentLogISR = null; try { String logType = valueStream.readUTF(); + long logUpLoadTime = valueStream.readLong(); String logLengthStr = valueStream.readUTF(); currentLogLength = Long.parseLong(logLengthStr); currentLogData = @@ -748,6 +762,7 @@ public class AggregatedLogFormat { currentLogData.setPropagateClose(false); currentLogISR = new InputStreamReader(currentLogData); currentLogType = logType; + currentLogUpLoadTime = logUpLoadTime; } catch (EOFException e) { } @@ -758,6 +773,10 @@ public class AggregatedLogFormat { return currentLogType; } + public long getCurrentLogUpLoadTime() { + return currentLogUpLoadTime; + } + public long getCurrentLogLength() { return currentLogLength; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index 16e6359..bba3258 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -198,12 +199,13 @@ public class AggregatedLogsBlock extends HtmlBlock { if (desiredLogType == null || desiredLogType.isEmpty() || desiredLogType.equals(logType)) { long logLength = logReader.getCurrentLogLength(); - + long logUpLoadTime = logReader.getCurrentLogUpLoadTime(); if (foundLog) { html.pre()._("\n\n")._(); } html.p()._("Log Type: " + logType)._(); + html.p()._("Log UpLoadTime: " + Times.format(logUpLoadTime))._(); html.p()._("Log Length: " + Long.toString(logLength))._(); long start = logLimits.start < 0 http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 0fae77d..fa9de74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -232,7 +232,9 @@ public class TestAggregatedLogFormat { // aggregated. String s = writer.toString(); int expectedLength = - "\n\nLogType:stdout".length() + ("\nLogLength:" + numChars).length() + "\n\nLogType:stdout".length() + + ("\nLogUploadTime:" + System.currentTimeMillis()).length() + + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 8a5441a..cea71fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -61,7 +61,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -768,19 +767,23 @@ public class TestLogAggregationService extends BaseContainerManagerTest { Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8)); String fileType = writtenLines[0].substring(9); - Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10)); - String fileLengthStr = writtenLines[1].substring(11); + Assert.assertEquals("LogUploadTime:", writtenLines[1].substring(0, 14)); + String fileUploadedTimeStr = writtenLines[1].substring(15); + + Assert.assertEquals("LogLength:", writtenLines[2].substring(0, 10)); + String fileLengthStr = writtenLines[2].substring(11); long fileLength = Long.parseLong(fileLengthStr); Assert.assertEquals("Log Contents:", - writtenLines[2].substring(0, 13)); + writtenLines[3].substring(0, 13)); String logContents = StringUtils.join( - Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n"); + Arrays.copyOfRange(writtenLines, 4, writtenLines.length), "\n"); perContainerMap.put(fileType, logContents); LOG.info("LogType:" + fileType); - LOG.info("LogType:" + fileLength); + LOG.info("LogUploadTime:" + fileUploadedTimeStr); + LOG.info("LogLength:" + fileLength); LOG.info("Log Contents:\n" + perContainerMap.get(fileType)); } catch (EOFException eof) { break;
