Repository: hadoop Updated Branches: refs/heads/trunk 9ada9284d -> caecd9fff
YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/caecd9ff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/caecd9ff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/caecd9ff Branch: refs/heads/trunk Commit: caecd9fffe7c6216be31f3ab65349182045451fa Parents: 9ada928 Author: Zhijie Shen <[email protected]> Authored: Sun Oct 26 12:57:31 2014 -0700 Committer: Zhijie Shen <[email protected]> Committed: Sun Oct 26 12:57:31 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/logaggregation/AggregatedLogFormat.java | 6 ++++++ .../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 9 +++++++++ 3 files changed, 18 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/caecd9ff/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 08965ed..11c1ac1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -760,6 +760,9 @@ Release 2.6.0 - UNRELEASED YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port. (Naganarasimha G R via xgong) + YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. + (Xuan Gong via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/caecd9ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 035d61d..22219be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -213,6 +213,12 @@ public class AggregatedLogFormat { Collections.sort(fileList); for (File logFile : fileList) { + // We only aggregate top level files. + // Ignore anything inside sub-folders. + if (logFile.isDirectory()) { + LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it."); + continue; + } FileInputStream in = null; try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/caecd9ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index fa9de74..1d607b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -194,6 +194,14 @@ public class TestAggregatedLogFormat { int numChars = 80000; + // create a sub-folder under srcFilePath + // and create file logs in this sub-folder. + // We only aggregate top level files. + // So, this log file should be ignored. + Path subDir = new Path(srcFilePath, "subDir"); + fs.mkdirs(subDir); + writeSrcFile(subDir, "logs", numChars); + // create file stderr and stdout in containerLogDir writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stdout", numChars); @@ -238,6 +246,7 @@ public class TestAggregatedLogFormat { + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); + Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
