Repository: hadoop Updated Branches: refs/heads/branch-2.7 f5a4f69e9 -> 086ee0adf
YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to fully qualified path. Contributed by Xuan Gong (cherry picked from commit d81109e588493cef31e68508a3d671203bd23e12) (cherry picked from commit f40f17489cacb22eccc4c9fad66a03abebf9426b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/086ee0ad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/086ee0ad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/086ee0ad Branch: refs/heads/branch-2.7 Commit: 086ee0adf8c9b299b12febfdedc495cfd1f34bc8 Parents: f5a4f69 Author: Junping Du <[email protected]> Authored: Fri Mar 20 13:41:22 2015 -0700 Committer: Zhe Zhang <[email protected]> Committed: Thu Feb 2 09:57:14 2017 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java | 5 +++-- .../containermanager/logaggregation/AppLogAggregatorImpl.java | 2 +- .../containermanager/logaggregation/LogAggregationService.java | 2 +- .../logaggregation/TestLogAggregationService.java | 4 +++- 5 files changed, 11 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/086ee0ad/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8a0e7aa..f07cf5a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -67,6 +67,9 @@ Release 2.7.4 - UNRELEASED YARN-4355. NPE while processing localizer heartbeat. Contributed by Varun Saxena & Jonathan Hung + YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to + fully qualified path. (Xuan Gong via junping_du) + Release 2.7.3 - 2016-08-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/086ee0ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index ad2ee50..57f655b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -379,7 +379,7 @@ public class AggregatedLogFormat { userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() { @Override public FSDataOutputStream run() throws Exception { - fc = FileContext.getFileContext(conf); + fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, @@ -471,7 +471,8 @@ public class AggregatedLogFormat { public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { - FileContext fileContext = FileContext.getFileContext(conf); + FileContext fileContext = + FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( http://git-wip-us.apache.org/repos/asf/hadoop/blob/086ee0ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 6cf9a78..7ffe0f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -304,7 +304,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator { userUgi.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws Exception { - FileSystem remoteFS = FileSystem.get(conf); + FileSystem remoteFS = remoteNodeLogFileForApp.getFileSystem(conf); if (remoteFS.exists(remoteNodeTmpLogFileForApp)) { if (rename) { remoteFS.rename(remoteNodeTmpLogFileForApp, renamedPath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/086ee0ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 53efb02..09b485a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -176,7 +176,7 @@ public class LogAggregationService extends AbstractService implements } protected FileSystem getFileSystem(Configuration conf) throws IOException { - return FileSystem.get(conf); + return this.remoteRootLogDir.getFileSystem(conf); } void verifyAndCreateRemoteLogDir(Configuration conf) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/086ee0ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 153237f..50be4b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -1392,8 +1392,10 @@ public class TestLogAggregationService extends BaseContainerManagerTest { logAggregationContextWithInterval.setRolledLogsIncludePattern(".*"); logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final"); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); + //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to + //have fully qualified path this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - this.remoteRootLogDir.getAbsolutePath()); + "file://" + this.remoteRootLogDir.getAbsolutePath()); this.conf.setLong( YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
