Repository: hadoop Updated Branches: refs/heads/branch-2.6 3e4182863 -> 2098c68ac
YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li (cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2098c68a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2098c68a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2098c68a Branch: refs/heads/branch-2.6 Commit: 2098c68acbcdf8cc412e0c2d43eddbfc1dfe3e79 Parents: 3e41828 Author: Jason Lowe <[email protected]> Authored: Mon Nov 3 20:37:47 2014 +0000 Committer: Jason Lowe <[email protected]> Committed: Mon Nov 3 20:40:02 2014 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../nodemanager/DefaultContainerExecutor.java | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2098c68a/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b84fc1e..cba4b34 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -764,6 +764,9 @@ Release 2.6.0 - UNRELEASED YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun Vasudev via zjshen) + YARN-2730. DefaultContainerExecutor runs only one localizer at a time + (Siqi Li via jlowe) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/2098c68a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 3e212bf..5bf8cec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -94,7 +94,7 @@ public class DefaultContainerExecutor extends ContainerExecutor { } @Override - public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, + public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { @@ -102,10 +102,6 @@ public class DefaultContainerExecutor extends ContainerExecutor { List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); - ContainerLocalizer localizer = - new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs), - RecordFactoryProvider.getRecordFactory(getConf())); - createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); @@ -118,8 +114,17 @@ public class DefaultContainerExecutor extends ContainerExecutor { Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); - lfs.setWorkingDirectory(appStorageDir); - LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory()); + + + FileContext localizerFc = FileContext.getFileContext( + lfs.getDefaultFileSystem(), getConf()); + localizerFc.setUMask(lfs.getUMask()); + localizerFc.setWorkingDirectory(appStorageDir); + LOG.info("Localizer CWD set to " + appStorageDir + " = " + + localizerFc.getWorkingDirectory()); + ContainerLocalizer localizer = + new ContainerLocalizer(localizerFc, user, appId, locId, + getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }
