YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
(cherry picked from commit 7bc33b63c78c6ee9a04c0c1511ef16e63f7f3074) (cherry picked from commit 6ed8989a6f897437ab53c85c42fa8b58fe556359) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02ab202a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02ab202a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02ab202a Branch: refs/heads/branch-2.7 Commit: 02ab202a2b4cf62a57e494641220fef8c5779698 Parents: fb57a1a Author: cnauroth <[email protected]> Authored: Wed May 27 14:21:05 2015 -0700 Committer: cnauroth <[email protected]> Committed: Wed May 27 14:21:26 2015 -0700 ---------------------------------------------------------------------- .../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 6 ++++++ .../java/org/apache/hadoop/mapreduce/v2/util/MRApps.java | 5 ----- .../org/apache/hadoop/yarn/api/ApplicationConstants.java | 8 ++++++++ .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +---------- .../containermanager/launcher/ContainerLaunch.java | 9 +++++---- .../containermanager/launcher/TestContainerLaunch.java | 8 +++----- 6 files changed, 23 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 5a8a658..a603a80 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -817,10 +817,16 @@ public abstract class TaskAttemptImpl implements // Fill in the fields needed per-container that are missing in the common // spec. + boolean userClassesTakesPrecedence = + conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); + // Setup environment by cloning from common env. Map<String, String> env = commonContainerSpec.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); + if (userClassesTakesPrecedence) { + myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); + } MapReduceChildJVM.setVMEnv(myEnv, remoteTask); // Set up the launch command http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index f4487af..95bfaec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -241,11 +241,6 @@ public class MRApps extends Apps { boolean userClassesTakesPrecedence = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); - if (userClassesTakesPrecedence) { - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - } - String classpathEnvVar = conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false) ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index d42cd67..42464da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; @@ -191,6 +192,13 @@ public interface ApplicationConstants { HADOOP_YARN_HOME("HADOOP_YARN_HOME"), /** + * $CLASSPATH_PREPEND_DISTCACHE + * Private, Windows specific + */ + @Private + CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"), + + /** * $CONTAINER_ID * Final, exported by NodeManager and non-modifiable by users. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 45fe075..b6b0838 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration { ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), + ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(), ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); /** address of node manager IPC.*/ @@ -1161,16 +1162,6 @@ public class YarnConfiguration extends Configuration { + "application.classpath"; /** - * Whether or not entries from the distributed cache should be preferred over - * the rest of the YARN CLASSPATH - */ - public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = - YARN_PREFIX + "application.classpath.prepend.distcache"; - - public static final boolean - DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false; - - /** * Default platform-agnostic CLASSPATH for YARN applications. A * comma-separated list of CLASSPATH entries. The parameter expansion marker * will be replaced with real parameter expansion marker ('%' for Windows and http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 3100b5e..e27cc2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -735,11 +735,12 @@ public class ContainerLaunch implements Callable<Integer> { //jar is created and so they "are lost" and have to be explicitly //added to the classpath instead. This also means that their position //is lost relative to other non-distcache classpath entries which will - //break things like mapreduce.job.user.classpath.first. + //break things like mapreduce.job.user.classpath.first. An environment + //variable can be set to indicate that distcache entries should come + //first - boolean preferLocalizedJars = conf.getBoolean( - YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE + boolean preferLocalizedJars = Boolean.valueOf( + environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name()) ); boolean needsSeparator = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/02ab202a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index efeeb1d..2856357 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); - userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH"); + userSetEnv.put(Environment.CLASSPATH.name(), "APATH"); containerLaunchContext.setEnvironment(userSetEnv); Container container = mock(Container.class); when(container.getContainerId()).thenReturn(cId); @@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest { Assert.assertTrue( result.get(result.size() - 1).endsWith("userjarlink.jar")); - //Now move userjar to the front + //Then, with user classpath first + userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true"); cId = ContainerId.newContainerId(appAttemptId, 1); when(container.getContainerId()).thenReturn(cId); - conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE, - "true"); - launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);
