Repository: hadoop Updated Branches: refs/heads/branch-2 c25154576 -> da18bbeda
MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. Contributed by Junping Du Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da18bbed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da18bbed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da18bbed Branch: refs/heads/branch-2 Commit: da18bbedaffad1f3ac9c078675583e7cfb68790f Parents: c251545 Author: Jian He <jia...@apache.org> Authored: Wed Jan 27 13:16:36 2016 -0800 Committer: Jian He <jia...@apache.org> Committed: Wed Jan 27 13:16:59 2016 -0800 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../apache/hadoop/mapreduce/v2/util/MRApps.java | 4 ++ .../hadoop/mapreduce/v2/util/TestMRApps.java | 55 ++++++++++++++++++++ 3 files changed, 62 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/da18bbed/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d08a1a2..4dcad88 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -409,6 +409,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6610. JobHistoryEventHandler should not swallow timeline response (Li Lu via jianhe) + MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. + (Junping Du via jianhe) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/da18bbed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index c645465..feea789 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -261,6 +261,10 @@ public class MRApps extends Apps { addClasspathToEnv(environment, classpathEnvVar, conf); addClasspathToEnv(environment, hadoopClasspathEnvVar, conf); + // MAPREDUCE-6619, retain $HADOOP_CLASSPATH + MRApps.addToEnvironment(environment, hadoopClasspathEnvVar, + System.getenv(hadoopClasspathEnvVar), conf); + if (userClassesTakesPrecedence) { MRApps.setMRFrameworkClasspath(environment, conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/da18bbed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 3a417a0..f849d72 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -30,8 +30,10 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.net.URI; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -239,6 +241,12 @@ public class TestMRApps { testTGZ.getAbsolutePath())).toString(); conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath); conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ"); + // add hadoop.tgz to env HADOOP_CLASSPATH + Map<String, String> newEnv = new HashMap<String, String>(); + newEnv.put(ApplicationConstants.Environment.HADOOP_CLASSPATH.name(), + "hadoop.tgz"); + setEnv(newEnv); + Map<String, String> environment = new HashMap<String, String>(); MRApps.setClasspath(environment, conf); assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.name()).startsWith( @@ -268,6 +276,53 @@ public class TestMRApps { assertTrue(environment.get( ApplicationConstants.Environment.HADOOP_CLASSPATH.name()). contains("testTGZ")); + assertTrue(environment.get( + ApplicationConstants.Environment.HADOOP_CLASSPATH.name()). + contains("hadoop.tgz")); + } + + // Only set env in runtime but not get persistent in OS (Linux or Windows) + @SuppressWarnings({"unchecked", "rawtypes"}) + private static void setEnv(Map<String, String> newEnv) { + try { + // Hack for Windows + Class<?> processEnvClass = + Class.forName("java.lang.ProcessEnvironment"); + Field theEnvField = + processEnvClass.getDeclaredField("theEnvironment"); + theEnvField.setAccessible(true); + Map<String, String> currentEnv = + (Map<String, String>)theEnvField.get(null); + currentEnv.putAll(newEnv); + + Field caseInsensitiveEnvField = + processEnvClass.getDeclaredField("theCaseInsensitiveEnvironment"); + caseInsensitiveEnvField.setAccessible(true); + Map<String, String> ciEnv = + (Map<String, String>)caseInsensitiveEnvField.get(null); + ciEnv.putAll(newEnv); + } catch (NoSuchFieldException e) { + // Hack for Linux + try { + Class[] classes = Collections.class.getDeclaredClasses(); + Map<String, String> env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map<String, String> map = (Map<String, String>) obj; + map.putAll(newEnv); + } + } + } catch (Exception e1) { + LOG.error("Hack env on Linux doesn't work:", e1); + throw new RuntimeException(e1); + } + } catch (Exception e) { + LOG.error("Hack env on Windows doesn't work:", e); + throw new RuntimeException(e); + } } @Test (timeout = 120000)
