Author: jlowe
Date: Tue May 13 18:55:05 2014
New Revision: 1594326
URL: http://svn.apache.org/r1594326
Log:
MAPREDUCE-5814. fat jar with *-default.xml may fail when
mapreduce.job.classloader=true. Contributed by Gera Shegalov
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1594326&r1=1594325&r2=1594326&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue May 13
18:55:05 2014
@@ -222,6 +222,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5888. Failed job leaves hung AM after it unregisters (Jason Lowe
via jeagles)
+ MAPREDUCE-5814. fat jar with *-default.xml may fail when
+ mapreduce.job.classloader=true. (Gera Shegalov via jlowe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1594326&r1=1594325&r2=1594326&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Tue May 13 18:55:05 2014
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -344,8 +345,7 @@ public class MRApps extends Apps {
if (LOG.isDebugEnabled()) {
LOG.debug("APP_CLASSPATH=" + appClasspath);
}
- String[] systemClasses = conf.getStrings(
- MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
+ String[] systemClasses = getSystemClasses(conf);
ClassLoader jobClassLoader = createJobClassLoader(appClasspath,
systemClasses);
if (jobClassLoader != null) {
@@ -356,6 +356,12 @@ public class MRApps extends Apps {
}
}
+ @VisibleForTesting
+ static String[] getSystemClasses(Configuration conf) {
+ return conf.getTrimmedStrings(
+ MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
+ }
+
private static ClassLoader createJobClassLoader(final String appClasspath,
final String[] systemClasses) throws IOException {
try {
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1594326&r1=1594325&r2=1594326&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Tue May 13 18:55:05 2014
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -492,4 +494,36 @@ public class TestMRApps {
assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED));
assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING));
}
+
+
+ private static final String[] SYS_CLASSES = new String[] {
+ "/java/fake/Klass",
+ "/javax/fake/Klass",
+ "/org/apache/commons/logging/fake/Klass",
+ "/org/apache/log4j/fake/Klass",
+ "/org/apache/hadoop/fake/Klass"
+ };
+
+ private static final String[] DEFAULT_XMLS = new String[] {
+ "core-default.xml",
+ "mapred-default.xml",
+ "hdfs-default.xml",
+ "yarn-default.xml"
+ };
+
+ @Test
+ public void testSystemClasses() {
+ final List<String> systemClasses =
+ Arrays.asList(MRApps.getSystemClasses(new Configuration()));
+ for (String defaultXml : DEFAULT_XMLS) {
+ assertTrue(defaultXml + " must be system resource",
+ ApplicationClassLoader.isSystemClass(defaultXml, systemClasses));
+ }
+ for (String klass : SYS_CLASSES) {
+ assertTrue(klass + " must be system class",
+ ApplicationClassLoader.isSystemClass(klass, systemClasses));
+ }
+ assertFalse("/fake/Klass must not be a system class",
+ ApplicationClassLoader.isSystemClass("/fake/Klass", systemClasses));
+ }
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1594326&r1=1594325&r2=1594326&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Tue May 13 18:55:05 2014
@@ -1215,7 +1215,9 @@
<property>
<name>mapreduce.job.classloader.system.classes</name>
-
<value>java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.</value>
+ <value>java.,javax.,org.apache.commons.logging.,org.apache.log4j.,
+ org.apache.hadoop.,core-default.xml,hdfs-default.xml,
+ mapred-default.xml,yarn-default.xml</value>
<description>A comma-separated list of classes that should be loaded from the
system classpath, not the user-supplied JARs, when
mapreduce.job.classloader
is enabled. Names ending in '.' (period) are treated as package names,
