Repository: hadoop Updated Branches: refs/heads/branch-2 c8eb13997 -> 8437c31b6
HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be order-independent. (Yitong Zhou via gera) (cherry picked from commit 0bcea111e5daa9a4315346cf6919a4cfc8d90e0d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8437c31b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8437c31b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8437c31b Branch: refs/heads/branch-2 Commit: 8437c31b6781b3c03270cfe28694ca93d42f135f Parents: c8eb139 Author: Gera Shegalov <[email protected]> Authored: Thu Dec 11 12:25:25 2014 -0800 Committer: Gera Shegalov <[email protected]> Committed: Thu Dec 11 13:29:40 2014 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop/util/ApplicationClassLoader.java | 25 ++++++++++++++++---- .../hadoop/util/TestApplicationClassLoader.java | 8 +++++-- .../src/main/resources/mapred-default.xml | 21 ++++++++++++---- 4 files changed, 46 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8437c31b/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c41261e..a68d2d6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -199,6 +199,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11386. Replace \n by %n in format hadoop-common format strings. (Li Lu via wheat9) + + HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be + be order-independent. (Yitong Zhou via gera) Release 2.6.0 - 2014-11-18 http://git-wip-us.apache.org/repos/asf/hadoop/blob/8437c31b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java index d2ab015..9f16b61 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java @@ -216,28 +216,43 @@ public class ApplicationClassLoader extends URLClassLoader { return c; } + /** + * Checks if a class should be included as a system class. + * + * A class is a system class if and only if it matches one of the positive + * patterns and none of the negative ones. + * + * @param name the class name to check + * @param systemClasses a list of system class configurations. + * @return true if the class is a system class + */ public static boolean isSystemClass(String name, List<String> systemClasses) { + boolean result = false; if (systemClasses != null) { String canonicalName = name.replace('/', '.'); while (canonicalName.startsWith(".")) { canonicalName=canonicalName.substring(1); } for (String c : systemClasses) { - boolean result = true; + boolean shouldInclude = true; if (c.startsWith("-")) { c = c.substring(1); - result = false; + shouldInclude = false; } if (canonicalName.startsWith(c)) { if ( c.endsWith(".") // package || canonicalName.length() == c.length() // class || canonicalName.length() > c.length() // nested && canonicalName.charAt(c.length()) == '$' ) { - return result; + if (shouldInclude) { + result = true; + } else { + return false; + } } } } } - return false; + return result; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8437c31b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java index cc16493..be8e61e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java @@ -87,7 +87,7 @@ public class TestApplicationClassLoader { assertEquals(jarFile.toURI().toURL(), urls[2]); // nofile should be ignored } - + @Test public void testIsSystemClass() { testIsSystemClassInternal(""); @@ -112,8 +112,12 @@ public class TestApplicationClassLoader { classes("-org.example.Foo,org.example."))); assertTrue(isSystemClass("org.example.Bar" + nestedClass, classes("-org.example.Foo.,org.example."))); + assertFalse(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example.,-org.example.Foo"))); + assertFalse(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example.Foo,-org.example.Foo"))); } - + private List<String> classes(String classes) { return Lists.newArrayList(Splitter.on(',').split(classes)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8437c31b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 83b7655..3798da7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1798,10 +1798,23 @@ <value></value> <description>Used to override the default definition of the system classes for the job classloader. The system classes are a comma-separated list of - classes that should be loaded from the system classpath, not the - user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending - in '.' (period) are treated as package names, and names starting with a '-' - are treated as negative matches. + patterns that indicate whether to load a class from the system classpath, + instead from the user-supplied JARs, when mapreduce.job.classloader is + enabled. + + A positive pattern is defined as: + 1. A single class name 'C' that matches 'C' and transitively all nested + classes 'C$*' defined in C; + 2. A package name ending with a '.' (e.g., "com.example.") that matches + all classes from that package. + A negative pattern is defined by a '-' in front of a positive pattern + (e.g., "-com.example."). + + A class is considered a system class if and only if it matches one of the + positive patterns and none of the negative ones. More formally: + A class is a member of the inclusion set I if it matches one of the positive + patterns. A class is a member of the exclusion set E if it matches one of + the negative patterns. The set of system classes S = I \ E. </description> </property>
