Author: todd
Date: Wed Dec 29 20:54:19 2010
New Revision: 1053730
URL: http://svn.apache.org/viewvc?rev=1053730&view=rev
Log:
MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux.
Contributed by Todd Lipcon
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1053730&r1=1053729&r2=1053730&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Dec 29 20:54:19 2010
@@ -453,6 +453,8 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2224. Fix synchronization bugs in JvmManager. (todd)
+ MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux
(todd)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=1053730&r1=1053729&r2=1053730&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed
Dec 29 20:54:19 2010
@@ -1833,7 +1833,7 @@ public class JobConf extends Configurati
* @return a jar file that contains the class, or null.
* @throws IOException
*/
- private static String findContainingJar(Class my_class) {
+ static String findContainingJar(Class my_class) {
ClassLoader loader = my_class.getClassLoader();
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
try {
@@ -1845,6 +1845,13 @@ public class JobConf extends Configurati
if (toReturn.startsWith("file:")) {
toReturn = toReturn.substring("file:".length());
}
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
toReturn = URLDecoder.decode(toReturn, "UTF-8");
return toReturn.replaceAll("!.*$", "");
}
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java?rev=1053730&view=auto
==============================================================================
---
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
(added)
+++
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobConf.java
Wed Dec 29 20:54:19 2010
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.junit.Test;
+import java.io.File;
+import java.net.URLClassLoader;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+
+
+import static org.junit.Assert.*;
+
+public class TestJobConf {
+ private static final String JAR_RELATIVE_PATH =
+ "build/test/mapred/testjar/testjob.jar";
+ private static final String CLASSNAME = "ClassWithNoPackage";
+
+ private static String TEST_DIR_WITH_SPECIAL_CHARS =
+ System.getProperty("test.build.data","/tmp") +
+ File.separator + "test jobconf with + and spaces";
+
+ @Test
+ public void testFindContainingJar() throws Exception {
+ testJarAtPath(JAR_RELATIVE_PATH);
+ }
+
+ /**
+ * Test that findContainingJar works correctly even if the
+ * path has a "+" sign or spaces in it
+ */
+ @Test
+ public void testFindContainingJarWithPlus() throws Exception {
+ new File(TEST_DIR_WITH_SPECIAL_CHARS).mkdirs();
+ Configuration conf = new Configuration();
+
+ FileSystem localfs = FileSystem.getLocal(conf);
+
+ FileUtil.copy(localfs, new Path(JAR_RELATIVE_PATH),
+ localfs, new Path(TEST_DIR_WITH_SPECIAL_CHARS, "test.jar"),
+ false, true, conf);
+ testJarAtPath(TEST_DIR_WITH_SPECIAL_CHARS + File.separator + "test.jar");
+ }
+
+ /**
+ * Given a path with a jar, make a classloader with that jar on the
+ * classpath, and check that findContainingJar can correctly
+ * identify the path of the jar.
+ */
+ private void testJarAtPath(String path) throws Exception {
+ File jar = new File(path).getAbsoluteFile();
+ assertTrue(jar.exists());
+
+ URL urls[] = new URL[] {
+ jar.toURI().toURL()
+ };
+
+ ClassLoader cl = new URLClassLoader(urls);
+ Class clazz = Class.forName(CLASSNAME, true, cl);
+ assertNotNull(clazz);
+
+ String containingJar = JobConf.findContainingJar(clazz);
+ assertEquals(jar.getAbsolutePath(), containingJar);
+ }
+}
\ No newline at end of file
