Author: olga Date: Tue Nov 27 11:15:04 2007 New Revision: 598736 URL: http://svn.apache.org/viewvc?rev=598736&view=rev Log: Fix for PIG-11
Added: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/PigServer.java incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=598736&r1=598735&r2=598736&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Tue Nov 27 11:15:04 2007 @@ -10,6 +10,7 @@ OPTIMIZATIONS BUG FIXES + PIG-11 Add capability to search for jar file to register. PIG-24 Files that were incorrectly placed under test/reports have been removed. ant clean now cleans test/reports. (milindb via gates) Modified: incubator/pig/trunk/src/org/apache/pig/PigServer.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/PigServer.java?rev=598736&r1=598735&r2=598736&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/PigServer.java (original) +++ incubator/pig/trunk/src/org/apache/pig/PigServer.java Tue Nov 27 11:15:04 2007 @@ -23,6 +23,11 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Enumeration; +import java.util.Vector; +import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import org.apache.hadoop.dfs.DistributedFileSystem; import org.apache.hadoop.fs.FileSystem; @@ -146,11 +151,59 @@ pigContext.registerFunction(function, functionSpec); } - public void registerJar(String path) throws IOException{ - File f = new File(path); - if (!f.canRead()) - throw new IOException("Can't read " + path); - pigContext.addJar(path); + private URL locateJarFromResources(String jarName) throws IOException { + Enumeration<URL> urls = ClassLoader.getSystemResources(jarName); + URL resourceLocation = null; + + if (urls.hasMoreElements()) { + resourceLocation = urls.nextElement(); + } + + if (pigContext.debug && urls.hasMoreElements()) { + String logMessage = "Found multiple resources that match " + + jarName + ": " + resourceLocation; + + while (urls.hasMoreElements()) { + logMessage += (logMessage + urls.nextElement() + "; "); + } + + pigContext.getLogger().debug(logMessage); + } + + return resourceLocation; + } + + /** + * Registers a jar file. Name of the jar file can be an absolute or + * relative path. + * + * If multiple resources are found with the specified name, the + * first one is registered as returned by getSystemResources. + * A warning is issued to inform the user. + * + * @param name of the jar file to register + * @throws IOException + */ + public void registerJar(String name) throws IOException { + // first try to locate jar via system resources + // if this fails, try by using "name" as File (this preserves + // compatibility with case when user passes absolute path or path + // relative to current working directory.) + if (name != null) { + URL resource = locateJarFromResources(name); + + if (resource == null) { + File f = new File(name); + + if (!f.canRead()) { + throw new IOException("Can't read jar file: " + name); + } + + resource = f.toURI().toURL(); + } + + pigContext.addJar(resource); + } } /** Modified: incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=598736&r1=598735&r2=598736&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java Tue Nov 27 11:15:04 2007 @@ -19,6 +19,7 @@ import java.io.FileInputStream; import java.io.IOException; +import java.io.File; import java.io.InputStream; import java.io.Serializable; import java.lang.reflect.Constructor; @@ -75,7 +76,7 @@ transient private JobConf conf = null; // extra jar files that are needed to run a job - transient public List<String> extraJars = new LinkedList<String>(); + transient public List<URL> extraJars = new LinkedList<URL>(); // The jars that should not be merged in. (Some functions may come from pig.jar and we don't want the whole jar file.) transient public Vector<String> skipJars = new Vector<String>(2); @@ -369,9 +370,18 @@ } } - public void addJar(String path) throws MalformedURLException{ - extraJars.add(path); - LogicalPlanBuilder.classloader = createCl(null); + public void addJar(String path) throws MalformedURLException { + if (path != null) { + URL resource = (new File(path)).toURI().toURL(); + addJar(resource); + } + } + + public void addJar(URL resource) throws MalformedURLException{ + if (resource != null) { + extraJars.add(resource); + LogicalPlanBuilder.classloader = createCl(null); + } } public void rename(String oldName, String newName) throws IOException { @@ -466,7 +476,7 @@ urls[0] = new URL("file:" + jarFile); } for (int i = 0; i < extraJars.size(); i++) { - urls[i + passedJar] = new URL("file:" + extraJars.get(i)); + urls[i + passedJar] = extraJars.get(i); } return new URLClassLoader(urls, PigMapReduce.class.getClassLoader()); } Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=598736&r1=598735&r2=598736&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java Tue Nov 27 11:15:04 2007 @@ -168,17 +168,29 @@ */ private static void mergeJar(JarOutputStream jarFile, String jar, String prefix, Map<String, String> contents) throws FileNotFoundException, IOException { - JarInputStream jis = new JarInputStream(new FileInputStream(jar)); + JarInputStream jarInput = new JarInputStream(new FileInputStream(jar)); + + mergeJar(jarFile, jarInput, prefix, contents); + } + + private static void mergeJar(JarOutputStream jarFile, URL jar, String prefix, Map<String, String> contents) + throws FileNotFoundException, IOException { + JarInputStream jarInput = new JarInputStream(jar.openStream()); + + mergeJar(jarFile, jarInput, prefix, contents); + } + + private static void mergeJar(JarOutputStream jarFile, JarInputStream jarInput, String prefix, Map<String, String> contents) + throws FileNotFoundException, IOException { JarEntry entry; - while ((entry = jis.getNextJarEntry()) != null) { + while ((entry = jarInput.getNextJarEntry()) != null) { if (prefix != null && !entry.getName().startsWith(prefix)) { continue; } - addStream(jarFile, entry.getName(), jis, contents); + addStream(jarFile, entry.getName(), jarInput, contents); } } - - /** + /** * Adds a stream to a Jar file. * * @param os Added: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=598736&view=auto ============================================================================== --- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (added) +++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Tue Nov 27 11:15:04 2007 @@ -0,0 +1,235 @@ + +package org.apache.pig.test; + +import java.io.File; +import java.io.IOException; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.util.List; +import java.util.Iterator; +import java.net.URL; +import java.net.URLClassLoader; +import java.lang.reflect.Method; + +import org.apache.pig.PigServer; + +import org.junit.Test; +import junit.framework.TestCase; + + +public class TestPigServer extends TestCase { + private PigServer pig = new PigServer(); + + private final static String FILE_SEPARATOR = System.getProperty("file.separator"); + + // make sure that name is included or not (depending on flag "included") + // in the given list of stings + private static void verifyStringContained(List<URL> list, String name, boolean included) { + Iterator<URL> iter = list.iterator(); + boolean nameIsSubstring = false; + int count = 0; + + while (iter.hasNext()) { + if (iter.next().toString().contains(name)) { + nameIsSubstring = true; + ++count; + } + } + + if (included) { + assertTrue(nameIsSubstring); + assertTrue(count == 1); + } + else { + assertFalse(nameIsSubstring); + } + } + + // creates an empty jar file + private static void createFakeJarFile(String location, String name) + throws IOException { + assertFalse((new File(name)).canRead()); + + assertTrue((new File(location)).mkdirs()); + + assertTrue((new File(location + FILE_SEPARATOR + name)). + createNewFile()); + } + + // dynamically add more resources to the system class loader + private static void registerNewResource(String file) throws Exception { + URL urlToAdd = new File(file).toURI().toURL(); + URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); + Method addMethod = URLClassLoader.class. + getDeclaredMethod("addURL", + new Class[]{URL.class}); + addMethod.setAccessible(true); + addMethod.invoke(sysLoader, new Object[]{urlToAdd}); + } + + private static void executeShellCommand(String cmd) throws Exception { + Process cmdProc = Runtime.getRuntime().exec(cmd); + + cmdProc.waitFor(); + + assertTrue(cmdProc.exitValue() == 0); + } + + /** + * The jar file to register is not present + */ + @Test + public void testRegisterJarFileNotPresent() throws Exception { + // resister a jar file that does not exist + + String jarName = "BadFileNameTestJarNotPresent.jar"; + + // jar name is not present to start with + verifyStringContained(pig.getPigContext().extraJars, jarName, false); + + boolean exceptionRaised = false; + try { + pig.registerJar(jarName); + } + catch (IOException e) { + exceptionRaised = true; + } + assertTrue(exceptionRaised); + verifyStringContained(pig.getPigContext().extraJars, jarName, false); + } + + /** + * Jar file to register is not present in the system resources + * in this case name of jar file is relative to current working dir + */ + @Test + public void testRegisterJarLocalDir() throws Exception { + String dir1 = "test1_register_jar_local"; + String dir2 = "test2_register_jar_local"; + String jarLocation = dir1 + FILE_SEPARATOR + + dir2 + FILE_SEPARATOR; + String jarName = "TestRegisterJarLocal.jar"; + + createFakeJarFile(jarLocation, jarName); + + verifyStringContained(pig.getPigContext().extraJars, jarName, false); + + boolean exceptionRaised = false; + try { + pig.registerJar(jarLocation + jarName); + } + catch (IOException e) { + exceptionRaised = true; + } + assertFalse(exceptionRaised); + verifyStringContained(pig.getPigContext().extraJars, jarName, true); + + // clean-up + assertTrue((new File(jarLocation + jarName)).delete()); + (new File(dir1 + FILE_SEPARATOR + dir2)).delete(); + (new File(dir1)).delete(); + } + + /** + * Jar file is located via system resources + * Test verifies that even with multiple resources matching, + * only one of them is registered. + */ + @Test + public void testRegisterJarFromResources () throws Exception { + String dir = "test_register_jar_res_dir"; + String subDir1 = "test_register_jar_res_sub_dir1"; + String subDir2 = "test_register_jar_res_sub_dir2"; + String jarName = "TestRegisterJarFromRes.jar"; + String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR; + String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR; + + createFakeJarFile(jarLocation1, jarName); + createFakeJarFile(jarLocation2, jarName); + + verifyStringContained(pig.getPigContext().extraJars, jarName, false); + + registerNewResource(jarLocation1); + registerNewResource(jarLocation2); + + boolean exceptionRaised = false; + try { + pig.registerJar(jarName); + } + catch (IOException e) { + exceptionRaised = true; + } + assertFalse(exceptionRaised); + verifyStringContained(pig.getPigContext().extraJars, jarName, true); + + // clean-up + assertTrue((new File(jarLocation1 + jarName)).delete()); + assertTrue((new File(jarLocation2 + jarName)).delete()); + (new File(jarLocation1)).delete(); + (new File(jarLocation2)).delete(); + (new File(dir)).delete(); + } + + /** + * Use a resource inside a jar file. + * Verify that the containing jar file is registered correctly. + * @throws Exception + */ + @Test + public void testRegisterJarResourceInJar() throws Exception { + String dir = "test_register_jar_res_in_jar"; + String subDir = "sub_dir"; + String jarName = "TestRegisterJarNonEmpty.jar"; + String className = "TestRegisterJar"; + String javaSrc = "package " + subDir + "; class " + className + " { }"; + + // create dirs + (new File(dir + FILE_SEPARATOR + subDir)).mkdirs(); + + // generate java file + FileOutputStream outStream = + new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir + + FILE_SEPARATOR + className + ".java")); + + OutputStreamWriter outWriter = new OutputStreamWriter(outStream); + outWriter.write(javaSrc); + outWriter.close(); + + // compile + executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir + + FILE_SEPARATOR + className + ".java"); + + // remove src file + (new File(dir + FILE_SEPARATOR + subDir + + FILE_SEPARATOR + className + ".java")).delete(); + + // generate jar file + executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " + + "-C " + dir + " " + subDir); + + // remove class file and sub_dir + (new File(dir + FILE_SEPARATOR + subDir + + FILE_SEPARATOR + className + ".class")).delete(); + (new File(dir + FILE_SEPARATOR + subDir)).delete(); + + // register resource + registerNewResource(dir + FILE_SEPARATOR + jarName); + + // load the specific resource + boolean exceptionRaised = false; + try { + pig.registerJar("sub_dir/TestRegisterJar.class"); + } + catch (IOException e) { + exceptionRaised = true; + } + + // verify proper jar file is located + assertFalse(exceptionRaised); + verifyStringContained(pig.getPigContext().extraJars, jarName, true); + + // clean up Jar file and test dir + (new File(dir + FILE_SEPARATOR + jarName)).delete(); + (new File(dir)).delete(); + } +} \ No newline at end of file