Repository: hbase Updated Branches: refs/heads/0.98 27c583467 -> 6dfdc0d89
HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li) Amending-Author: Andrew Purtell <apurt...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f750acb7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f750acb7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f750acb7 Branch: refs/heads/0.98 Commit: f750acb751c5cc3b4a3292451c0504942b62bf39 Parents: 27c5834 Author: Jerry He <jerry...@apache.org> Authored: Sat Jul 9 18:18:22 2016 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Mon Aug 1 15:52:44 2016 -0700 ---------------------------------------------------------------------- .../hbase/util/CoprocessorClassLoader.java | 74 +++++++++++++------- .../hbase/util/TestCoprocessorClassLoader.java | 43 ++++++++++++ 2 files changed, 93 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java index fb2a127..9b65807 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.util; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; @@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.io.IOUtils; import com.google.common.base.Preconditions; @@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase { super(parent); } - private void init(Path path, String pathPrefix, + private void init(Path pathPattern, String pathPrefix, Configuration conf) throws IOException { // Copy the jar to the local filesystem String parentDirStr = @@ -173,31 +177,53 @@ public class CoprocessorClassLoader extends ClassLoaderBase { } } - FileSystem fs = path.getFileSystem(conf); - File dst = new File(parentDirStr, "." + pathPrefix + "." - + path.getName() + "." + System.currentTimeMillis() + ".jar"); - fs.copyToLocalFile(path, new Path(dst.toString())); - dst.deleteOnExit(); - - addURL(dst.getCanonicalFile().toURI().toURL()); + FileSystem fs = pathPattern.getFileSystem(conf); + Path pathPattern1 = fs.isDirectory(pathPattern) ? + new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified + FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern + if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches + throw new FileNotFoundException(pathPattern1.toString()); + } else { + boolean validFileEncountered = false; + for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern + if (fs.isFile(path)) { // only process files, skip for directories + File dst = new File(parentDirStr, "." + pathPrefix + "." + + path.getName() + "." + System.currentTimeMillis() + ".jar"); + fs.copyToLocalFile(path, new Path(dst.toString())); + dst.deleteOnExit(); + + addURL(dst.getCanonicalFile().toURI().toURL()); + + JarFile jarFile = new JarFile(dst.toString()); + try { + Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + Matcher m = libJarPattern.matcher(entry.getName()); + if (m.matches()) { + File file = new File(parentDirStr, "." + pathPrefix + "." + + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1)); + FileOutputStream outStream = new FileOutputStream(file); + try { + IOUtils.copyBytes(jarFile.getInputStream(entry), + outStream, conf, true); + } finally { + outStream.close(); + } + file.deleteOnExit(); + addURL(file.toURI().toURL()); + } + } + } finally { + jarFile.close(); + } - JarFile jarFile = new JarFile(dst.toString()); - try { - Enumeration<JarEntry> entries = jarFile.entries(); - while (entries.hasMoreElements()) { - JarEntry entry = entries.nextElement(); - Matcher m = libJarPattern.matcher(entry.getName()); - if (m.matches()) { - File file = new File(parentDirStr, "." + pathPrefix + "." - + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1)); - IOUtils.copyBytes(jarFile.getInputStream(entry), - new FileOutputStream(file), conf, true); - file.deleteOnExit(); - addURL(file.toURI().toURL()); + validFileEncountered = true; // Set to true when encountering a file } } - } finally { - jarFile.close(); + if (validFileEncountered == false) { // all items returned by globStatus() are directories + throw new FileNotFoundException("No file found matching " + pathPattern1.toString()); + } } } @@ -238,7 +264,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase { return cl; } - if (!pathStr.endsWith(".jar")) { + if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) { throw new IOException(pathStr + ": not a jar file?"); } http://git-wip-us.apache.org/repos/asf/hbase/blob/f750acb7/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java index f4b2002..e1048da 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import java.io.File; @@ -108,4 +109,46 @@ public class TestCoprocessorClassLoader { } fail("Could not find the expected lib jar file"); } + + // HBASE-14548 + @Test + public void testDirectoryAndWildcard() throws Exception { + String testClassName = "TestClass"; + String dataTestDir = TEST_UTIL.getDataTestDir().toString(); + System.out.println(dataTestDir); + String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf); + ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar); + ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader(); + CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered + + CoprocessorClassLoader coprocessorClassLoader = null; + Path testPath = null; + + // Directory + testPath = new Path(localDirContainingJar); + coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf); + verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName); + + // Wildcard - *.jar + testPath = new Path(localDirContainingJar, "*.jar"); + coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf); + verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName); + + // Wildcard - *.j* + testPath = new Path(localDirContainingJar, "*.j*"); + coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf); + verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName); + } + + /** + * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully + * @param coprocessorClassLoader the CoprocessorClassLoader to verify + * @param className the expected class to be loaded by the coprocessorClassLoader + * @throws ClassNotFoundException + */ + private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className) + throws ClassNotFoundException{ + assertNotNull("Classloader should be created and not null", coprocessorClassLoader); + assertEquals(className, coprocessorClassLoader.loadClass(className).getName()); + } }
