Repository: sqoop Updated Branches: refs/heads/sqoop2 7e5075b2a -> 52e9c11ad
SQOOP-2578: Sqoop2: Port the ApplicationClassLoader in Hadoop into Sqoop (Dian Fu via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/52e9c11a Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/52e9c11a Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/52e9c11a Branch: refs/heads/sqoop2 Commit: 52e9c11ad25c7063e9b0790d718d740be6537797 Parents: 7e5075b Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Sep 28 09:45:55 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Sep 28 09:45:55 2015 -0700 ---------------------------------------------------------------------- common/pom.xml | 18 -- .../sqoop/utils/ConnectorClassLoader.java | 248 +++++++++++++++++++ ...pache.sqoop.connector-classloader.properties | 55 ++++ .../sqoop/utils/TestConnectorClassLoader.java | 149 +++++++++++ 4 files changed, 452 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 748fa7a..078a785 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -39,24 +39,6 @@ limitations under the License. </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-jobclient</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>com.googlecode.json-simple</groupId> <artifactId>json-simple</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java new file mode 100644 index 0000000..69e972d --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.apache.log4j.Logger; + +/** + * A {@link URLClassLoader} for connector isolation. Classes from the + * connector JARs are loaded in preference to the parent loader. + */ +public class ConnectorClassLoader extends URLClassLoader { + /** + * Default value of the system classes if the user did not override them. + * JDK classes, sqoop classes and resources, and some select third-party + * classes are considered system classes, and are not loaded by the + * connector classloader. + */ + public static final String SYSTEM_CLASSES_DEFAULT; + + private static final String PROPERTIES_FILE = + "org.apache.sqoop.connector-classloader.properties"; + private static final String SYSTEM_CLASSES_DEFAULT_KEY = + "system.classes.default"; + + private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class); + + private static final FilenameFilter JAR_FILENAME_FILTER = + new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar") || name.endsWith(".JAR"); + } + }; + + static { + try (InputStream is = ConnectorClassLoader.class.getClassLoader() + .getResourceAsStream(PROPERTIES_FILE);) { + if (is == null) { + throw new ExceptionInInitializerError("properties file " + + PROPERTIES_FILE + " is not found"); + } + Properties props = new Properties(); + props.load(is); + // get the system classes default + String systemClassesDefault = + props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY); + if (systemClassesDefault == null) { + throw new ExceptionInInitializerError("property " + + SYSTEM_CLASSES_DEFAULT_KEY + " is not found"); + } + SYSTEM_CLASSES_DEFAULT = systemClassesDefault; + } catch (IOException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final ClassLoader parent; + private final List<String> systemClasses; + + public ConnectorClassLoader(URL[] urls, ClassLoader parent, + List<String> systemClasses) { + super(urls, parent); + if (LOG.isDebugEnabled()) { + LOG.debug("urls: " + Arrays.toString(urls)); + LOG.debug("system classes: " + systemClasses); + } + this.parent = parent; + if (parent == null) { + throw new IllegalArgumentException("No parent classloader!"); + } + // if the caller-specified system classes are null or empty, use the default + this.systemClasses = (systemClasses == null || systemClasses.isEmpty()) ? + Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")) : + systemClasses; + LOG.info("system classes: " + this.systemClasses); + } + + public ConnectorClassLoader(String classpath, ClassLoader parent, + List<String> systemClasses) throws MalformedURLException { + this(constructUrlsFromClasspath(classpath), parent, systemClasses); + } + + static URL[] constructUrlsFromClasspath(String classpath) + throws MalformedURLException { + List<URL> urls = new ArrayList<URL>(); + for (String element : classpath.split(File.pathSeparator)) { + if (element.endsWith("/*")) { + String dir = element.substring(0, element.length() - 1); + File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER); + if (files != null) { + for (File file : files) { + urls.add(file.toURI().toURL()); + } + } + } else { + File file = new File(element); + if (file.exists()) { + urls.add(new File(element).toURI().toURL()); + } + } + } + return urls.toArray(new URL[urls.size()]); + } + + @Override + public URL getResource(String name) { + URL url = null; + + if (!isSystemClass(name, systemClasses)) { + url= findResource(name); + if (url == null && name.startsWith("/")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Remove leading / off " + name); + } + url= findResource(name.substring(1)); + } + } + + if (url == null) { + url= parent.getResource(name); + } + + if (url != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getResource("+name+")=" + url); + } + } + + return url; + } + + @Override + public Class<?> loadClass(String name) throws ClassNotFoundException { + return this.loadClass(name, false); + } + + @Override + protected synchronized Class<?> loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (LOG.isDebugEnabled()) { + LOG.debug("Loading class: " + name); + } + + Class<?> c = findLoadedClass(name); + ClassNotFoundException ex = null; + + if (c == null && !isSystemClass(name, systemClasses)) { + // Try to load class from this classloader's URLs. Note that this is like + // the servlet spec, not the usual Java 2 behaviour where we ask the + // parent to attempt to load first. + try { + c = findClass(name); + if (LOG.isDebugEnabled() && c != null) { + LOG.debug("Loaded class: " + name + " "); + } + } catch (ClassNotFoundException e) { + if (LOG.isDebugEnabled()) { + LOG.debug(e); + } + ex = e; + } + } + + if (c == null) { // try parent + c = parent.loadClass(name); + if (LOG.isDebugEnabled() && c != null) { + LOG.debug("Loaded class from parent: " + name + " "); + } + } + + if (c == null) { + throw ex != null ? ex : new ClassNotFoundException(name); + } + + if (resolve) { + resolveClass(c); + } + + return c; + } + + /** + * Checks if a class should be included as a system class. + * + * A class is a system class if and only if it matches one of the positive + * patterns and none of the negative ones. + * + * @param name the class name to check + * @param systemClasses a list of system class configurations. + * @return true if the class is a system class + */ + public static boolean isSystemClass(String name, List<String> systemClasses) { + boolean result = false; + if (systemClasses != null) { + String canonicalName = name.replace('/', '.'); + while (canonicalName.startsWith(".")) { + canonicalName=canonicalName.substring(1); + } + for (String c : systemClasses) { + boolean shouldInclude = true; + if (c.startsWith("-")) { + c = c.substring(1); + shouldInclude = false; + } + if (canonicalName.startsWith(c)) { + if ( c.endsWith(".") // package + || canonicalName.length() == c.length() // class + || canonicalName.length() > c.length() // nested + && canonicalName.charAt(c.length()) == '$' ) { + if (shouldInclude) { + result = true; + } else { + return false; + } + } + } + } + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties new file mode 100644 index 0000000..e2936a9 --- /dev/null +++ b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# contains key properties for setting up the connector classloader +system.classes.default=java.,\ + javax.accessibility.,\ + javax.activation.,\ + javax.activity.,\ + javax.annotation.,\ + javax.annotation.processing.,\ + javax.crypto.,\ + javax.imageio.,\ + javax.jws.,\ + javax.lang.model.,\ + -javax.management.j2ee.,\ + javax.management.,\ + javax.naming.,\ + javax.net.,\ + javax.print.,\ + javax.rmi.,\ + javax.script.,\ + -javax.security.auth.message.,\ + javax.security.auth.,\ + javax.security.cert.,\ + javax.security.sasl.,\ + javax.sound.,\ + javax.sql.,\ + javax.swing.,\ + javax.tools.,\ + javax.transaction.,\ + -javax.xml.registry.,\ + -javax.xml.rpc.,\ + javax.xml.,\ + org.w3c.dom.,\ + org.xml.sax.,\ + org.apache.commons.logging.,\ + org.apache.log4j.,\ + org.apache.sqoop.,\ + sqoop.properties,\ + sqoop_bootstrap.properties http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java new file mode 100644 index 0000000..442be4d --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import static org.apache.sqoop.utils.ConnectorClassLoader.constructUrlsFromClasspath; +import static org.apache.sqoop.utils.ConnectorClassLoader.isSystemClass; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.List; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + +public class TestConnectorClassLoader { + private static File testDir = new File(System.getProperty("maven.build.directory", + System.getProperty("java.io.tmpdir")), "connectorclassloader"); + + @BeforeMethod(alwaysRun = true) + public void setUp() { + FileUtils.deleteQuietly(testDir); + testDir.mkdirs(); + } + + @Test + public void testConstructUrlsFromClasspath() throws Exception { + File file = new File(testDir, "file"); + assertTrue(file.createNewFile(), "Create file"); + + File dir = new File(testDir, "dir"); + assertTrue(dir.mkdir(), "Make dir"); + + File jarsDir = new File(testDir, "jarsdir"); + assertTrue(jarsDir.mkdir(), "Make jarsDir"); + File nonJarFile = new File(jarsDir, "nonjar"); + assertTrue(nonJarFile.createNewFile(), "Create non-jar file"); + File jarFile = new File(jarsDir, "a.jar"); + assertTrue(jarFile.createNewFile(), "Create jar file"); + + File nofile = new File(testDir, "nofile"); + // don't create nofile + + StringBuilder cp = new StringBuilder(); + cp.append(file.getAbsolutePath()).append(File.pathSeparator) + .append(dir.getAbsolutePath()).append(File.pathSeparator) + .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator) + .append(nofile.getAbsolutePath()).append(File.pathSeparator) + .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator); + + URL[] urls = constructUrlsFromClasspath(cp.toString()); + + assertEquals(3, urls.length); + assertEquals(file.toURI().toURL(), urls[0]); + assertEquals(dir.toURI().toURL(), urls[1]); + assertEquals(jarFile.toURI().toURL(), urls[2]); + // nofile should be ignored + } + + @Test + public void testIsSystemClass() { + testIsSystemClassInternal(""); + } + + @Test + public void testIsSystemNestedClass() { + testIsSystemClassInternal("$Klass"); + } + + private void testIsSystemClassInternal(String nestedClass) { + assertFalse(isSystemClass("org.example.Foo" + nestedClass, null)); + assertTrue(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example.Foo"))); + assertTrue(isSystemClass("/org.example.Foo" + nestedClass, + classes("org.example.Foo"))); + assertTrue(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example."))); + assertTrue(isSystemClass("net.example.Foo" + nestedClass, + classes("org.example.,net.example."))); + assertFalse(isSystemClass("org.example.Foo" + nestedClass, + classes("-org.example.Foo,org.example."))); + assertTrue(isSystemClass("org.example.Bar" + nestedClass, + classes("-org.example.Foo.,org.example."))); + assertFalse(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example.,-org.example.Foo"))); + assertFalse(isSystemClass("org.example.Foo" + nestedClass, + classes("org.example.Foo,-org.example.Foo"))); + } + + private List<String> classes(String classes) { + return Lists.newArrayList(Splitter.on(',').split(classes)); + } + + @Test + public void testGetResource() throws IOException { + URL testJar = makeTestJar().toURI().toURL(); + + ClassLoader currentClassLoader = getClass().getClassLoader(); + ClassLoader connectorClassloader = new ConnectorClassLoader( + new URL[] { testJar }, currentClassLoader, null); + + assertNull(currentClassLoader.getResourceAsStream("resource.txt"), + "Resource should be null for current classloader"); + + InputStream in = connectorClassloader.getResourceAsStream("resource.txt"); + assertNotNull(in, "Resource should not be null for connector classloader"); + assertEquals("hello", IOUtils.toString(in)); + } + + private File makeTestJar() throws IOException { + File jarFile = new File(testDir, "test.jar"); + JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile)); + ZipEntry entry = new ZipEntry("resource.txt"); + out.putNextEntry(entry); + out.write("hello".getBytes()); + out.closeEntry(); + out.close(); + return jarFile; + } +}
