Repository: sqoop Updated Branches: refs/heads/sqoop2 89e377a4d -> 5abd3f91f
SQOOP-2342: Sqoop2: Provide an external application classloader (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5abd3f91 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5abd3f91 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5abd3f91 Branch: refs/heads/sqoop2 Commit: 5abd3f91f0d7c4480aa8b0f4f7d53c8304c2c276 Parents: 89e377a Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Apr 30 14:25:43 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Apr 30 14:25:43 2015 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/error/code/CoreError.java | 5 +- .../sqoop/connector/ConnectorManager.java | 6 -- .../sqoop/connector/ConnectorManagerUtils.java | 64 -------------------- .../sqoop/core/ConfigurationConstants.java | 6 +- .../apache/sqoop/core/SqoopConfiguration.java | 53 ++++++++++++++++ .../connector/TestConnectorManagerUtils.java | 28 --------- .../sqoop/core/TestSqoopConfiguration.java | 3 + core/src/test/resources/test_config.properties | 3 + dist/src/main/server/conf/sqoop.properties | 5 ++ docs/src/site/sphinx/ConnectorDevelopment.rst | 9 ++- .../test/minicluster/SqoopMiniCluster.java | 7 --- 11 files changed, 74 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/common/src/main/java/org/apache/sqoop/error/code/CoreError.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/error/code/CoreError.java b/common/src/main/java/org/apache/sqoop/error/code/CoreError.java index 24e73c3..bc3076c 100644 --- a/common/src/main/java/org/apache/sqoop/error/code/CoreError.java +++ b/common/src/main/java/org/apache/sqoop/error/code/CoreError.java @@ -52,7 +52,10 @@ public enum CoreError implements ErrorCode { CORE_0007("System not initialized"), /** The system has not been reconfigured */ - CORE_0008("System not reconfigured"); + CORE_0008("System not reconfigured"), + + /** Sqoop classpath error */ + CORE_0009("Sqoop classpath error"), ; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index 0517413..d1537bf 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.ResourceBundle; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.SqoopConnector; @@ -162,11 +161,6 @@ public class ConnectorManager implements Reconfigurable { LOG.trace("Begin connector manager initialization"); } - // add external connectors into the class path - // NOTE: class loading happens later in the ConnectorHandler - ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext() - .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY)); - List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(); LOG.info("Connector config urls: " + connectorConfigs); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java index 99736c6..7402c5a 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java @@ -17,21 +17,16 @@ */ package org.apache.sqoop.connector; -import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.ConfigurationConstants; import org.apache.sqoop.error.code.ConnectorError; import java.io.File; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; -import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Enumeration; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -75,25 +70,6 @@ public class ConnectorManagerUtils { return connectorConfigs; } - public static Set<File> getConnectorJars(String path) { - if (StringUtils.isEmpty(path)) { - return null; - } - Set<File> jarFiles = new HashSet<File>(); - File folder = new File(path); - if (folder.exists()) { - for (File file : folder.listFiles()) { - if (file.isDirectory()) { - jarFiles.addAll(getConnectorJars(file.getPath())); - } - if (file.getName().endsWith(".jar") && isConnectorJar(file)) { - jarFiles.add(file); - } - } - } - return jarFiles; - } - static boolean isConnectorJar(File file) { try { @SuppressWarnings("resource") @@ -103,44 +79,4 @@ public class ConnectorManagerUtils { throw new RuntimeException(e); } } - - public static void addExternalConnectorsJarsToClasspath(String path) { - if (StringUtils.isEmpty(path)) { - return; - } - - ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader(); - if (currentThreadClassLoader != null) { - - // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath - // Chain the current thread classloader - ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new URL[] {}, - currentThreadClassLoader); - // the property always holds a path to the folder containing the jars - Set<File> connectorJars = getConnectorJars(path); - if (connectorJars != null && !connectorJars.isEmpty()) { - for (File jar : connectorJars) { - connectorUrlClassLoader.addJarFile(jar.getPath()); - } - // Replace the thread classloader- assuming there is permission to do so - Thread.currentThread().setContextClassLoader(connectorUrlClassLoader); - } - } - } - - public static class ExternalConnectorJarFileLoader extends URLClassLoader { - public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - } - - public void addJarFile(String path) { - String urlPath = "jar:file://" + path + "!/"; - try { - addURL(new URL(urlPath)); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } - } - - } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java index 49e5d6f..d7fe27b 100644 --- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java +++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java @@ -77,11 +77,9 @@ public final class ConfigurationConstants { "org.apache.sqoop.driver.autoupgrade"; /** - # Support loading external connector jars only - # The loader will look for sqoopconnector.properties file in the jar before loading - # "/path/to/external/connectors/": Add all the connector JARs in the specified folder + * Add external jars to application classpath. */ - public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath"; + public static final String CLASSPATH = "org.apache.sqoop.classpath.extra"; /** * Enable Sqoop App to kill Tomcat in case that it will fail to load. http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java index 15ee12f..58c4c34 100644 --- a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java +++ b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java @@ -21,9 +21,18 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import org.apache.sqoop.classification.InterfaceAudience; @@ -175,6 +184,8 @@ public class SqoopConfiguration implements Reconfigurable { provider.registerListener(new CoreConfigurationListener(SqoopConfiguration.getInstance())); initialized = true; + + configureClassLoader(ConfigurationConstants.CLASSPATH); } public synchronized MapContext getContext() { @@ -212,6 +223,48 @@ public class SqoopConfiguration implements Reconfigurable { initialized = false; } + /** + * Load extra classpath from sqoop configuration. + * @param classpathProperty + */ + private synchronized void configureClassLoader(String classpathProperty) { + LOG.info("Adding jars to current classloader from property: " + classpathProperty); + + String classpath = getContext().getString(classpathProperty); + + if (StringUtils.isEmpty(classpath)) { + LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra jars."); + return; + } + + ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader(); + if (currentThreadClassLoader == null) { + throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override."); + } + + // CSV URL list separated by ":". + Set<String> paths = new HashSet(Arrays.asList(classpath.split(":"))); + List<URL> urls = new LinkedList<URL>(); + + for (String path : paths) { + try { + LOG.debug("Found jar in path: " + path); + URL url = new File(path).toURI().toURL(); + urls.add(url); + LOG.debug("Using URL: " + url.toString()); + } catch (MalformedURLException e) { + throw new SqoopException(CoreError.CORE_0009, "Malformed URL found.", e); + } + } + + // Chain the current thread classloader so that + // configured classpath adds to existing classloader. + // Existing classpath is not changed. + URLClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), + currentThreadClassLoader); + Thread.currentThread().setContextClassLoader(classLoader); + } + private synchronized void configureLogging() { Properties props = new Properties(); for (String key : config.keySet()) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java index f2e64cd..258b5f9 100644 --- a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java +++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java @@ -20,16 +20,10 @@ package org.apache.sqoop.connector; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertTrue; -import static org.testng.AssertJUnit.assertNull; -import static org.testng.AssertJUnit.assertEquals; -import org.apache.sqoop.utils.ClassUtils; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; -import java.net.URL; -import java.util.List; -import java.util.Set; public class TestConnectorManagerUtils { @@ -41,19 +35,6 @@ public class TestConnectorManagerUtils { } @Test - public void testGetConnectorJarsNullPath() { - Set<File> files = ConnectorManagerUtils.getConnectorJars(null); - assertNull(files); - } - - @Test - public void testGetConnectorJarsNonNullPath() { - String path = workingDir + "/src/test/resources"; - Set<File> files = ConnectorManagerUtils.getConnectorJars(path); - assertEquals(1, files.size()); - } - - @Test public void testIsConnectorJar() { String path = workingDir + "/src/test/resources/test-connector.jar"; File connectorJar = new File(path); @@ -69,13 +50,4 @@ public class TestConnectorManagerUtils { assertFalse(ConnectorManagerUtils.isConnectorJar(file)); } - @Test - public void testAddExternalConnectorJarToClasspath() { - String path = workingDir + "/src/test/resources"; - ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path); - List<URL> urls = ConnectorManagerUtils.getConnectorConfigs(); - assertEquals(1, urls.size()); - ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector"); - } - } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java index b11587c..499c560 100644 --- a/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java +++ b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java @@ -162,5 +162,8 @@ public class TestSqoopConfiguration { public void testConfigurationInitSuccess() throws Exception { TestUtils.setupTestConfigurationWithExtraConfig(null, null); SqoopConfiguration.getInstance().initialize(); + + // Make sure "test" is in classpath. + Assert.assertNotNull(Thread.currentThread().getContextClassLoader().getResource("test_config.properties")); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/resources/test_config.properties ---------------------------------------------------------------------- diff --git a/core/src/test/resources/test_config.properties b/core/src/test/resources/test_config.properties index 4ad1267..50aeb3b 100644 --- a/core/src/test/resources/test_config.properties +++ b/core/src/test/resources/test_config.properties @@ -23,3 +23,6 @@ org.apache.sqoop.log4j.logger.org.apache.sqoop=debug,console org.apache.sqoop.log4j.appender.console=org.apache.log4j.ConsoleAppender org.apache.sqoop.log4j.appender.console.layout=org.apache.log4j.PatternLayout org.apache.sqoop.log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Extra classpath +org.apache.sqoop.classpath.extra=core/src/test/resources/test_config.properties \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/dist/src/main/server/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index 5226a19..ba6e09f 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -171,3 +171,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce # "/path/to/external/connectors/": Add all the connector JARs in the specified folder # org.apache.sqoop.connector.external.loadpath= + +# Sqoop application classpath +# ":" separated list of jars to be included in sqoop. +# +org.apache.sqoop.classpath.extra= http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/docs/src/site/sphinx/ConnectorDevelopment.rst ---------------------------------------------------------------------- diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst index 2d9e7d2..1ea1881 100644 --- a/docs/src/site/sphinx/ConnectorDevelopment.rst +++ b/docs/src/site/sphinx/ConnectorDevelopment.rst @@ -483,15 +483,14 @@ Loading new connector say sqoop-foo-connector to the sqoop2, here are the steps 1. Create a ``sqoop-foo-connector.jar``. Make sure the jar contains the ``sqoopconnector.properties`` for it to be picked up by Sqoop -2. Add this jar to the a folder on your installation machine and update the path to this folder in the sqoop.properties located under the ``server/conf`` directory under the Sqoop2 for the key ``org.apache.sqoop.connector.external.loadpath`` +2. Add this jar to the ``org.apache.sqoop.classpath.extra`` property in the sqoop.properties located under the ``server/conf`` directory. :: + # Sqoop application classpath + # ":" separated list of jars to be included in sqoop. # - # External connectors load path - # "/path/to/external/connectors/": Add all the connector JARs in the specified folder - # - org.apache.sqoop.connector.external.loadpath=/path/to/connector + org.apache.sqoop.classpath.extra=/path/to/connector.jar 3. Start the Sqoop 2 server and while initializing the server this jar should be loaded into the Sqoop 2's class path and registered into the Sqoop 2 repository http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index ad45189..6ec6883 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -132,7 +132,6 @@ public abstract class SqoopMiniCluster { mapToProperties(sqoopProperties, getSecurityConfiguration()); mapToProperties(sqoopProperties, getConnectorManagerConfiguration()); mapToProperties(sqoopProperties, getDriverManagerConfiguration()); - mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration()); FileUtils.writeLines(f, sqoopProperties); @@ -222,10 +221,4 @@ public abstract class SqoopMiniCluster { properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true"); return properties; } - - protected Map<String, String> getExternalConnectorLoadPathConfiguration() { - Map<String, String> properties = new HashMap<String, String>(); - properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, ""); - return properties; - } }
