Repository: sqoop Updated Branches: refs/heads/sqoop2 49f104bb5 -> 913521e1a
SQOOP-1821: Sqoop2: External connector loading (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/913521e1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/913521e1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/913521e1 Branch: refs/heads/sqoop2 Commit: 913521e1a93b2571d2c83b10c51609b54744eee9 Parents: 49f104b Author: Abraham Elmahrek <[email protected]> Authored: Tue Jan 20 12:50:15 2015 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Tue Jan 20 12:50:15 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/sqoop/utils/ClassUtils.java | 2 + .../org/apache/sqoop/utils/TestClassUtils.java | 2 +- .../sqoop/connector/ConnectorHandler.java | 5 +- .../sqoop/connector/ConnectorManager.java | 6 ++ .../sqoop/connector/ConnectorManagerUtils.java | 87 +++++++++++++++++-- .../sqoop/core/ConfigurationConstants.java | 7 ++ .../connector/TestConnectorManagerUtils.java | 64 ++++++++++++++ core/src/test/resources/test-connector.jar | Bin 0 -> 55938 bytes core/src/test/resources/test-non-connector.jar | Bin 0 -> 19827 bytes dist/src/main/server/conf/sqoop.properties | 8 +- .../test/minicluster/SqoopMiniCluster.java | 13 +-- 11 files changed, 179 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java index 0be4d41..d6a8254 100644 --- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java +++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java @@ -57,6 +57,8 @@ public final class ClassUtils { klass = ctxLoader.loadClass(className); } catch (ClassNotFoundException ex) { LOG.debug("Exception while load class: " + className, ex); + // wrapping it in runtime, to avoid chainging the signature of methods currently invoking this method + throw new RuntimeException(ex); } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java index 161a1fa..58a27ba 100644 --- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java +++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java @@ -29,7 +29,7 @@ import static org.testng.Assert.assertNull; */ public class TestClassUtils { - @Test + @Test(expectedExceptions = Exception.class) public void testLoadClass() { assertNull(ClassUtils.loadClass("A")); assertEquals(A.class, ClassUtils.loadClass(A.class.getName())); http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java index 1919b4b..bfdb7b3 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java @@ -31,6 +31,7 @@ import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MLinkConfig; import org.apache.sqoop.model.MToConfig; +import org.apache.sqoop.utils.ClassUtils; public final class ConnectorHandler { @@ -71,8 +72,8 @@ public final class ConnectorHandler { Class<?> connectorClass = null; try { - connectorClass = Class.forName(connectorClassName); - } catch (ClassNotFoundException ex) { + connectorClass = ClassUtils.loadClass(connectorClassName); + } catch (Exception ex) { throw new SqoopException(ConnectorError.CONN_0005, connectorClassName, ex); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index b9d4d60..0907efb 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.ResourceBundle; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.SqoopConnector; @@ -158,6 +159,11 @@ public class ConnectorManager implements Reconfigurable { LOG.trace("Begin connector manager initialization"); } + // add external connectors into the class path + // NOTE: class loading happens later in the ConnectorHandler + ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext() + .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY)); + List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(); LOG.info("Connector config urls: " + connectorConfigs); http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java index c7193ee..bd85e99 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java @@ -17,14 +17,22 @@ */ package org.apache.sqoop.connector; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.ConfigurationConstants; +import java.io.File; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Enumeration; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; /** * Utilities for ConnectorManager. @@ -33,7 +41,7 @@ public class ConnectorManagerUtils { /** * Get a list of URLs of connectors that are installed. - * Check + * * @return List of URLs. */ public static List<URL> getConnectorConfigs() { @@ -41,9 +49,8 @@ public class ConnectorManagerUtils { try { // Check ConnectorManager classloader. - Enumeration<URL> appPathConfigs = - ConnectorManager.class.getClassLoader().getResources( - ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); + Enumeration<URL> appPathConfigs = ConnectorManager.class.getClassLoader().getResources( + ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); while (appPathConfigs.hasMoreElements()) { connectorConfigs.add(appPathConfigs.nextElement()); } @@ -51,8 +58,7 @@ public class ConnectorManagerUtils { // Check thread context classloader. ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader(); if (ctxLoader != null) { - Enumeration<URL> ctxPathConfigs = ctxLoader.getResources( - ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); + Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); while (ctxPathConfigs.hasMoreElements()) { URL configUrl = ctxPathConfigs.nextElement(); @@ -67,4 +73,73 @@ public class ConnectorManagerUtils { return connectorConfigs; } + + public static Set<File> getConnectorJars(String path) { + if (StringUtils.isEmpty(path)) { + return null; + } + Set<File> jarFiles = new HashSet<File>(); + File folder = new File(path); + if (folder.exists()) { + for (File file : folder.listFiles()) { + if (file.isDirectory()) { + jarFiles.addAll(getConnectorJars(file.getPath())); + } + if (file.getName().endsWith(".jar") && isConnectorJar(file)) { + jarFiles.add(file); + } + } + } + return jarFiles; + } + + static boolean isConnectorJar(File file) { + try { + @SuppressWarnings("resource") + JarEntry entry = new JarFile(file).getJarEntry(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); + return entry != null; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static void addExternalConnectorsJarsToClasspath(String path) { + if (StringUtils.isEmpty(path)) { + return; + } + + ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader(); + if (currentThreadClassLoader != null) { + + // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath + // Chain the current thread classloader + ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new URL[] {}, + currentThreadClassLoader); + // the property always holds a path to the folder containing the jars + Set<File> connectorJars = getConnectorJars(path); + if (connectorJars != null && !connectorJars.isEmpty()) { + for (File jar : connectorJars) { + connectorUrlClassLoader.addJarFile(jar.getPath()); + } + // Replace the thread classloader- assuming there is permission to do so + Thread.currentThread().setContextClassLoader(connectorUrlClassLoader); + } + } + } + + public static class ExternalConnectorJarFileLoader extends URLClassLoader { + public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + public void addJarFile(String path) { + String urlPath = "jar:file://" + path + "!/"; + try { + addURL(new URL(urlPath)); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java index f341108..49e5d6f 100644 --- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java +++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java @@ -77,6 +77,13 @@ public final class ConfigurationConstants { "org.apache.sqoop.driver.autoupgrade"; /** + # Support loading external connector jars only + # The loader will look for sqoopconnector.properties file in the jar before loading + # "/path/to/external/connectors/": Add all the connector JARs in the specified folder + */ + public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath"; + + /** * Enable Sqoop App to kill Tomcat in case that it will fail to load. */ public static final String KILL_TOMCAT_ON_FAILURE = "sqoop.kill_tomcat_on_load_failure"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java new file mode 100644 index 0000000..18f8edb --- /dev/null +++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java @@ -0,0 +1,64 @@ +package org.apache.sqoop.connector; + +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertEquals; +import org.apache.sqoop.utils.ClassUtils; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.testng.AssertJUnit; + +import java.io.File; +import java.net.URL; +import java.util.List; +import java.util.Set; + +public class TestConnectorManagerUtils { + + private String workingDir; + + @BeforeMethod(alwaysRun = true) + public void setUp() { + workingDir = System.getProperty("user.dir"); + } + + @Test + public void testGetConnectorJarsNullPath() { + Set<File> files = ConnectorManagerUtils.getConnectorJars(null); + assertNull(files); + } + + @Test + public void testGetConnectorJarsNonNullPath() { + String path = workingDir + "/src/test/resources"; + Set<File> files = ConnectorManagerUtils.getConnectorJars(path); + assertEquals(1, files.size()); + } + + @Test + public void testIsConnectorJar() { + String path = workingDir + "/src/test/resources/test-connector.jar"; + File connectorJar = new File(path); + assertTrue(connectorJar.exists()); + assertTrue(ConnectorManagerUtils.isConnectorJar(connectorJar)); + } + + @Test + public void testIsNotConnectorJar() { + String path = workingDir + "/src/test/resources/test-non-connector.jar"; + File file = new File(path); + assertTrue(file.exists()); + assertFalse(ConnectorManagerUtils.isConnectorJar(file)); + } + + @Test + public void testAddExternalConnectorJarToClasspath() { + String path = workingDir + "/src/test/resources"; + ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path); + List<URL> urls = ConnectorManagerUtils.getConnectorConfigs(); + assertEquals(1, urls.size()); + ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector"); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-connector.jar ---------------------------------------------------------------------- diff --git a/core/src/test/resources/test-connector.jar b/core/src/test/resources/test-connector.jar new file mode 100644 index 0000000..fd3b116 Binary files /dev/null and b/core/src/test/resources/test-connector.jar differ http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-non-connector.jar ---------------------------------------------------------------------- diff --git a/core/src/test/resources/test-non-connector.jar b/core/src/test/resources/test-non-connector.jar new file mode 100644 index 0000000..01f82ca Binary files /dev/null and b/core/src/test/resources/test-non-connector.jar differ http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/dist/src/main/server/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index 824c5eb..fbcb1fa 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -163,4 +163,10 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce # #org.apache.sqoop.security.authorization.handler=org.apache.sqoop.security.Authorization.DefaultAuthorizationHandler #org.apache.sqoop.security.authorization.access_controller=org.apache.sqoop.security.Authorization.DefaultAuthorizationAccessController -#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator \ No newline at end of file +#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator + + +# External connectors load path +# "/path/to/external/connectors/": Add all the connector JARs in the specified folder +# +org.apache.sqoop.connector.external.loadpath= http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index e4eecbf..ad45189 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -132,6 +132,7 @@ public abstract class SqoopMiniCluster { mapToProperties(sqoopProperties, getSecurityConfiguration()); mapToProperties(sqoopProperties, getConnectorManagerConfiguration()); mapToProperties(sqoopProperties, getDriverManagerConfiguration()); + mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration()); FileUtils.writeLines(f, sqoopProperties); @@ -212,17 +213,19 @@ public abstract class SqoopMiniCluster { protected Map<String, String> getConnectorManagerConfiguration() { Map<String, String> properties = new HashMap<String, String>(); - - properties.put("org.apache.sqoop.connector.autoupgrade", "true"); - + properties.put(ConfigurationConstants.CONNECTOR_AUTO_UPGRADE, "true"); return properties; } protected Map<String, String> getDriverManagerConfiguration() { Map<String, String> properties = new HashMap<String, String>(); + properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true"); + return properties; + } - properties.put("org.apache.sqoop.driver.autoupgrade", "true"); - + protected Map<String, String> getExternalConnectorLoadPathConfiguration() { + Map<String, String> properties = new HashMap<String, String>(); + properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, ""); return properties; } }
