Repository: sqoop Updated Branches: refs/heads/sqoop2 c54f92854 -> f33554a19
SQOOP-2629: Sqoop2: Add ability to "blacklist" connectors (Abraham Fine via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f33554a1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f33554a1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f33554a1 Branch: refs/heads/sqoop2 Commit: f33554a19e287ec38e331d46bc8fad4ae1e5d9aa Parents: c54f928 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Oct 27 20:14:19 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Oct 27 20:14:19 2015 -0700 ---------------------------------------------------------------------- .../sqoop/connector/ConnectorManager.java | 17 +- .../sqoop/connector/ConnectorManagerUtils.java | 25 +- .../sqoop/core/ConfigurationConstants.java | 5 + .../connector/TestConnectorManagerUtils.java | 29 +- .../org/apache/sqoop/driver/TestJobManager.java | 9 + .../sqoop/repository/TestJdbcRepository.java | 7 + dist/src/main/conf/sqoop.properties | 5 + .../derby/DerbyRepositoryHandler.java | 3 +- .../test/minicluster/SqoopMiniCluster.java | 5 + .../integration/classpath/ClasspathTest.java | 333 ------------------- .../BlacklistedConnectorTest.java | 71 ++++ .../connectorloading/ClasspathTest.java | 333 +++++++++++++++++++ .../test/resources/classpath-tests-suite.xml | 6 +- 13 files changed, 501 insertions(+), 347 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index 285e893..f606950 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -18,6 +18,7 @@ package org.apache.sqoop.connector; import java.net.URL; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -27,6 +28,7 @@ import java.util.Map; import java.util.ResourceBundle; import java.util.Set; +import org.apache.commons.lang.ArrayUtils; import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.SqoopConnector; @@ -40,6 +42,7 @@ import org.apache.sqoop.model.MConnector; import org.apache.sqoop.repository.Repository; import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.repository.RepositoryTransaction; +import org.apache.sqoop.utils.ContextUtils; public class ConnectorManager implements Reconfigurable { @@ -58,6 +61,8 @@ public class ConnectorManager implements Reconfigurable { */ private static boolean DEFAULT_AUTO_UPGRADE = false; + private Set<String> blacklistedConnectors; + /** * Create default object by default. * @@ -70,7 +75,15 @@ public class ConnectorManager implements Reconfigurable { /** * The private constructor for the singleton class. */ - private ConnectorManager() {} + private ConnectorManager() { + String blacklistedConnectorsString = + SqoopConfiguration.getInstance().getContext().getString(ConfigurationConstants.BLACKLISTED_CONNECTORS); + if (blacklistedConnectorsString == null) { + blacklistedConnectors = Collections.EMPTY_SET; + } else { + blacklistedConnectors = ContextUtils.getUniqueStrings(blacklistedConnectorsString); + } + } /** * Return current instance. @@ -176,7 +189,7 @@ public class ConnectorManager implements Reconfigurable { LOG.trace("Begin connector manager initialization"); } - List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(); + List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(blacklistedConnectors); LOG.info("Connector config urls: " + connectorConfigs); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java index 522ed08..9f9be57 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java @@ -19,6 +19,7 @@ package org.apache.sqoop.connector; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.error.code.ConnectorError; import java.io.File; @@ -27,6 +28,8 @@ import java.net.URL; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; +import java.util.Properties; +import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -40,15 +43,18 @@ public class ConnectorManagerUtils { * * @return List of URLs. */ - public static List<URL> getConnectorConfigs() { - List<URL> connectorConfigs = new ArrayList<URL>(); + public static List<URL> getConnectorConfigs(Set<String> connectorBlacklist) { + List<URL> connectorConfigs = new ArrayList<>(); try { // Check ConnectorManager classloader. Enumeration<URL> appPathConfigs = ConnectorManager.class.getClassLoader().getResources( ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); while (appPathConfigs.hasMoreElements()) { - connectorConfigs.add(appPathConfigs.nextElement()); + URL connectorConfig = appPathConfigs.nextElement(); + if (!isBlacklisted(connectorConfig, connectorBlacklist)) { + connectorConfigs.add(connectorConfig); + } } // Check thread context classloader. @@ -58,7 +64,7 @@ public class ConnectorManagerUtils { while (ctxPathConfigs.hasMoreElements()) { URL configUrl = ctxPathConfigs.nextElement(); - if (!connectorConfigs.contains(configUrl)) { + if (!connectorConfigs.contains(configUrl) && !isBlacklisted(configUrl, connectorBlacklist)) { connectorConfigs.add(configUrl); } } @@ -70,6 +76,17 @@ public class ConnectorManagerUtils { return connectorConfigs; } + static boolean isBlacklisted(URL connectorConfig, Set<String> connectorBlacklist) throws IOException { + Properties properties = new Properties(); + properties.load(connectorConfig.openStream()); + String connectorName = properties.getProperty(ConfigurationConstants.CONNPROP_CONNECTOR_NAME); + if (connectorName == null) { + throw new IOException("malformed connector properties: " + connectorConfig.getPath()); + } else { + return connectorBlacklist.contains(connectorName); + } + } + static boolean isConnectorJar(File file) { try (JarFile jarFile = new JarFile(file)) { @SuppressWarnings("resource") http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java index f49fca3..a33bc05 100644 --- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java +++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java @@ -86,6 +86,11 @@ public final class ConfigurationConstants { */ public static final String JOB_CLASSPATH = "org.apache.sqoop.classpath.job"; + /** + * List of connectors that will not be loaded at startup + */ + public static final String BLACKLISTED_CONNECTORS = "org.apache.sqoop.connector.blacklist"; + private ConfigurationConstants() { // Disable explicit object creation } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java index fd2d1b4..423b3df 100644 --- a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java +++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java @@ -20,34 +20,55 @@ package org.apache.sqoop.connector; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; + +import org.apache.sqoop.core.ConfigurationConstants; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; +import java.net.URL; +import java.util.HashSet; +import java.util.Set; public class TestConnectorManagerUtils { private String workingDir; + private String testConnectorPath; + private String testNonConnectorPath; @BeforeMethod(alwaysRun = true) public void setUp() { workingDir = System.getProperty("user.dir"); + testConnectorPath = workingDir + "/src/test/resources/test-connector.jar"; + testNonConnectorPath = workingDir + "/src/test/resources/test-non-connector.jar"; } @Test public void testIsConnectorJar() { - String path = workingDir + "/src/test/resources/test-connector.jar"; - File connectorJar = new File(path); + File connectorJar = new File(testConnectorPath); assertTrue(connectorJar.exists()); assertTrue(ConnectorManagerUtils.isConnectorJar(connectorJar)); } @Test public void testIsNotConnectorJar() { - String path = workingDir + "/src/test/resources/test-non-connector.jar"; - File file = new File(path); + File file = new File(testNonConnectorPath); assertTrue(file.exists()); assertFalse(ConnectorManagerUtils.isConnectorJar(file)); } + @Test + public void testIsBlacklisted() throws Exception { + URL url = new URL("jar:file:" + testConnectorPath + "!/" + ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES); + + Set<String> blacklistedConnectors = new HashSet<>(); + // The test connector is a copy of the generic jdbc connector + blacklistedConnectors.add("generic-jdbc-connector"); + assertTrue(ConnectorManagerUtils.isBlacklisted(url, blacklistedConnectors)); + + blacklistedConnectors = new HashSet<>(); + blacklistedConnectors.add("not-there"); + assertFalse(ConnectorManagerUtils.isBlacklisted(url, blacklistedConnectors)); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java index 2d7dfef..f137eef 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java @@ -18,8 +18,10 @@ package org.apache.sqoop.driver; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; import static org.mockito.Mockito.mock; @@ -28,9 +30,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.common.MapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.ConnectorManager; import org.apache.sqoop.connector.spi.SqoopConnector; +import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.error.code.DriverError; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; @@ -48,9 +52,14 @@ public class TestJobManager { private ConnectorManager connectorMgrMock; private RepositoryManager repositoryManagerMock; private Repository jdbcRepoMock; + private SqoopConfiguration configurationMock; @BeforeMethod(alwaysRun = true) public void setUp() { + configurationMock = mock(SqoopConfiguration.class); + doReturn(new MapContext(Collections.EMPTY_MAP)).when(configurationMock).getContext(); + SqoopConfiguration.setInstance(configurationMock); + jobManager = JobManager.getInstance(); connectorMgrMock = mock(ConnectorManager.class); sqoopConnectorMock = mock(SqoopConnector.class); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java index bb4da65..00b0511 100644 --- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java +++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java @@ -40,11 +40,13 @@ import java.util.LinkedList; import java.util.List; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.common.MapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.ConnectorManager; import org.apache.sqoop.connector.common.EmptyConfiguration; import org.apache.sqoop.connector.spi.ConnectorConfigurableUpgrader; import org.apache.sqoop.connector.spi.SqoopConnector; +import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.driver.DriverUpgrader; import org.apache.sqoop.json.DriverBean; @@ -76,9 +78,14 @@ public class TestJdbcRepository { private JdbcRepositoryHandler repoHandlerMock; private ConnectorConfigurableUpgrader connectorUpgraderMock; private DriverUpgrader driverUpgraderMock; + private SqoopConfiguration configurationMock; @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { + configurationMock = mock(SqoopConfiguration.class); + doReturn(new MapContext(Collections.EMPTY_MAP)).when(configurationMock).getContext(); + SqoopConfiguration.setInstance(configurationMock); + repoTransactionMock = mock(JdbcRepositoryTransaction.class); connectorMgrMock = mock(ConnectorManager.class); driverMock = mock(Driver.class); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/dist/src/main/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 7c1ec18..2895530 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -199,3 +199,8 @@ org.apache.sqoop.classpath.job= #org.apache.sqoop.jetty.thread.pool.worker.min=5 #org.apache.sqoop.jetty.thread.pool.worker.alive.time=60 #org.apache.sqoop.jetty.port=12000 + +# Blacklisted Connectors +# ":" separated list of connector names as specified in their +# sqoopconnector.properties file +org.apache.sqoop.connector.blacklist= http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 0302150..ee5e8d1 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -32,6 +32,7 @@ import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -752,7 +753,7 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler { LOG.trace("Begin HDFS Connector pre-loading."); } - List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(); + List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs(Collections.EMPTY_SET); if (LOG.isInfoEnabled()) { LOG.info("Connector configs: " + connectorConfigs); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index 0f04bdf..9ae941f 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -140,6 +140,7 @@ public abstract class SqoopMiniCluster { mapToProperties(sqoopProperties, getConnectorManagerConfiguration()); mapToProperties(sqoopProperties, getDriverManagerConfiguration()); mapToProperties(sqoopProperties, getClasspathConfiguration()); + mapToProperties(sqoopProperties, getBlacklistedConnectorConfiguration()); FileUtils.writeLines(f, sqoopProperties); @@ -230,4 +231,8 @@ public abstract class SqoopMiniCluster { protected Map<String, String> getClasspathConfiguration() { return MapUtils.EMPTY_MAP; } + + protected Map<String, String> getBlacklistedConnectorConfiguration() { + return MapUtils.EMPTY_MAP; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/test/src/test/java/org/apache/sqoop/integration/classpath/ClasspathTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/classpath/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/classpath/ClasspathTest.java deleted file mode 100644 index 393b800..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/classpath/ClasspathTest.java +++ /dev/null @@ -1,333 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sqoop.integration.classpath; - -import org.apache.commons.collections.ListUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.client.SqoopClient; -import org.apache.sqoop.core.ConfigurationConstants; -import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.testcases.ConnectorTestCase; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import javax.tools.JavaCompiler; -import javax.tools.JavaFileObject; -import javax.tools.StandardJavaFileManager; -import javax.tools.StandardLocation; -import javax.tools.ToolProvider; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.jar.Attributes; -import java.util.jar.JarEntry; -import java.util.jar.JarOutputStream; -import java.util.jar.Manifest; - -public class ClasspathTest extends ConnectorTestCase { - - private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar"; - private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar"; - - private ClassLoader classLoader; - - public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { - - private String extraClasspath; - private String jobExtraClasspath; - - public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception { - super(temporaryPath, configuration); - this.extraClasspath = extraClasspath; - this.jobExtraClasspath = jobExtraClasspath; - } - - @Override - protected Map<String, String> getClasspathConfiguration() { - Map<String, String> properties = new HashMap<>(); - - if (extraClasspath != null) { - properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); - } - if (jobExtraClasspath != null) { - properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath); - } - - - return properties; - } - } - - class JarContents { - private List<File> sourceFiles; - private List<File> properitesFiles; - - public JarContents(List<File> sourceFiles, List<File> properitesFiles){ - this.sourceFiles = sourceFiles; - this.properitesFiles = properitesFiles; - } - - public List<File> getSourceFiles() { - return sourceFiles; - } - - public List<File> getProperitesFiles() { - return properitesFiles; - } - } - - public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception { - // And use them for new Derby repo instance - setCluster(new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getSqoopMiniClusterTemporaryPath(), getTestName()), hadoopCluster.getConfiguration(), extraClasspath, jobExtraClasspath)); - - // Start server - getCluster().start(); - - // Initialize Sqoop Client API - setClient(new SqoopClient(getServerUrl())); - } - - @BeforeMethod - public void captureClasspath() { - classLoader = Thread.currentThread().getContextClassLoader(); - } - - @AfterMethod - public void restoreClasspath(){ - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Test - public void testClasspathSqoopProperties() throws Exception { - Map<String, String> jarMap = compileTestConnectorAndDependency(); - startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get - (TEST_DEPENDENCY_JAR_NAME)); - createAndLoadTableCities(); - - MJob job = prepareJob(); - - prepareDriverConfig(job); - - saveJob(job); - - executeJob(job); - - stopSqoop(); - deleteJars(jarMap); - } - - @Test - public void testClasspathDriverInput() throws Exception{ - Map<String, String> jarMap = compileTestConnectorAndDependency(); - startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null); - createAndLoadTableCities(); - - MJob job = prepareJob(); - - MDriverConfig driverConfig = prepareDriverConfig(job); - - List<String> extraJars = new ArrayList<>(); - extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME)); - driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars); - - saveJob(job); - - executeJob(job); - - stopSqoop(); - deleteJars(jarMap); - } - - private MJob prepareJob() { - MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkConfig(rdbmsConnection); - saveLink(rdbmsConnection); - - MLink testConnection = getClient().createLink("test-connector"); - saveLink(testConnection); - - MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), testConnection.getPersistenceId()); - - fillRdbmsFromConfig(job, "id"); - - return job; - } - - private MDriverConfig prepareDriverConfig(MJob job) { - MDriverConfig driverConfig = job.getDriverConfig(); - driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); - - return driverConfig; - } - - private Map<String, String> compileTestConnectorAndDependency() throws Exception { - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new IllegalStateException( - "Cannot find the system Java compiler. " - + "Check that your class path includes tools.jar"); - } - - Path outputDir = Files.createTempDirectory(null); - - Map<String, JarContents> sourceFileToJarMap = new HashMap<>(); - - ClassLoader classLoader = getClass().getClassLoader(); - List<File> sourceFiles = new ArrayList<>(); - File file = new File(classLoader.getResource("TestConnector/TestConnector.java").getFile()); - sourceFiles.add(file); - file = new File(classLoader.getResource("TestConnector/TestLinkConfiguration.java").getFile()); - sourceFiles.add(file); - file = new File(classLoader.getResource("TestConnector/TestLoader.java").getFile()); - sourceFiles.add(file); - file = new File(classLoader.getResource("TestConnector/TestToDestroyer.java").getFile()); - sourceFiles.add(file); - file = new File(classLoader.getResource("TestConnector/TestToInitializer.java").getFile()); - sourceFiles.add(file); - file = new File(classLoader.getResource("TestConnector/TestToJobConfiguration.java").getFile()); - sourceFiles.add(file); - - List<File> propertiesFiles = new ArrayList<>(); - file = new File(classLoader.getResource("TestConnector/sqoopconnector.properties").getFile()); - propertiesFiles.add(file); - sourceFileToJarMap.put("test-connector.jar", new JarContents(sourceFiles, propertiesFiles)); - - sourceFiles = new ArrayList<>(); - file = new File(classLoader.getResource("TestConnector/TestDependency.java").getFile()); - sourceFiles.add(file); - sourceFileToJarMap.put("test-dependency.jar", new JarContents(sourceFiles, ListUtils.EMPTY_LIST)); - - return buildJar(outputDir.toString(), sourceFileToJarMap); - } - - private Map<String, String> buildJar(String outputDir, Map<String, JarContents> sourceFileToJarMap) throws Exception { - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - StandardJavaFileManager fileManager = compiler.getStandardFileManager - (null, null, null); - - List<File> sourceFiles = new ArrayList<>(); - for(JarContents jarContents : sourceFileToJarMap.values()) { - sourceFiles.addAll(jarContents.sourceFiles); - } - - fileManager.setLocation(StandardLocation.CLASS_OUTPUT, - Arrays.asList(new File(outputDir.toString()))); - - Iterable<? extends JavaFileObject> compilationUnits1 = - fileManager.getJavaFileObjectsFromFiles(sourceFiles); - - boolean compiled = compiler.getTask(null, fileManager, null, null, null, compilationUnits1).call(); - if (!compiled) { - throw new RuntimeException("failed to compile"); - } - - for(Map.Entry<String, JarContents> jarNameAndContents : sourceFileToJarMap.entrySet()) { - Manifest manifest = new Manifest(); - manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); - manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "."); - - - JarOutputStream target = new JarOutputStream(new FileOutputStream(outputDir.toString() + File.separator + jarNameAndContents.getKey()), manifest); - List<String> classesForJar = new ArrayList<>(); - for(File sourceFile : jarNameAndContents.getValue().getSourceFiles()) { - //split the file on dot to get the filename from FILENAME.java - String fileName = sourceFile.getName().split("\\.")[0]; - classesForJar.add(fileName); - } - - File dir = new File(outputDir); - File[] directoryListing = dir.listFiles(); - for (File compiledClass : directoryListing) { - String classFileName = compiledClass.getName().split("\\$")[0].split("\\.")[0]; - if (classesForJar.contains(classFileName)){ - addFileToJar(compiledClass, target); - } - } - - for (File propertiesFile : jarNameAndContents.getValue().getProperitesFiles()) { - addFileToJar(propertiesFile, target); - } - - target.close(); - - - } - //delete non jar files - File dir = new File(outputDir); - File[] directoryListing = dir.listFiles(); - for (File file : directoryListing) { - String extension = file.getName().split("\\.")[1]; - if (!extension.equals("jar")) { - file.delete(); - } - } - - Map<String, String> jarMap = new HashMap<>(); - jarMap.put(TEST_CONNECTOR_JAR_NAME, outputDir.toString() + File.separator - + TEST_CONNECTOR_JAR_NAME); - jarMap.put(TEST_DEPENDENCY_JAR_NAME, outputDir.toString() + File.separator + TEST_DEPENDENCY_JAR_NAME); - return jarMap; - } - - private void addFileToJar(File source, JarOutputStream target) throws Exception { - JarEntry entry = new JarEntry(source.getName()); - entry.setTime(source.lastModified()); - target.putNextEntry(entry); - BufferedInputStream in = new BufferedInputStream(new FileInputStream(source)); - - long bufferSize = source.length(); - if (bufferSize < Integer.MIN_VALUE || bufferSize > Integer.MAX_VALUE) { - throw new RuntimeException("file to large to be added to jar"); - } - - byte[] buffer = new byte[(int) bufferSize]; - while (true) { - int count = in.read(buffer); - if (count == -1) - break; - target.write(buffer, 0, count); - } - target.closeEntry(); - if (in != null) in.close(); - } - - private void deleteJars(Map<String, String> jarMap) throws Exception { - for (String jarPath : jarMap.values()) { - (new File(jarPath)).delete(); - } - } - - @Override - public void startSqoop() throws Exception { - // Do nothing so that Sqoop isn't started before Suite. - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java new file mode 100644 index 0000000..2d6d37f --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.connectorloading; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.testcases.ConnectorTestCase; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +public class BlacklistedConnectorTest extends ConnectorTestCase { + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + super(temporaryPath, configuration); + } + + @Override + protected Map<String, String> getBlacklistedConnectorConfiguration() { + Map<String, String> properties = new HashMap<>(); + + properties.put(ConfigurationConstants.BLACKLISTED_CONNECTORS, "fake-connector:generic-jdbc-connector"); + return properties; + } + } + + public void startSqoopMiniCluster() throws Exception { + // And use them for new Derby repo instance + setCluster(new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super + .getSqoopMiniClusterTemporaryPath(), getTestName()), hadoopCluster.getConfiguration())); + + // Start server + getCluster().start(); + + // Initialize Sqoop Client API + setClient(new SqoopClient(getServerUrl())); + } + + @Test(expectedExceptions = {SqoopException.class}) + public void testCreateLinkWithNonexistantConnector() throws Exception { + startSqoopMiniCluster(); + + getClient().createLink("generic-jdbc-connector"); + } + + @Override + public void startSqoop() throws Exception { + // Do nothing so that Sqoop isn't started before Suite. + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java new file mode 100644 index 0000000..9d8460d --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java @@ -0,0 +1,333 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.integration.connectorloading; + +import org.apache.commons.collections.ListUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.testcases.ConnectorTestCase; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.StandardLocation; +import javax.tools.ToolProvider; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +public class ClasspathTest extends ConnectorTestCase { + + private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar"; + private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar"; + + private ClassLoader classLoader; + + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + + private String extraClasspath; + private String jobExtraClasspath; + + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception { + super(temporaryPath, configuration); + this.extraClasspath = extraClasspath; + this.jobExtraClasspath = jobExtraClasspath; + } + + @Override + protected Map<String, String> getClasspathConfiguration() { + Map<String, String> properties = new HashMap<>(); + + if (extraClasspath != null) { + properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); + } + if (jobExtraClasspath != null) { + properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath); + } + + + return properties; + } + } + + class JarContents { + private List<File> sourceFiles; + private List<File> properitesFiles; + + public JarContents(List<File> sourceFiles, List<File> properitesFiles){ + this.sourceFiles = sourceFiles; + this.properitesFiles = properitesFiles; + } + + public List<File> getSourceFiles() { + return sourceFiles; + } + + public List<File> getProperitesFiles() { + return properitesFiles; + } + } + + public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception { + // And use them for new Derby repo instance + setCluster(new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getSqoopMiniClusterTemporaryPath(), getTestName()), hadoopCluster.getConfiguration(), extraClasspath, jobExtraClasspath)); + + // Start server + getCluster().start(); + + // Initialize Sqoop Client API + setClient(new SqoopClient(getServerUrl())); + } + + @BeforeMethod + public void captureClasspath() { + classLoader = Thread.currentThread().getContextClassLoader(); + } + + @AfterMethod + public void restoreClasspath(){ + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Test + public void testClasspathSqoopProperties() throws Exception { + Map<String, String> jarMap = compileTestConnectorAndDependency(); + startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get + (TEST_DEPENDENCY_JAR_NAME)); + createAndLoadTableCities(); + + MJob job = prepareJob(); + + prepareDriverConfig(job); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + deleteJars(jarMap); + } + + @Test + public void testClasspathDriverInput() throws Exception{ + Map<String, String> jarMap = compileTestConnectorAndDependency(); + startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null); + createAndLoadTableCities(); + + MJob job = prepareJob(); + + MDriverConfig driverConfig = prepareDriverConfig(job); + + List<String> extraJars = new ArrayList<>(); + extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME)); + driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + deleteJars(jarMap); + } + + private MJob prepareJob() { + MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsConnection); + saveLink(rdbmsConnection); + + MLink testConnection = getClient().createLink("test-connector"); + saveLink(testConnection); + + MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), testConnection.getPersistenceId()); + + fillRdbmsFromConfig(job, "id"); + + return job; + } + + private MDriverConfig prepareDriverConfig(MJob job) { + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + + return driverConfig; + } + + private Map<String, String> compileTestConnectorAndDependency() throws Exception { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new IllegalStateException( + "Cannot find the system Java compiler. " + + "Check that your class path includes tools.jar"); + } + + Path outputDir = Files.createTempDirectory(null); + + Map<String, JarContents> sourceFileToJarMap = new HashMap<>(); + + ClassLoader classLoader = getClass().getClassLoader(); + List<File> sourceFiles = new ArrayList<>(); + File file = new File(classLoader.getResource("TestConnector/TestConnector.java").getFile()); + sourceFiles.add(file); + file = new File(classLoader.getResource("TestConnector/TestLinkConfiguration.java").getFile()); + sourceFiles.add(file); + file = new File(classLoader.getResource("TestConnector/TestLoader.java").getFile()); + sourceFiles.add(file); + file = new File(classLoader.getResource("TestConnector/TestToDestroyer.java").getFile()); + sourceFiles.add(file); + file = new File(classLoader.getResource("TestConnector/TestToInitializer.java").getFile()); + sourceFiles.add(file); + file = new File(classLoader.getResource("TestConnector/TestToJobConfiguration.java").getFile()); + sourceFiles.add(file); + + List<File> propertiesFiles = new ArrayList<>(); + file = new File(classLoader.getResource("TestConnector/sqoopconnector.properties").getFile()); + propertiesFiles.add(file); + sourceFileToJarMap.put("test-connector.jar", new JarContents(sourceFiles, propertiesFiles)); + + sourceFiles = new ArrayList<>(); + file = new File(classLoader.getResource("TestConnector/TestDependency.java").getFile()); + sourceFiles.add(file); + sourceFileToJarMap.put("test-dependency.jar", new JarContents(sourceFiles, ListUtils.EMPTY_LIST)); + + return buildJar(outputDir.toString(), sourceFileToJarMap); + } + + private Map<String, String> buildJar(String outputDir, Map<String, JarContents> sourceFileToJarMap) throws Exception { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + StandardJavaFileManager fileManager = compiler.getStandardFileManager + (null, null, null); + + List<File> sourceFiles = new ArrayList<>(); + for(JarContents jarContents : sourceFileToJarMap.values()) { + sourceFiles.addAll(jarContents.sourceFiles); + } + + fileManager.setLocation(StandardLocation.CLASS_OUTPUT, + Arrays.asList(new File(outputDir.toString()))); + + Iterable<? extends JavaFileObject> compilationUnits1 = + fileManager.getJavaFileObjectsFromFiles(sourceFiles); + + boolean compiled = compiler.getTask(null, fileManager, null, null, null, compilationUnits1).call(); + if (!compiled) { + throw new RuntimeException("failed to compile"); + } + + for(Map.Entry<String, JarContents> jarNameAndContents : sourceFileToJarMap.entrySet()) { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "."); + + + JarOutputStream target = new JarOutputStream(new FileOutputStream(outputDir.toString() + File.separator + jarNameAndContents.getKey()), manifest); + List<String> classesForJar = new ArrayList<>(); + for(File sourceFile : jarNameAndContents.getValue().getSourceFiles()) { + //split the file on dot to get the filename from FILENAME.java + String fileName = sourceFile.getName().split("\\.")[0]; + classesForJar.add(fileName); + } + + File dir = new File(outputDir); + File[] directoryListing = dir.listFiles(); + for (File compiledClass : directoryListing) { + String classFileName = compiledClass.getName().split("\\$")[0].split("\\.")[0]; + if (classesForJar.contains(classFileName)){ + addFileToJar(compiledClass, target); + } + } + + for (File propertiesFile : jarNameAndContents.getValue().getProperitesFiles()) { + addFileToJar(propertiesFile, target); + } + + target.close(); + + + } + //delete non jar files + File dir = new File(outputDir); + File[] directoryListing = dir.listFiles(); + for (File file : directoryListing) { + String extension = file.getName().split("\\.")[1]; + if (!extension.equals("jar")) { + file.delete(); + } + } + + Map<String, String> jarMap = new HashMap<>(); + jarMap.put(TEST_CONNECTOR_JAR_NAME, outputDir.toString() + File.separator + + TEST_CONNECTOR_JAR_NAME); + jarMap.put(TEST_DEPENDENCY_JAR_NAME, outputDir.toString() + File.separator + TEST_DEPENDENCY_JAR_NAME); + return jarMap; + } + + private void addFileToJar(File source, JarOutputStream target) throws Exception { + JarEntry entry = new JarEntry(source.getName()); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + BufferedInputStream in = new BufferedInputStream(new FileInputStream(source)); + + long bufferSize = source.length(); + if (bufferSize < Integer.MIN_VALUE || bufferSize > Integer.MAX_VALUE) { + throw new RuntimeException("file to large to be added to jar"); + } + + byte[] buffer = new byte[(int) bufferSize]; + while (true) { + int count = in.read(buffer); + if (count == -1) + break; + target.write(buffer, 0, count); + } + target.closeEntry(); + if (in != null) in.close(); + } + + private void deleteJars(Map<String, String> jarMap) throws Exception { + for (String jarPath : jarMap.values()) { + (new File(jarPath)).delete(); + } + } + + @Override + public void startSqoop() throws Exception { + // Do nothing so that Sqoop isn't started before Suite. + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f33554a1/test/src/test/resources/classpath-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/classpath-tests-suite.xml b/test/src/test/resources/classpath-tests-suite.xml index e5a9e94..02c1df3 100644 --- a/test/src/test/resources/classpath-tests-suite.xml +++ b/test/src/test/resources/classpath-tests-suite.xml @@ -18,15 +18,15 @@ limitations under the License. <!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > -<suite name="ClasspathTests" verbose="2" parallel="false"> +<suite name="ConnectorLoadingTests" verbose="2" parallel="false"> <listeners> <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> </listeners> - <test name="ClasspathTests"> + <test name="ConnectorLoadingTests"> <packages> - <package name="org.apache.sqoop.integration.classpath"/> + <package name="org.apache.sqoop.integration.connectorloading"/> </packages> </test>
