Repository: sqoop Updated Branches: refs/heads/sqoop2 96f3d9c19 -> f9d7c3a8e
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java deleted file mode 100644 index 1829257..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sqoop.integration.serverproperties; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.core.ConfigurationConstants; -import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.test.infrastructure.Infrastructure; -import org.apache.sqoop.test.infrastructure.SqoopTestCase; -import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.utils.ConnectorUtils; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test(groups = "no-real-cluster") -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class}) -public class ConnectorClasspathIsolationTest extends SqoopTestCase { - - private static final String TEST_FROM_CONNECTOR_JAR_NAME = "test-from-connector.jar"; - private static final String TEST_TO_CONNECTOR_JAR_NAME = "test-to-connector.jar"; - private static final String TEST_FROM_DEPENDENCY_JAR_NAME = "test-from-dependency.jar"; - private static final String TEST_TO_DEPENDENCY_JAR_NAME = "test-to-dependency.jar"; - - private static final String[] FROM_CONNECTOR_SOURCE_FILES = { - "TestConnectorClasspathIsolation/from/TestFromConnector.java", - "TestConnectorClasspathIsolation/from/TestExtractor.java", - "TestConnectorClasspathIsolation/from/TestFromDestroyer.java", - "TestConnectorClasspathIsolation/from/TestFromInitializer.java", - "TestConnectorClasspathIsolation/from/TestFromJobConfiguration.java", - "TestConnectorClasspathIsolation/from/TestPartition.java", - "TestConnectorClasspathIsolation/from/TestPartitioner.java", - "TestConnectorClasspathIsolation/from/TestFromLinkConfiguration.java" - }; - - private static final String[] FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES = { - "TestConnectorClasspathIsolation/from/TestClasspathIsolation.java" - }; - - private static final String[] FROM_CONNECTOR_PROPERTY_FILES = { - "TestConnectorClasspathIsolation/from/sqoopconnector.properties" - }; - - private static final String[] TO_CONNECTOR_SOURCE_FILES = { - "TestConnectorClasspathIsolation/to/TestToConnector.java", - "TestConnectorClasspathIsolation/to/TestLoader.java", - "TestConnectorClasspathIsolation/to/TestToDestroyer.java", - "TestConnectorClasspathIsolation/to/TestToInitializer.java", - "TestConnectorClasspathIsolation/to/TestToJobConfiguration.java", - "TestConnectorClasspathIsolation/to/TestToLinkConfiguration.java" - }; - - private static final String[] TO_CONNECTOR_DEPENDENCY_SOURCE_FILES = { - "TestConnectorClasspathIsolation/to/TestClasspathIsolation.java" - }; - - private static final String[] TO_CONNECTOR_PROPERTY_FILES = { - "TestConnectorClasspathIsolation/to/sqoopconnector.properties" - }; - - private ClassLoader classLoader; - private SqoopMiniCluster sqoopMiniCluster; - - public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { - - private String extraClasspath; - - public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath) throws Exception { - super(temporaryPath, configuration); - this.extraClasspath = extraClasspath; - } - - @Override - protected Map<String, String> getClasspathConfiguration() { - Map<String, String> properties = new HashMap<>(); - - if (extraClasspath != null) { - properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); - } - - return properties; - } - } - - public void startSqoopMiniCluster(String extraClasspath) throws Exception { - // And use them for new Derby repo instance - sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath); - KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); - if (kdcProvider != null) { - sqoopMiniCluster.setKdc(kdcProvider.getInstance()); - } - - // Start server - sqoopMiniCluster.start(); - - // Initialize Sqoop Client API - initSqoopClient(sqoopMiniCluster.getServerUrl()); - } - - @BeforeMethod - public void captureClasspath() { - classLoader = Thread.currentThread().getContextClassLoader(); - } - - @AfterMethod - public void restoreClasspath(){ - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Test - public void testConnectorClasspathIsolation() throws Exception { - Map<String, String> fromConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( - FROM_CONNECTOR_SOURCE_FILES, - FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES, - FROM_CONNECTOR_PROPERTY_FILES, - TEST_FROM_CONNECTOR_JAR_NAME, - TEST_FROM_DEPENDENCY_JAR_NAME, - true); - Map<String, String> toConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( - TO_CONNECTOR_SOURCE_FILES, - TO_CONNECTOR_DEPENDENCY_SOURCE_FILES, - TO_CONNECTOR_PROPERTY_FILES, - TEST_TO_CONNECTOR_JAR_NAME, - TEST_TO_DEPENDENCY_JAR_NAME, - true); - startSqoopMiniCluster( - StringUtils.join(Arrays.asList(fromConnectorJarMap.get(TEST_FROM_CONNECTOR_JAR_NAME), toConnectorJarMap.get(TEST_TO_CONNECTOR_JAR_NAME)), ":")); - - MJob job = prepareJob(); - - prepareDriverConfig(job); - - saveJob(job); - - executeJob(job); - - stopSqoop(); - ConnectorUtils.deleteJars(fromConnectorJarMap); - } - - private MJob prepareJob() { - MLink rdbmsConnection = getClient().createLink("test-from-connector"); - saveLink(rdbmsConnection); - - MLink testConnection = getClient().createLink("test-to-connector"); - saveLink(testConnection); - - MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName()); - - return job; - } - - private MDriverConfig prepareDriverConfig(MJob job) { - MDriverConfig driverConfig = job.getDriverConfig(); - driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); - - return driverConfig; - } - - private void stopSqoop() throws Exception { - sqoopMiniCluster.stop(); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java deleted file mode 100644 index 17503f3..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration.serverproperties; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; -import org.apache.sqoop.security.SecurityConstants; -import org.apache.sqoop.test.infrastructure.Infrastructure; -import org.apache.sqoop.test.infrastructure.SqoopTestCase; -import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.apache.sqoop.test.utils.SecurityUtils; -import org.apache.sqoop.test.utils.SqoopUtils; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.ws.rs.HttpMethod; -import javax.ws.rs.core.MediaType; -import java.io.File; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import java.util.Map; - -import static org.testng.Assert.assertEquals; - -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class}) -@Test(groups = {"no-real-cluster"}) -public class SslTest extends SqoopTestCase { - - private SqoopMiniCluster sqoopMiniCluster; - private SSLContext defaultSslContext; - private HostnameVerifier defaultHostNameVerifier; - - public static class SslSqoopMiniCluster extends JettySqoopMiniCluster { - - private String keyStoreFilePath; - private String keyStorePassword; - private String keyManagerPassword; - - public SslSqoopMiniCluster(String temporaryPath, Configuration configuration, String keyStoreFilePath, String keyStorePassword, String keyManagerPassword) throws Exception { - super(temporaryPath, configuration); - this.keyStoreFilePath = keyStoreFilePath; - this.keyStorePassword = keyStorePassword; - this.keyManagerPassword = keyManagerPassword; - } - - @Override - protected Map<String, String> getSecurityConfiguration() { - Map<String, String> properties = super.getSecurityConfiguration(); - - properties.put(SecurityConstants.TLS_ENABLED, String.valueOf(true)); - properties.put(SecurityConstants.TLS_PROTOCOL, "TLSv1.2"); - properties.put(SecurityConstants.KEYSTORE_LOCATION, keyStoreFilePath); - properties.put(SecurityConstants.KEYSTORE_PASSWORD, keyStorePassword); - properties.put(SecurityConstants.KEYMANAGER_PASSWORD, keyManagerPassword); - - return properties; - } - } - - @BeforeMethod - public void backupSslContext() throws Exception { - defaultSslContext = SSLContext.getDefault(); - defaultHostNameVerifier = HttpsURLConnection.getDefaultHostnameVerifier(); - } - - @AfterMethod - public void restoreSslContext() { - SSLContext.setDefault(defaultSslContext); - HttpsURLConnection.setDefaultHostnameVerifier(defaultHostNameVerifier); - } - - @AfterMethod - public void stopCluster() throws Exception { - sqoopMiniCluster.stop(); - } - - @Test - public void testSslInUse() throws Exception { - String sslKeystoresDir = getTemporaryPath() + "ssl-keystore/"; - String sslConfDir = SqoopUtils.getClasspathDir(SslTest.class); - FileUtils.deleteDirectory(new File(sslKeystoresDir)); - FileUtils.forceMkdir(new File(sslKeystoresDir)); - X509Certificate serverCertificate = SecurityUtils.setupSSLConfig( - sslKeystoresDir, sslConfDir, new Configuration(), false, true); - - sqoopMiniCluster = - new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()), getHadoopConf(), sslKeystoresDir + SecurityUtils.SERVER_KEYSTORE, SecurityUtils.SERVER_KEY_STORE_PASSWORD, SecurityUtils.SERVER_KEY_PASSWORD); - - KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); - if (kdcProvider != null) { - sqoopMiniCluster.setKdc(kdcProvider.getInstance()); - } - - sqoopMiniCluster.start(); - - // Bypass hostname verification - HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - try { - if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) { - return true; - } - } catch (MalformedURLException e) { - return false; - } - return false; - } - }); - - SslContextFactory sslContextFactory = new SslContextFactory(); - sslContextFactory.setKeyStorePath(sslKeystoresDir + SecurityUtils.TRUSTSTORE); - - sslContextFactory.start(); - - SSLContext.setDefault(sslContextFactory.getSslContext()); - - initSqoopClient(sqoopMiniCluster.getServerUrl()); - - // Make a request and check the cert - URL url = new URL(sqoopMiniCluster.getServerUrl() + "version?" + - PseudoAuthenticator.USER_NAME + "=" + System.getProperty("user.name")); - HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, getAuthToken()); - conn.setRequestMethod(HttpMethod.GET); - conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON); - - assertEquals(conn.getResponseCode(), 200); - - HttpsURLConnection secured = (HttpsURLConnection) conn; - Certificate actualCertificate = secured.getServerCertificates()[0]; - assertEquals(actualCertificate, serverCertificate); - } - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/resources/connector-loading-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/connector-loading-tests-suite.xml b/test/src/test/resources/connector-loading-tests-suite.xml new file mode 100644 index 0000000..c03fb4f --- /dev/null +++ b/test/src/test/resources/connector-loading-tests-suite.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > + +<suite name="ConnectorLoadingTests" verbose="2" parallel="false"> + + <listeners> + <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> + </listeners> + + <test name="ConnectorLoadingTests"> + <packages> + <package name="org.apache.sqoop.integration.connectorloading"/> + </packages> + </test> + +</suite> http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/resources/server-properties-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/server-properties-tests-suite.xml b/test/src/test/resources/server-properties-tests-suite.xml deleted file mode 100644 index 2743996..0000000 --- a/test/src/test/resources/server-properties-tests-suite.xml +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > - -<suite name="ServerPropertiesTests" verbose="2" parallel="false"> - - <listeners> - <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> - <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> - </listeners> - - <test name="ServerPropertiesTests"> - <packages> - <package name="org.apache.sqoop.integration.serverproperties"/> - </packages> - </test> - -</suite>
