Repository: sqoop Updated Branches: refs/heads/sqoop2 9aec8f965 -> 96f3d9c19
http://git-wip-us.apache.org/repos/asf/sqoop/blob/96f3d9c1/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java new file mode 100644 index 0000000..1829257 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.integration.serverproperties; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.utils.ConnectorUtils; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "no-real-cluster") +@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class}) +public class ConnectorClasspathIsolationTest extends SqoopTestCase { + + private static final String TEST_FROM_CONNECTOR_JAR_NAME = "test-from-connector.jar"; + private static final String TEST_TO_CONNECTOR_JAR_NAME = "test-to-connector.jar"; + private static final String TEST_FROM_DEPENDENCY_JAR_NAME = "test-from-dependency.jar"; + private static final String TEST_TO_DEPENDENCY_JAR_NAME = "test-to-dependency.jar"; + + private static final String[] FROM_CONNECTOR_SOURCE_FILES = { + "TestConnectorClasspathIsolation/from/TestFromConnector.java", + "TestConnectorClasspathIsolation/from/TestExtractor.java", + "TestConnectorClasspathIsolation/from/TestFromDestroyer.java", + "TestConnectorClasspathIsolation/from/TestFromInitializer.java", + "TestConnectorClasspathIsolation/from/TestFromJobConfiguration.java", + "TestConnectorClasspathIsolation/from/TestPartition.java", + "TestConnectorClasspathIsolation/from/TestPartitioner.java", + "TestConnectorClasspathIsolation/from/TestFromLinkConfiguration.java" + }; + + private static final String[] FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES = { + "TestConnectorClasspathIsolation/from/TestClasspathIsolation.java" + }; + + private static final String[] FROM_CONNECTOR_PROPERTY_FILES = { + "TestConnectorClasspathIsolation/from/sqoopconnector.properties" + }; + + private static final String[] TO_CONNECTOR_SOURCE_FILES = { + "TestConnectorClasspathIsolation/to/TestToConnector.java", + "TestConnectorClasspathIsolation/to/TestLoader.java", + "TestConnectorClasspathIsolation/to/TestToDestroyer.java", + "TestConnectorClasspathIsolation/to/TestToInitializer.java", + "TestConnectorClasspathIsolation/to/TestToJobConfiguration.java", + "TestConnectorClasspathIsolation/to/TestToLinkConfiguration.java" + }; + + private static final String[] TO_CONNECTOR_DEPENDENCY_SOURCE_FILES = { + "TestConnectorClasspathIsolation/to/TestClasspathIsolation.java" + }; + + private static final String[] TO_CONNECTOR_PROPERTY_FILES = { + "TestConnectorClasspathIsolation/to/sqoopconnector.properties" + }; + + private ClassLoader classLoader; + private SqoopMiniCluster sqoopMiniCluster; + + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + + private String extraClasspath; + + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath) throws Exception { + super(temporaryPath, configuration); + this.extraClasspath = extraClasspath; + } + + @Override + protected Map<String, String> getClasspathConfiguration() { + Map<String, String> properties = new HashMap<>(); + + if (extraClasspath != null) { + properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); + } + + return properties; + } + } + + public void startSqoopMiniCluster(String extraClasspath) throws Exception { + // And use them for new Derby repo instance + sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath); + KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); + if (kdcProvider != null) { + sqoopMiniCluster.setKdc(kdcProvider.getInstance()); + } + + // Start server + sqoopMiniCluster.start(); + + // Initialize Sqoop Client API + initSqoopClient(sqoopMiniCluster.getServerUrl()); + } + + @BeforeMethod + public void captureClasspath() { + classLoader = Thread.currentThread().getContextClassLoader(); + } + + @AfterMethod + public void restoreClasspath(){ + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Test + public void testConnectorClasspathIsolation() throws Exception { + Map<String, String> fromConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( + FROM_CONNECTOR_SOURCE_FILES, + FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES, + FROM_CONNECTOR_PROPERTY_FILES, + TEST_FROM_CONNECTOR_JAR_NAME, + TEST_FROM_DEPENDENCY_JAR_NAME, + true); + Map<String, String> toConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( + TO_CONNECTOR_SOURCE_FILES, + TO_CONNECTOR_DEPENDENCY_SOURCE_FILES, + TO_CONNECTOR_PROPERTY_FILES, + TEST_TO_CONNECTOR_JAR_NAME, + TEST_TO_DEPENDENCY_JAR_NAME, + true); + startSqoopMiniCluster( + StringUtils.join(Arrays.asList(fromConnectorJarMap.get(TEST_FROM_CONNECTOR_JAR_NAME), toConnectorJarMap.get(TEST_TO_CONNECTOR_JAR_NAME)), ":")); + + MJob job = prepareJob(); + + prepareDriverConfig(job); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + ConnectorUtils.deleteJars(fromConnectorJarMap); + } + + private MJob prepareJob() { + MLink rdbmsConnection = getClient().createLink("test-from-connector"); + saveLink(rdbmsConnection); + + MLink testConnection = getClient().createLink("test-to-connector"); + saveLink(testConnection); + + MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName()); + + return job; + } + + private MDriverConfig prepareDriverConfig(MJob job) { + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + + return driverConfig; + } + + private void stopSqoop() throws Exception { + sqoopMiniCluster.stop(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/96f3d9c1/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java new file mode 100644 index 0000000..17503f3 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.serverproperties; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.sqoop.security.SecurityConstants; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.apache.sqoop.test.utils.SecurityUtils; +import org.apache.sqoop.test.utils.SqoopUtils; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MediaType; +import java.io.File; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class}) +@Test(groups = {"no-real-cluster"}) +public class SslTest extends SqoopTestCase { + + private SqoopMiniCluster sqoopMiniCluster; + private SSLContext defaultSslContext; + private HostnameVerifier defaultHostNameVerifier; + + public static class SslSqoopMiniCluster extends JettySqoopMiniCluster { + + private String keyStoreFilePath; + private String keyStorePassword; + private String keyManagerPassword; + + public SslSqoopMiniCluster(String temporaryPath, Configuration configuration, String keyStoreFilePath, String keyStorePassword, String keyManagerPassword) throws Exception { + super(temporaryPath, configuration); + this.keyStoreFilePath = keyStoreFilePath; + this.keyStorePassword = keyStorePassword; + this.keyManagerPassword = keyManagerPassword; + } + + @Override + protected Map<String, String> getSecurityConfiguration() { + Map<String, String> properties = super.getSecurityConfiguration(); + + properties.put(SecurityConstants.TLS_ENABLED, String.valueOf(true)); + properties.put(SecurityConstants.TLS_PROTOCOL, "TLSv1.2"); + properties.put(SecurityConstants.KEYSTORE_LOCATION, keyStoreFilePath); + properties.put(SecurityConstants.KEYSTORE_PASSWORD, keyStorePassword); + properties.put(SecurityConstants.KEYMANAGER_PASSWORD, keyManagerPassword); + + return properties; + } + } + + @BeforeMethod + public void backupSslContext() throws Exception { + defaultSslContext = SSLContext.getDefault(); + defaultHostNameVerifier = HttpsURLConnection.getDefaultHostnameVerifier(); + } + + @AfterMethod + public void restoreSslContext() { + SSLContext.setDefault(defaultSslContext); + HttpsURLConnection.setDefaultHostnameVerifier(defaultHostNameVerifier); + } + + @AfterMethod + public void stopCluster() throws Exception { + sqoopMiniCluster.stop(); + } + + @Test + public void testSslInUse() throws Exception { + String sslKeystoresDir = getTemporaryPath() + "ssl-keystore/"; + String sslConfDir = SqoopUtils.getClasspathDir(SslTest.class); + FileUtils.deleteDirectory(new File(sslKeystoresDir)); + FileUtils.forceMkdir(new File(sslKeystoresDir)); + X509Certificate serverCertificate = SecurityUtils.setupSSLConfig( + sslKeystoresDir, sslConfDir, new Configuration(), false, true); + + sqoopMiniCluster = + new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()), getHadoopConf(), sslKeystoresDir + SecurityUtils.SERVER_KEYSTORE, SecurityUtils.SERVER_KEY_STORE_PASSWORD, SecurityUtils.SERVER_KEY_PASSWORD); + + KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); + if (kdcProvider != null) { + sqoopMiniCluster.setKdc(kdcProvider.getInstance()); + } + + sqoopMiniCluster.start(); + + // Bypass hostname verification + HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + try { + if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) { + return true; + } + } catch (MalformedURLException e) { + return false; + } + return false; + } + }); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(sslKeystoresDir + SecurityUtils.TRUSTSTORE); + + sslContextFactory.start(); + + SSLContext.setDefault(sslContextFactory.getSslContext()); + + initSqoopClient(sqoopMiniCluster.getServerUrl()); + + // Make a request and check the cert + URL url = new URL(sqoopMiniCluster.getServerUrl() + "version?" + + PseudoAuthenticator.USER_NAME + "=" + System.getProperty("user.name")); + HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, getAuthToken()); + conn.setRequestMethod(HttpMethod.GET); + conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON); + + assertEquals(conn.getResponseCode(), 200); + + HttpsURLConnection secured = (HttpsURLConnection) conn; + Certificate actualCertificate = secured.getServerCertificates()[0]; + assertEquals(actualCertificate, serverCertificate); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/96f3d9c1/test/src/test/resources/connector-loading-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/connector-loading-tests-suite.xml b/test/src/test/resources/connector-loading-tests-suite.xml deleted file mode 100644 index c03fb4f..0000000 --- a/test/src/test/resources/connector-loading-tests-suite.xml +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> - -<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > - -<suite name="ConnectorLoadingTests" verbose="2" parallel="false"> - - <listeners> - <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> - <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> - </listeners> - - <test name="ConnectorLoadingTests"> - <packages> - <package name="org.apache.sqoop.integration.connectorloading"/> - </packages> - </test> - -</suite> http://git-wip-us.apache.org/repos/asf/sqoop/blob/96f3d9c1/test/src/test/resources/server-properties-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/server-properties-tests-suite.xml b/test/src/test/resources/server-properties-tests-suite.xml new file mode 100644 index 0000000..2743996 --- /dev/null +++ b/test/src/test/resources/server-properties-tests-suite.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > + +<suite name="ServerPropertiesTests" verbose="2" parallel="false"> + + <listeners> + <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" /> + <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" /> + </listeners> + + <test name="ServerPropertiesTests"> + <packages> + <package name="org.apache.sqoop.integration.serverproperties"/> + </packages> + </test> + +</suite>
