SQOOP-2852: Sqoop2: Revert and re-commit SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls
* Revert "SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls" * This reverts commit 96f3d9c195c78c1f315a61f7c84aa095a041f72b. Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5fd80fd5 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5fd80fd5 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5fd80fd5 Branch: refs/heads/sqoop2 Commit: 5fd80fd5b66dce6cd9dec66bb7070cd3ff67004c Parents: 96f3d9c Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Feb 26 09:36:13 2016 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Feb 26 09:36:13 2016 -0800 ---------------------------------------------------------------------- .../sqoop/security/SecurityConstants.java | 43 --- dist/src/main/conf/sqoop.properties | 9 - .../sqoop/docs/generator/DocPreprocessor.java | 2 - .../plugins/GenerateConnectorPages.java | 65 ----- .../apache/sqoop/server/SqoopJettyServer.java | 49 +--- .../apache/sqoop/server/common/ServerError.java | 3 - .../apache/sqoop/shell/ShellEnvironment.java | 13 - .../sqoop/shell/ShowConnectorFunction.java | 3 +- .../org/apache/sqoop/shell/ShowJobFunction.java | 6 +- .../apache/sqoop/shell/ShowLinkFunction.java | 8 +- .../sqoop/shell/ShowPrincipalFunction.java | 4 +- .../sqoop/shell/ShowPrivilegeFunction.java | 4 +- .../apache/sqoop/shell/ShowRoleFunction.java | 4 +- .../sqoop/shell/ShowSubmissionFunction.java | 3 +- .../sqoop/shell/utils/TableDisplayer.java | 41 +-- test/pom.xml | 6 +- .../apache/sqoop/test/kdc/MiniKdcRunner.java | 247 ++++++++++++++++- .../apache/sqoop/test/utils/SecurityUtils.java | 269 ------------------- .../org/apache/sqoop/test/utils/SqoopUtils.java | 25 +- .../BlacklistedConnectorTest.java | 82 ++++++ .../connectorloading/ClasspathTest.java | 202 ++++++++++++++ .../ConnectorClasspathIsolationTest.java | 190 +++++++++++++ .../BlacklistedConnectorTest.java | 82 ------ .../serverproperties/ClasspathTest.java | 202 -------------- .../ConnectorClasspathIsolationTest.java | 190 ------------- .../integration/serverproperties/SslTest.java | 164 ----------- .../resources/connector-loading-tests-suite.xml | 34 +++ .../resources/server-properties-tests-suite.xml | 34 --- 28 files changed, 785 insertions(+), 1199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java index 91a1b8b..6f32e04 100644 --- a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java +++ b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java @@ -37,13 +37,6 @@ public final class SecurityConstants { PREFIX_SECURITY_CONFIG + "authentication."; /** - * All tls related configuration is prefixed with this: - * <tt>org.apache.sqoop.security.tls.</tt> - */ - public static final String PREFIX_TLS_CONFIG = - PREFIX_SECURITY_CONFIG + "tls."; - - /** * The config specifies the sqoop authentication type (SIMPLE, KERBEROS). * The default type is SIMPLE * <tt>org.apache.sqoop.security.authentication.type</tt>. @@ -165,42 +158,6 @@ public final class SecurityConstants { PREFIX_AUTHORIZATION_CONFIG + "server_name"; /** - * The config specifies the whether the http server should use TLS. - * <tt>org.apache.sqoop.security.tls.enabled</tt>. - */ - public static final String TLS_ENABLED = - PREFIX_TLS_CONFIG + "enabled"; - - /** - * The config specifies the tls protocol version - * <tt>org.apache.sqoop.security.tls.protocol</tt>. - */ - public static final String TLS_PROTOCOL = - PREFIX_TLS_CONFIG + "protocol"; - - /** - * The config specifies the location of the JKS formatted keystore - * <tt>org.apache.sqoop.security.tls.keystore</tt>. - */ - public static final String KEYSTORE_LOCATION = - PREFIX_TLS_CONFIG + "keystore"; - - /** - * The config specifies the password of the JKS formatted keystore - * <tt>org.apache.sqoop.security.tls.keystorepassword</tt>. - */ - public static final String KEYSTORE_PASSWORD = - PREFIX_TLS_CONFIG + "keystore_password"; - - /** - * The config specifies the password for the private key in the JKS formatted - * keystore - * <tt>org.apache.sqoop.security.tls.keymanagerpassword</tt>. - */ - public static final String KEYMANAGER_PASSWORD = - PREFIX_TLS_CONFIG + "keymanager_password"; - - /** * The config specifies the token kind in delegation token. */ public static final String TOKEN_KIND = "sqoop_token_kind"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/dist/src/main/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties index 767d3f2..620146d 100755 --- a/dist/src/main/conf/sqoop.properties +++ b/dist/src/main/conf/sqoop.properties @@ -180,15 +180,6 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce #org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider #org.apache.sqoop.security.authorization.server_name=SqoopServer1 -# -# SSL/TLS configuration -# -#org.apache.sqoop.security.tls.enabled=false -#org.apache.sqoop.security.tls.protocol="TLSv1.2" -#org.apache.sqoop.security.tls.keystore= -#org.apache.sqoop.security.tls.keystore_password= - - # External connectors load path # "/path/to/external/connectors/": Add all the connector JARs in the specified folder # http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java ---------------------------------------------------------------------- diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java index cb79654..5ea3aa0 100644 --- a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java +++ b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java @@ -20,7 +20,6 @@ package org.apache.sqoop.docs.generator; import org.apache.log4j.Logger; import org.apache.sqoop.docs.generator.plugins.AbstractPlugin; import org.apache.sqoop.docs.generator.plugins.CopySourceToDestination; -import org.apache.sqoop.docs.generator.plugins.GenerateConnectorPages; import org.apache.sqoop.utils.ClassUtils; import java.util.LinkedList; @@ -37,7 +36,6 @@ public class DocPreprocessor { static { plugins = new LinkedList<>(); plugins.add(CopySourceToDestination.class); - plugins.add(GenerateConnectorPages.class); } public static void main(String []args) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java ---------------------------------------------------------------------- diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java b/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java deleted file mode 100644 index 119b76d..0000000 --- a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.docs.generator.plugins; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.sqoop.connector.ConnectorHandler; -import org.apache.sqoop.connector.ConnectorManagerUtils; -import org.apache.sqoop.connector.spi.SqoopConnector; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URL; -import java.util.Collections; -import java.util.List; - -public class GenerateConnectorPages extends AbstractPlugin { - - private static final Logger LOG = Logger.getLogger(GenerateConnectorPages.class); - - @Override - public void run() { - try { - runWithException(); - } catch (IOException e) { - throw new RuntimeException("Can't generate connector documentation", e); - } - } - - public void runWithException() throws IOException { - List<URL> connectorUrls = ConnectorManagerUtils.getConnectorConfigs(Collections.<String>emptySet()); - for (URL url : connectorUrls) { - SqoopConnector connector = new ConnectorHandler(url).getSqoopConnector(); - LOG.info("Loaded " + connector.getConnectorName() + " from " + url.toString()); - - String connectorName = connector.getConnectorName(); - - File outputFile = FileUtils.getFile(getDestination(), "user", "connectors", connectorName + ".rst"); - PrintWriter writer = new PrintWriter(outputFile); - - // Writing headline - writer.println(connectorName); - writer.println(StringUtils.repeat("=", connectorName.length())); - - writer.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java index 60368af..2c4cb7a 100644 --- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java +++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java @@ -19,21 +19,12 @@ package org.apache.sqoop.server; import org.apache.log4j.Logger; -import org.apache.sqoop.common.MapContext; -import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.core.SqoopServer; import org.apache.sqoop.filter.SqoopAuthenticationFilter; -import org.apache.sqoop.security.SecurityConstants; -import org.apache.sqoop.server.common.ServerError; import org.apache.sqoop.server.v1.*; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.server.ServerConnector; @@ -67,45 +58,7 @@ public class SqoopJettyServer { webServer = new Server(threadPool); // Connector configs - ServerConnector connector; - - MapContext configurationContext = SqoopConfiguration.getInstance().getContext(); - - if (configurationContext.getBoolean(SecurityConstants.TLS_ENABLED, false)) { - String keyStorePath = configurationContext.getString(SecurityConstants.KEYSTORE_LOCATION); - if (keyStorePath == null) { - throw new SqoopException(ServerError.SERVER_0007); - } - - SslContextFactory sslContextFactory = new SslContextFactory(); - sslContextFactory.setKeyStorePath(keyStorePath); - - String protocol = configurationContext.getString(SecurityConstants.TLS_PROTOCOL); - if (protocol != null && protocol.length() > 0) { - sslContextFactory.setProtocol(protocol.trim()); - } - - String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD); - if (keyStorePassword != null && keyStorePassword.length() > 0) { - sslContextFactory.setKeyStorePassword(keyStorePassword); - } - - String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD); - if (keyManagerPassword != null && keyManagerPassword.length() > 0) { - sslContextFactory.setKeyManagerPassword(keyManagerPassword); - } - - HttpConfiguration https = new HttpConfiguration(); - https.addCustomizer(new SecureRequestCustomizer()); - - connector = new ServerConnector(webServer, - new SslConnectionFactory(sslContextFactory, "http/1.1"), - new HttpConnectionFactory(https)); - } else { - connector = new ServerConnector(webServer); - } - - + ServerConnector connector = new ServerConnector(webServer); connector.setPort(sqoopJettyContext.getPort()); webServer.addConnector(connector); webServer.setHandler(createServletContextHandler()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/server/src/main/java/org/apache/sqoop/server/common/ServerError.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java index a644e9f..1b021cf 100644 --- a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java +++ b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java @@ -38,9 +38,6 @@ public enum ServerError implements ErrorCode { /** Entity requested doesn't exist*/ SERVER_0006("Entity requested doesn't exist"), - - /** TLS enabled but keystore location not set*/ - SERVER_0007("TLS enabled but keystore location not set"), ; private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java index b7f1129..36d0712 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java @@ -21,14 +21,12 @@ import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.shell.core.ShellError; import org.apache.sqoop.shell.core.Constants; -import org.apache.sqoop.shell.utils.TableDisplayer; import org.codehaus.groovy.tools.shell.IO; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; -import java.util.List; import java.util.Locale; import java.util.ResourceBundle; @@ -59,13 +57,6 @@ public final class ShellEnvironment { private static boolean interactive = false; private static long pollTimeout = DEFAULT_POLL_TIMEOUT; - private static TableDisplayer tableDisplayer = new TableDisplayer(new TableDisplayer.TableDisplayerWriter() { - @Override - public void append(String text) { - print(text); - } - }); - static ResourceBundle resource = ResourceBundle.getBundle(Constants.RESOURCE_NAME, Locale.getDefault()); static SqoopClient client = new SqoopClient(getServerUrl()); static IO io; @@ -235,10 +226,6 @@ public final class ShellEnvironment { io.out.printf(format, args); } - public static void displayTable(List<String> headers, List<String> ...columns) { - tableDisplayer.display(headers, columns); - } - // for tests only public static void cleanup() { serverHost = DEFAULT_SERVER_HOST; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java index c90fe94..d973499 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java @@ -27,6 +27,7 @@ import org.apache.sqoop.client.ClientError; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import static org.apache.sqoop.shell.ShellEnvironment.*; @@ -82,7 +83,7 @@ public class ShowConnectorFunction extends SqoopFunction { supportedDirections.add(connector.getSupportedDirections().toString()); } - displayTable(header, uniqueNames, versions, classes, supportedDirections); + TableDisplayer.display(header, uniqueNames, versions, classes, supportedDirections); } private void showConnectors() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java index 139e756..ebbfe82 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java @@ -19,8 +19,12 @@ package org.apache.sqoop.shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.text.DateFormat; @@ -93,7 +97,7 @@ public class ShowJobFunction extends SqoopFunction { availabilities.add(String.valueOf(job.getEnabled())); } - displayTable(header, ids, names, fromConnectors, toConnectors, availabilities); + TableDisplayer.display(header, ids, names, fromConnectors, toConnectors, availabilities); } private void showJobs(String jArg) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java index 4976c98..25bd1db 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java @@ -19,13 +19,19 @@ package org.apache.sqoop.shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MLink; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.text.DateFormat; +import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import static org.apache.sqoop.shell.ShellEnvironment.*; import static org.apache.sqoop.shell.utils.ConfigDisplayer.*; @@ -80,7 +86,7 @@ public class ShowLinkFunction extends SqoopFunction { availabilities.add(String.valueOf(link.getEnabled())); } - displayTable(header, names, connectorNames, availabilities); + TableDisplayer.display(header, names, connectorNames, availabilities); } private void showLinks() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java index 45abf14..a450aaf 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java @@ -22,13 +22,13 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.model.MPrincipal; import org.apache.sqoop.model.MRole; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -70,6 +70,6 @@ public class ShowPrincipalFunction extends SqoopFunction { types.add(principal.getType()); } - displayTable(header, names, types); + TableDisplayer.display(header, names, types); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java index efd4b19..2cf6972 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java @@ -25,13 +25,13 @@ import org.apache.sqoop.model.MPrivilege; import org.apache.sqoop.model.MResource; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.ShellError; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -108,6 +108,6 @@ public class ShowPrivilegeFunction extends SqoopFunction { withGrant.add(Boolean.toString(privilege.isWith_grant_option())); } - displayTable(header, actions, resourceNames, resourceTypes, withGrant); + TableDisplayer.display(header, actions, resourceNames, resourceTypes, withGrant); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java index 91ceff3..6b61921 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java @@ -24,13 +24,13 @@ import org.apache.sqoop.model.MPrincipal; import org.apache.sqoop.model.MRole; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.ShellError; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import java.util.LinkedList; import java.util.List; import static org.apache.sqoop.shell.ShellEnvironment.client; -import static org.apache.sqoop.shell.ShellEnvironment.displayTable; import static org.apache.sqoop.shell.ShellEnvironment.resourceString; @SuppressWarnings("serial") @@ -83,6 +83,6 @@ public class ShowRoleFunction extends SqoopFunction { names.add(role.getName()); } - displayTable(header, names); + TableDisplayer.display(header, names); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java index c7f371f..8989913 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.utils.SubmissionDisplayer; +import org.apache.sqoop.shell.utils.TableDisplayer; import org.apache.sqoop.validation.Status; import static org.apache.sqoop.shell.ShellEnvironment.*; @@ -90,7 +91,7 @@ public class ShowSubmissionFunction extends SqoopFunction { dates.add(submission.getLastUpdateDate().toString()); } - displayTable(header, jnames, eids, status, dates); + TableDisplayer.display(header, jnames, eids, status, dates); } private void showSubmissions(String jArg) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java index ba3698e..51030d0 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java @@ -22,43 +22,23 @@ import org.apache.commons.lang.StringUtils; import java.util.LinkedList; import java.util.List; +import static org.apache.sqoop.shell.ShellEnvironment.*; + /** * Display table based data */ public class TableDisplayer { /** - * Interface that this displayer will use to write out formatted text. - */ - public interface TableDisplayerWriter { - /** - * Print out addition formatted text to the output - * - * @param text - */ - void append(String text); - } - - public TableDisplayer(TableDisplayerWriter writer) { - this.writer = writer; - } - - /** - * Writer instance that we should use to write something out - */ - private TableDisplayerWriter writer; - - /** * Display given columns in nice table structure to given IO object. * * @param headers List of headers * @param columns Array of columns */ - public void display(List<String> headers, List<String> ...columns) { + public static void display(List<String> headers, List<String> ...columns) { assert headers != null; assert columns != null; assert headers.size() == columns.length; - assert writer != null; // Count of columns int columnCount = headers.size(); @@ -105,7 +85,7 @@ public class TableDisplayer { * * @param widths List of widths of each column */ - private void drawLine(List<Integer> widths) { + private static void drawLine(List<Integer> widths) { int last = widths.size() - 1; print("+-"); for(int i = 0; i < widths.size(); i++) { @@ -122,7 +102,7 @@ public class TableDisplayer { * @param column All column values * @return Maximal */ - private int getMaximalWidth(String header, List<String> column) { + private static int getMaximalWidth(String header, List<String> column) { assert header != null; assert column != null; @@ -143,7 +123,7 @@ public class TableDisplayer { * @param columns Array with all column values * @return */ - private int getMaximalRows(List<String>... columns) { + private static int getMaximalRows(List<String>... columns) { int max = 0; for(List<String> column : columns) { @@ -155,12 +135,7 @@ public class TableDisplayer { return max; } - private void print(String text) { - writer.append(text); + private TableDisplayer() { + // Instantiation is prohibited } - - private void println() { - writer.append("\n"); - } - } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index 4bac683..1e88b34 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -274,17 +274,17 @@ limitations under the License. </configuration> </execution> <execution> - <id>server-properties-test</id> + <id>connector-loading-test</id> <goals> <goal>test</goal> </goals> <phase>integration-test</phase> <configuration> <suiteXmlFiles> - <suiteXmlFile>src/test/resources/server-properties-tests-suite.xml</suiteXmlFile> + <suiteXmlFile>src/test/resources/connector-loading-tests-suite.xml</suiteXmlFile> </suiteXmlFiles> <properties> - <suitename>server-properties-tests</suitename> + <suitename>connector-loading-tests</suitename> </properties> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java index bca4412..83f960b 100644 --- a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java @@ -18,12 +18,31 @@ package org.apache.sqoop.test.kdc; import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.lang.reflect.Constructor; +import java.math.BigInteger; import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; import java.security.Principal; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -34,16 +53,19 @@ import java.util.concurrent.Callable; import javax.security.auth.Subject; import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.LoginContext; +import javax.security.auth.x500.X500Principal; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.test.utils.HdfsUtils; -import org.apache.sqoop.test.utils.SecurityUtils; import org.apache.sqoop.test.utils.SqoopUtils; +import org.bouncycastle.x509.X509V1CertificateGenerator; /** * Represents a Minikdc setup. Minikdc should be only used together with @@ -85,17 +107,17 @@ public class MiniKdcRunner extends KdcRunner { config.set("dfs.datanode.kerberos.principal", hadoopPrincipal); config.set("dfs.datanode.keytab.file", hadoopKeytabFile); String sslKeystoresDir = getTemporaryPath() + "/ssl-keystore"; - String sslConfDir = SqoopUtils.getClasspathDir(MiniKdcRunner.class); + String sslConfDir = getClasspathDir(MiniKdcRunner.class); FileUtils.deleteDirectory(new File(sslKeystoresDir)); FileUtils.forceMkdir(new File(sslKeystoresDir)); - SecurityUtils.setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true); - config.set("dfs.https.server.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-server")); + setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true); + config.set("dfs.https.server.keystore.resource", getSSLConfigFileName("ssl-server")); // Configurations used by both NameNode and DataNode config.set("dfs.block.access.token.enable", "true"); config.set("dfs.http.policy", "HTTPS_ONLY"); // Configurations used by DFSClient config.set("dfs.data.transfer.protection", "privacy"); - config.set("dfs.client.https.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-client")); + config.set("dfs.client.https.keystore.resource", getSSLConfigFileName("ssl-client")); // YARN related configurations config.set("yarn.resourcemanager.principal", hadoopPrincipal); @@ -309,4 +331,219 @@ public class MiniKdcRunner extends KdcRunner { } } } + + @SuppressWarnings("rawtypes") + private static String getClasspathDir(Class klass) throws Exception { + String file = klass.getName(); + file = file.replace('.', '/') + ".class"; + URL url = Thread.currentThread().getContextClassLoader().getResource(file); + String baseDir = url.toURI().getPath(); + baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); + return baseDir; + } + + /** + * Performs complete setup of SSL configuration. This includes keys, certs, + * keystores, truststores, the server SSL configuration file, + * the client SSL configuration file. + * + * @param keystoresDir String directory to save keystores + * @param sslConfDir String directory to save SSL configuration files + * @param conf Configuration + * @param useClientCert boolean true to make the client present a cert in the + * SSL handshake + * @param trustStore boolean true to create truststore, false not to create it + */ + private void setupSSLConfig(String keystoresDir, String sslConfDir, + Configuration conf, boolean useClientCert, boolean trustStore) + throws Exception { + String clientKS = keystoresDir + "/clientKS.jks"; + String clientPassword = "clientP"; + String serverKS = keystoresDir + "/serverKS.jks"; + String serverPassword = "serverP"; + String trustKS = null; + String trustPassword = "trustP"; + + File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client")); + File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server")); + + Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>(); + + if (useClientCert) { + KeyPair cKP = generateKeyPair("RSA"); + X509Certificate cCert = generateCertificate("CN=localhost, O=client", cKP, 30, "SHA1withRSA"); + createKeyStore(clientKS, clientPassword, "client", cKP.getPrivate(), cCert); + certs.put("client", cCert); + } + + KeyPair sKP = generateKeyPair("RSA"); + X509Certificate sCert = generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA"); + createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert); + certs.put("server", sCert); + + if (trustStore) { + trustKS = keystoresDir + "/trustKS.jks"; + createTrustStore(trustKS, trustPassword, certs); + } + + Configuration clientSSLConf = createSSLConfig( + SSLFactory.Mode.CLIENT, clientKS, clientPassword, clientPassword, trustKS); + Configuration serverSSLConf = createSSLConfig( + SSLFactory.Mode.SERVER, serverKS, serverPassword, serverPassword, trustKS); + + saveConfig(sslClientConfFile, clientSSLConf); + saveConfig(sslServerConfFile, serverSSLConf); + + conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); + conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); + conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); + conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); + } + + /** + * Returns an SSL configuration file name. Under parallel test + * execution, this file name is parameterized by a unique ID to ensure that + * concurrent tests don't collide on an SSL configuration file. + * + * @param base the base of the file name + * @return SSL configuration file name for base + */ + private static String getSSLConfigFileName(String base) { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : ""; + return base + fileSuffix + ".xml"; + } + + /** + * Creates SSL configuration. + * + * @param mode SSLFactory.Mode mode to configure + * @param keystore String keystore file + * @param password String store password, or null to avoid setting store + * password + * @param keyPassword String key password, or null to avoid setting key + * password + * @param trustKS String truststore file + * @return Configuration for SSL + */ + private static Configuration createSSLConfig(SSLFactory.Mode mode, + String keystore, String password, String keyPassword, String trustKS) { + String trustPassword = "trustP"; + + Configuration sslConf = new Configuration(false); + if (keystore != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore); + } + if (password != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password); + } + if (keyPassword != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), + keyPassword); + } + if (trustKS != null) { + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); + } + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), + trustPassword); + sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, + FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); + + return sslConf; + } + + private static KeyPair generateKeyPair(String algorithm) + throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); + keyGen.initialize(1024); + return keyGen.genKeyPair(); + } + + /** + * Create a self-signed X.509 Certificate. + * + * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" + * @param pair the KeyPair + * @param days how many days from now the Certificate is valid for + * @param algorithm the signing algorithm, eg "SHA1withRSA" + * @return the self-signed certificate + */ + private static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm) + throws CertificateEncodingException, + InvalidKeyException, + IllegalStateException, + NoSuchProviderException, NoSuchAlgorithmException, SignatureException{ + Date from = new Date(); + Date to = new Date(from.getTime() + days * 86400000l); + BigInteger sn = new BigInteger(64, new SecureRandom()); + KeyPair keyPair = pair; + X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); + X500Principal dnName = new X500Principal(dn); + + certGen.setSerialNumber(sn); + certGen.setIssuerDN(dnName); + certGen.setNotBefore(from); + certGen.setNotAfter(to); + certGen.setSubjectDN(dnName); + certGen.setPublicKey(keyPair.getPublic()); + certGen.setSignatureAlgorithm(algorithm); + + X509Certificate cert = certGen.generate(pair.getPrivate()); + return cert; + } + + private static void createKeyStore(String filename, + String password, String alias, + Key privateKey, Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + ks.setKeyEntry(alias, privateKey, password.toCharArray(), + new Certificate[]{cert}); + saveKeyStore(ks, filename, password); + } + + private static <T extends Certificate> void createTrustStore( + String filename, String password, Map<String, T> certs) + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + for (Map.Entry<String, T> cert : certs.entrySet()) { + ks.setCertificateEntry(cert.getKey(), cert.getValue()); + } + saveKeyStore(ks, filename, password); + } + + private static void saveKeyStore(KeyStore ks, String filename, + String password) + throws GeneralSecurityException, IOException { + FileOutputStream out = new FileOutputStream(filename); + try { + ks.store(out, password.toCharArray()); + } finally { + out.close(); + } + } + + /** + * Saves configuration to a file. + * + * @param file File to save + * @param conf Configuration contents to write to file + * @throws IOException if there is an I/O error saving the file + */ + private static void saveConfig(File file, Configuration conf) + throws IOException { + Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java deleted file mode 100644 index 35b9b7d..0000000 --- a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.test.utils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory; -import org.apache.hadoop.security.ssl.SSLFactory; -import org.bouncycastle.x509.X509V1CertificateGenerator; - -import javax.security.auth.x500.X500Principal; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.math.BigInteger; -import java.security.GeneralSecurityException; -import java.security.InvalidKeyException; -import java.security.Key; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.SecureRandom; -import java.security.SignatureException; -import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; -import java.security.cert.X509Certificate; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -public class SecurityUtils { - - public static final String CLIENT_KEYSTORE = "clientKS.jks"; - public static final String SERVER_KEYSTORE = "serverKS.jks"; - public static final String TRUSTSTORE = "trustKS.jks"; - - public static final String CLIENT_KEY_PASSWORD = "client_key"; - public static final String CLIENT_KEY_STORE_PASSWORD = "client_keystore"; - public static final String SERVER_KEY_PASSWORD = "server_key"; - public static final String SERVER_KEY_STORE_PASSWORD = "server_keystore"; - - /** - * Performs complete setup of SSL configuration. This includes keys, certs, - * keystores, truststores, the server SSL configuration file, - * the client SSL configuration file. - * - * @param keystoresDir String directory to save keystores - * @param sslConfDir String directory to save SSL configuration files - * @param conf Configuration - * @param useClientCert boolean true to make the client present a cert in the - * SSL handshake - * @param trustStore boolean true to create truststore, false not to create it - */ - public static X509Certificate setupSSLConfig(String keystoresDir, String sslConfDir, - Configuration conf, boolean useClientCert, boolean trustStore) - throws Exception { - String clientKeyStorePath = keystoresDir + "/" + CLIENT_KEYSTORE; - String serverKeyStorePath = keystoresDir + "/" + SERVER_KEYSTORE; - String serverPassword = "serverP"; - String trustKS = null; - String trustPassword = "trustP"; - - File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client")); - File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server")); - - Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>(); - - String hostname = SqoopUtils.getLocalHostName(); - - if (useClientCert) { - KeyPair cKP = generateKeyPair("RSA"); - X509Certificate cCert = generateCertificate("CN=" + hostname + ", O=client", cKP, 30, "SHA1withRSA"); - createKeyStore(clientKeyStorePath, CLIENT_KEY_PASSWORD, CLIENT_KEY_STORE_PASSWORD, "client", cKP.getPrivate(), cCert); - certs.put("client", cCert); - } - - KeyPair sKP = generateKeyPair("RSA"); - X509Certificate sCert = generateCertificate("CN=" + hostname + ", O=server", sKP, 30, "SHA1withRSA"); - createKeyStore(serverKeyStorePath, SERVER_KEY_PASSWORD, SERVER_KEY_STORE_PASSWORD, "server", sKP.getPrivate(), sCert); - certs.put("server", sCert); - - if (trustStore) { - trustKS = keystoresDir + TRUSTSTORE; - createTrustStore(trustKS, trustPassword, certs); - } - - Configuration clientSSLConf = createSSLConfig( - SSLFactory.Mode.CLIENT, clientKeyStorePath, CLIENT_KEY_STORE_PASSWORD, CLIENT_KEY_PASSWORD, trustKS); - Configuration serverSSLConf = createSSLConfig( - SSLFactory.Mode.SERVER, serverKeyStorePath, SERVER_KEY_STORE_PASSWORD, SERVER_KEY_PASSWORD, trustKS); - - saveConfig(sslClientConfFile, clientSSLConf); - saveConfig(sslServerConfFile, serverSSLConf); - - conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL"); - conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName()); - conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName()); - conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert); - - return sCert; - } - - /** - * Returns an SSL configuration file name. Under parallel test - * execution, this file name is parameterized by a unique ID to ensure that - * concurrent tests don't collide on an SSL configuration file. - * - * @param base the base of the file name - * @return SSL configuration file name for base - */ - public static String getSSLConfigFileName(String base) { - String testUniqueForkId = System.getProperty("test.unique.fork.id"); - String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : ""; - return base + fileSuffix + ".xml"; - } - - /** - * Creates SSL configuration. - * - * @param mode SSLFactory.Mode mode to configure - * @param keystore String keystore file - * @param keyStorePassword String store password, or null to avoid setting store - * password - * @param keyPassword String key password, or null to avoid setting key - * password - * @param trustKS String truststore file - * @return Configuration for SSL - */ - public static Configuration createSSLConfig(SSLFactory.Mode mode, - String keystore, String keyStorePassword, - String keyPassword, String trustKS) { - String trustPassword = "trustP"; - - Configuration sslConf = new Configuration(false); - if (keystore != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore); - } - - if (keyStorePassword != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), keyStorePassword); - } - if (keyPassword != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), - keyPassword); - } - if (trustKS != null) { - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS); - } - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY), - trustPassword); - sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode, - FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000"); - - return sslConf; - } - - public static KeyPair generateKeyPair(String algorithm) - throws NoSuchAlgorithmException { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm); - keyGen.initialize(1024); - return keyGen.genKeyPair(); - } - - /** - * Create a self-signed X.509 Certificate. - * - * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB" - * @param pair the KeyPair - * @param days how many days from now the Certificate is valid for - * @param algorithm the signing algorithm, eg "SHA1withRSA" - * @return the self-signed certificate - */ - public static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm) - throws CertificateEncodingException, - InvalidKeyException, - IllegalStateException, - NoSuchProviderException, NoSuchAlgorithmException, SignatureException { - Date from = new Date(); - Date to = new Date(from.getTime() + days * 86400000l); - BigInteger sn = new BigInteger(64, new SecureRandom()); - KeyPair keyPair = pair; - X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); - X500Principal dnName = new X500Principal(dn); - - certGen.setSerialNumber(sn); - certGen.setIssuerDN(dnName); - certGen.setNotBefore(from); - certGen.setNotAfter(to); - certGen.setSubjectDN(dnName); - certGen.setPublicKey(keyPair.getPublic()); - certGen.setSignatureAlgorithm(algorithm); - - X509Certificate cert = certGen.generate(pair.getPrivate()); - return cert; - } - - public static void createKeyStore(String filename, - String keyPassword, String keyStorePassword, - String alias, Key privateKey, Certificate cert) - throws GeneralSecurityException, IOException { - KeyStore ks = KeyStore.getInstance("JKS"); - ks.load(null, null); // initialize - ks.setKeyEntry(alias, privateKey, keyPassword.toCharArray(), - new Certificate[]{cert}); - saveKeyStore(ks, filename, keyStorePassword); - } - - public static <T extends Certificate> void createTrustStore( - String filename, String password, Map<String, T> certs) - throws GeneralSecurityException, IOException { - KeyStore ks = KeyStore.getInstance("JKS"); - ks.load(null, null); // initialize - for (Map.Entry<String, T> cert : certs.entrySet()) { - ks.setCertificateEntry(cert.getKey(), cert.getValue()); - } - saveKeyStore(ks, filename, password); - } - - public static void saveKeyStore(KeyStore ks, String filename, - String password) - throws GeneralSecurityException, IOException { - FileOutputStream out = new FileOutputStream(filename); - try { - ks.store(out, password.toCharArray()); - } finally { - out.close(); - } - } - - /** - * Saves configuration to a file. - * - * @param file File to save - * @param conf Configuration contents to write to file - * @throws IOException if there is an I/O error saving the file - */ - public static void saveConfig(File file, Configuration conf) - throws IOException { - Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); - try { - conf.writeXml(writer); - } finally { - writer.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java index 3e0566f..6614b19 100644 --- a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java +++ b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java @@ -18,7 +18,6 @@ package org.apache.sqoop.test.utils; import java.net.InetAddress; -import java.net.URL; import java.net.UnknownHostException; import java.util.Locale; import java.util.Random; @@ -45,7 +44,7 @@ public class SqoopUtils { object.setName(prefix + rand.nextLong()); } - // Retrieve the FQDN of the current host + //Retrieve the FQDN of the current host public static String getLocalHostName() { String fqdn; try { @@ -55,26 +54,4 @@ public class SqoopUtils { } return fqdn; } - - // Retrieve the IP address of the current host - public static String getLocalIpAddress() { - String address; - try { - address = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e1) { - address = "127.0.0.1"; - } - return address; - } - - - @SuppressWarnings("rawtypes") - public static String getClasspathDir(Class klass) throws Exception { - String file = klass.getName(); - file = file.replace('.', '/') + ".class"; - URL url = Thread.currentThread().getContextClassLoader().getResource(file); - String baseDir = url.toURI().getPath(); - baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1); - return baseDir; - } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java new file mode 100644 index 0000000..7f0575b --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.connectorloading; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +@Test(groups = "no-real-cluster") +@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class}) +public class BlacklistedConnectorTest extends SqoopTestCase { + private SqoopMiniCluster sqoopMiniCluster; + + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { + super(temporaryPath, configuration); + } + + @Override + protected Map<String, String> getBlacklistedConnectorConfiguration() { + Map<String, String> properties = new HashMap<>(); + + properties.put(ConfigurationConstants.BLACKLISTED_CONNECTORS, "fake-connector:generic-jdbc-connector"); + return properties; + } + } + + @BeforeMethod(dependsOnMethods = { "init" }) + public void startSqoopMiniCluster() throws Exception { + // And use them for new Derby repo instance + sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super + .getTemporaryPath(), getTestName()), getHadoopConf()); + KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); + if (kdcProvider != null) { + sqoopMiniCluster.setKdc(kdcProvider.getInstance()); + } + + // Start server + sqoopMiniCluster.start(); + + // Initialize Sqoop Client API + initSqoopClient(sqoopMiniCluster.getServerUrl()); + } + + @Test(expectedExceptions = {SqoopException.class}) + public void testCreateLinkWithNonexistantConnector() throws Exception { + getClient().createLink("generic-jdbc-connector"); + } + + @AfterMethod + public void stopCluster() throws Exception { + sqoopMiniCluster.stop(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java new file mode 100644 index 0000000..87f0eb1 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.integration.connectorloading; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.utils.ConnectorUtils; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Test(groups = "no-real-cluster") +@Infrastructure(dependencies = {KdcInfrastructureProvider.class, DatabaseInfrastructureProvider.class, HadoopInfrastructureProvider.class}) +public class ClasspathTest extends SqoopTestCase { + + private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar"; + private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar"; + + private static final String[] CONNECTOR_SOURCE_FILES = { + "TestConnector/TestConnector.java", + "TestConnector/TestLinkConfiguration.java", + "TestConnector/TestLoader.java", + "TestConnector/TestToDestroyer.java", + "TestConnector/TestToInitializer.java", + "TestConnector/TestToJobConfiguration.java" + }; + + private static final String[] CONNECTOR_DEPENDENCY_SOURCE_FILES = { + "TestConnector/TestDependency.java" + }; + + private static final String[] CONNECTOR_PROPERTY_FILES = { + "TestConnector/sqoopconnector.properties" + }; + + private ClassLoader classLoader; + private SqoopMiniCluster sqoopMiniCluster; + + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + + private String extraClasspath; + private String jobExtraClasspath; + + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception { + super(temporaryPath, configuration); + this.extraClasspath = extraClasspath; + this.jobExtraClasspath = jobExtraClasspath; + } + + @Override + protected Map<String, String> getClasspathConfiguration() { + Map<String, String> properties = new HashMap<>(); + + if (extraClasspath != null) { + properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); + } + if (jobExtraClasspath != null) { + properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath); + } + + + return properties; + } + } + + public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception { + // And use them for new Derby repo instance + sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments + (super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath, jobExtraClasspath); + KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); + if (kdcProvider != null) { + sqoopMiniCluster.setKdc(kdcProvider.getInstance()); + } + + // Start server + sqoopMiniCluster.start(); + + // Initialize Sqoop Client API + initSqoopClient(sqoopMiniCluster.getServerUrl()); + } + + @BeforeMethod + public void captureClasspath() { + classLoader = Thread.currentThread().getContextClassLoader(); + } + + @AfterMethod + public void restoreClasspath(){ + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Test + public void testClasspathSqoopProperties() throws Exception { + Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency( + CONNECTOR_SOURCE_FILES, + CONNECTOR_DEPENDENCY_SOURCE_FILES, + CONNECTOR_PROPERTY_FILES, + TEST_CONNECTOR_JAR_NAME, + TEST_DEPENDENCY_JAR_NAME, + false); + startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get + (TEST_DEPENDENCY_JAR_NAME)); + createAndLoadTableCities(); + + MJob job = prepareJob(); + + prepareDriverConfig(job); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + ConnectorUtils.deleteJars(jarMap); + } + + @Test + public void testClasspathDriverInput() throws Exception{ + Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency( + CONNECTOR_SOURCE_FILES, + CONNECTOR_DEPENDENCY_SOURCE_FILES, + CONNECTOR_PROPERTY_FILES, + TEST_CONNECTOR_JAR_NAME, + TEST_DEPENDENCY_JAR_NAME, + false); + startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null); + createAndLoadTableCities(); + + MJob job = prepareJob(); + + MDriverConfig driverConfig = prepareDriverConfig(job); + + List<String> extraJars = new ArrayList<>(); + extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME)); + driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + ConnectorUtils.deleteJars(jarMap); + } + + private MJob prepareJob() { + MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsConnection); + saveLink(rdbmsConnection); + + MLink testConnection = getClient().createLink("test-connector"); + saveLink(testConnection); + + MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName()); + + fillRdbmsFromConfig(job, "id"); + + return job; + } + + private MDriverConfig prepareDriverConfig(MJob job) { + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + + return driverConfig; + } + + private void stopSqoop() throws Exception { + sqoopMiniCluster.stop(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java new file mode 100644 index 0000000..a82a4da --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.integration.connectorloading; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; +import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; +import org.apache.sqoop.test.minicluster.SqoopMiniCluster; +import org.apache.sqoop.test.utils.ConnectorUtils; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "no-real-cluster") +@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class}) +public class ConnectorClasspathIsolationTest extends SqoopTestCase { + + private static final String TEST_FROM_CONNECTOR_JAR_NAME = "test-from-connector.jar"; + private static final String TEST_TO_CONNECTOR_JAR_NAME = "test-to-connector.jar"; + private static final String TEST_FROM_DEPENDENCY_JAR_NAME = "test-from-dependency.jar"; + private static final String TEST_TO_DEPENDENCY_JAR_NAME = "test-to-dependency.jar"; + + private static final String[] FROM_CONNECTOR_SOURCE_FILES = { + "TestConnectorClasspathIsolation/from/TestFromConnector.java", + "TestConnectorClasspathIsolation/from/TestExtractor.java", + "TestConnectorClasspathIsolation/from/TestFromDestroyer.java", + "TestConnectorClasspathIsolation/from/TestFromInitializer.java", + "TestConnectorClasspathIsolation/from/TestFromJobConfiguration.java", + "TestConnectorClasspathIsolation/from/TestPartition.java", + "TestConnectorClasspathIsolation/from/TestPartitioner.java", + "TestConnectorClasspathIsolation/from/TestFromLinkConfiguration.java" + }; + + private static final String[] FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES = { + "TestConnectorClasspathIsolation/from/TestClasspathIsolation.java" + }; + + private static final String[] FROM_CONNECTOR_PROPERTY_FILES = { + "TestConnectorClasspathIsolation/from/sqoopconnector.properties" + }; + + private static final String[] TO_CONNECTOR_SOURCE_FILES = { + "TestConnectorClasspathIsolation/to/TestToConnector.java", + "TestConnectorClasspathIsolation/to/TestLoader.java", + "TestConnectorClasspathIsolation/to/TestToDestroyer.java", + "TestConnectorClasspathIsolation/to/TestToInitializer.java", + "TestConnectorClasspathIsolation/to/TestToJobConfiguration.java", + "TestConnectorClasspathIsolation/to/TestToLinkConfiguration.java" + }; + + private static final String[] TO_CONNECTOR_DEPENDENCY_SOURCE_FILES = { + "TestConnectorClasspathIsolation/to/TestClasspathIsolation.java" + }; + + private static final String[] TO_CONNECTOR_PROPERTY_FILES = { + "TestConnectorClasspathIsolation/to/sqoopconnector.properties" + }; + + private ClassLoader classLoader; + private SqoopMiniCluster sqoopMiniCluster; + + public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { + + private String extraClasspath; + + public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath) throws Exception { + super(temporaryPath, configuration); + this.extraClasspath = extraClasspath; + } + + @Override + protected Map<String, String> getClasspathConfiguration() { + Map<String, String> properties = new HashMap<>(); + + if (extraClasspath != null) { + properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); + } + + return properties; + } + } + + public void startSqoopMiniCluster(String extraClasspath) throws Exception { + // And use them for new Derby repo instance + sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath); + KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); + if (kdcProvider != null) { + sqoopMiniCluster.setKdc(kdcProvider.getInstance()); + } + + // Start server + sqoopMiniCluster.start(); + + // Initialize Sqoop Client API + initSqoopClient(sqoopMiniCluster.getServerUrl()); + } + + @BeforeMethod + public void captureClasspath() { + classLoader = Thread.currentThread().getContextClassLoader(); + } + + @AfterMethod + public void restoreClasspath(){ + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Test + public void testConnectorClasspathIsolation() throws Exception { + Map<String, String> fromConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( + FROM_CONNECTOR_SOURCE_FILES, + FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES, + FROM_CONNECTOR_PROPERTY_FILES, + TEST_FROM_CONNECTOR_JAR_NAME, + TEST_FROM_DEPENDENCY_JAR_NAME, + true); + Map<String, String> toConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency( + TO_CONNECTOR_SOURCE_FILES, + TO_CONNECTOR_DEPENDENCY_SOURCE_FILES, + TO_CONNECTOR_PROPERTY_FILES, + TEST_TO_CONNECTOR_JAR_NAME, + TEST_TO_DEPENDENCY_JAR_NAME, + true); + startSqoopMiniCluster( + StringUtils.join(Arrays.asList(fromConnectorJarMap.get(TEST_FROM_CONNECTOR_JAR_NAME), toConnectorJarMap.get(TEST_TO_CONNECTOR_JAR_NAME)), ":")); + + MJob job = prepareJob(); + + prepareDriverConfig(job); + + saveJob(job); + + executeJob(job); + + stopSqoop(); + ConnectorUtils.deleteJars(fromConnectorJarMap); + } + + private MJob prepareJob() { + MLink rdbmsConnection = getClient().createLink("test-from-connector"); + saveLink(rdbmsConnection); + + MLink testConnection = getClient().createLink("test-to-connector"); + saveLink(testConnection); + + MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName()); + + return job; + } + + private MDriverConfig prepareDriverConfig(MJob job) { + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + + return driverConfig; + } + + private void stopSqoop() throws Exception { + sqoopMiniCluster.stop(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java deleted file mode 100644 index 438e182..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration.serverproperties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.core.ConfigurationConstants; -import org.apache.sqoop.test.infrastructure.Infrastructure; -import org.apache.sqoop.test.infrastructure.SqoopTestCase; -import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.util.HashMap; -import java.util.Map; - -@Test(groups = "no-real-cluster") -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class}) -public class BlacklistedConnectorTest extends SqoopTestCase { - private SqoopMiniCluster sqoopMiniCluster; - - public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { - public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception { - super(temporaryPath, configuration); - } - - @Override - protected Map<String, String> getBlacklistedConnectorConfiguration() { - Map<String, String> properties = new HashMap<>(); - - properties.put(ConfigurationConstants.BLACKLISTED_CONNECTORS, "fake-connector:generic-jdbc-connector"); - return properties; - } - } - - @BeforeMethod(dependsOnMethods = { "init" }) - public void startSqoopMiniCluster() throws Exception { - // And use them for new Derby repo instance - sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super - .getTemporaryPath(), getTestName()), getHadoopConf()); - KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); - if (kdcProvider != null) { - sqoopMiniCluster.setKdc(kdcProvider.getInstance()); - } - - // Start server - sqoopMiniCluster.start(); - - // Initialize Sqoop Client API - initSqoopClient(sqoopMiniCluster.getServerUrl()); - } - - @Test(expectedExceptions = {SqoopException.class}) - public void testCreateLinkWithNonexistantConnector() throws Exception { - getClient().createLink("generic-jdbc-connector"); - } - - @AfterMethod - public void stopCluster() throws Exception { - sqoopMiniCluster.stop(); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java deleted file mode 100644 index 1a33230..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sqoop.integration.serverproperties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.core.ConfigurationConstants; -import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.test.infrastructure.Infrastructure; -import org.apache.sqoop.test.infrastructure.SqoopTestCase; -import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; -import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider; -import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; -import org.apache.sqoop.test.minicluster.SqoopMiniCluster; -import org.apache.sqoop.test.utils.ConnectorUtils; -import org.apache.sqoop.test.utils.HdfsUtils; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Test(groups = "no-real-cluster") -@Infrastructure(dependencies = {KdcInfrastructureProvider.class, DatabaseInfrastructureProvider.class, HadoopInfrastructureProvider.class}) -public class ClasspathTest extends SqoopTestCase { - - private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar"; - private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar"; - - private static final String[] CONNECTOR_SOURCE_FILES = { - "TestConnector/TestConnector.java", - "TestConnector/TestLinkConfiguration.java", - "TestConnector/TestLoader.java", - "TestConnector/TestToDestroyer.java", - "TestConnector/TestToInitializer.java", - "TestConnector/TestToJobConfiguration.java" - }; - - private static final String[] CONNECTOR_DEPENDENCY_SOURCE_FILES = { - "TestConnector/TestDependency.java" - }; - - private static final String[] CONNECTOR_PROPERTY_FILES = { - "TestConnector/sqoopconnector.properties" - }; - - private ClassLoader classLoader; - private SqoopMiniCluster sqoopMiniCluster; - - public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster { - - private String extraClasspath; - private String jobExtraClasspath; - - public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception { - super(temporaryPath, configuration); - this.extraClasspath = extraClasspath; - this.jobExtraClasspath = jobExtraClasspath; - } - - @Override - protected Map<String, String> getClasspathConfiguration() { - Map<String, String> properties = new HashMap<>(); - - if (extraClasspath != null) { - properties.put(ConfigurationConstants.CLASSPATH, extraClasspath); - } - if (jobExtraClasspath != null) { - properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath); - } - - - return properties; - } - } - - public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception { - // And use them for new Derby repo instance - sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments - (super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath, jobExtraClasspath); - KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class); - if (kdcProvider != null) { - sqoopMiniCluster.setKdc(kdcProvider.getInstance()); - } - - // Start server - sqoopMiniCluster.start(); - - // Initialize Sqoop Client API - initSqoopClient(sqoopMiniCluster.getServerUrl()); - } - - @BeforeMethod - public void captureClasspath() { - classLoader = Thread.currentThread().getContextClassLoader(); - } - - @AfterMethod - public void restoreClasspath(){ - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Test - public void testClasspathSqoopProperties() throws Exception { - Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency( - CONNECTOR_SOURCE_FILES, - CONNECTOR_DEPENDENCY_SOURCE_FILES, - CONNECTOR_PROPERTY_FILES, - TEST_CONNECTOR_JAR_NAME, - TEST_DEPENDENCY_JAR_NAME, - false); - startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get - (TEST_DEPENDENCY_JAR_NAME)); - createAndLoadTableCities(); - - MJob job = prepareJob(); - - prepareDriverConfig(job); - - saveJob(job); - - executeJob(job); - - stopSqoop(); - ConnectorUtils.deleteJars(jarMap); - } - - @Test - public void testClasspathDriverInput() throws Exception{ - Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency( - CONNECTOR_SOURCE_FILES, - CONNECTOR_DEPENDENCY_SOURCE_FILES, - CONNECTOR_PROPERTY_FILES, - TEST_CONNECTOR_JAR_NAME, - TEST_DEPENDENCY_JAR_NAME, - false); - startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null); - createAndLoadTableCities(); - - MJob job = prepareJob(); - - MDriverConfig driverConfig = prepareDriverConfig(job); - - List<String> extraJars = new ArrayList<>(); - extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME)); - driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars); - - saveJob(job); - - executeJob(job); - - stopSqoop(); - ConnectorUtils.deleteJars(jarMap); - } - - private MJob prepareJob() { - MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkConfig(rdbmsConnection); - saveLink(rdbmsConnection); - - MLink testConnection = getClient().createLink("test-connector"); - saveLink(testConnection); - - MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName()); - - fillRdbmsFromConfig(job, "id"); - - return job; - } - - private MDriverConfig prepareDriverConfig(MJob job) { - MDriverConfig driverConfig = job.getDriverConfig(); - driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); - - return driverConfig; - } - - private void stopSqoop() throws Exception { - sqoopMiniCluster.stop(); - } -}
