Repository: sqoop Updated Branches: refs/heads/sqoop2 d615b39c1 -> 45d1c32d7
SQOOP-2016: Sqoop2: Create integration test for JDBC to Hive (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/45d1c32d Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/45d1c32d Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/45d1c32d Branch: refs/heads/sqoop2 Commit: 45d1c32d7ed4811452c42626940d80a4b091f797 Parents: d615b39 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Mar 24 17:41:13 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Mar 24 17:41:13 2015 -0700 ---------------------------------------------------------------------- .../sqoop/common/test/db/HiveProvider.java | 101 ++++++++++++++ .../sqoop/common/test/utils/NetworkUtils.java | 36 +++++ .../connector/kite/KiteDatasetExecutor.java | 2 +- .../sqoop/connector/kite/TestKiteExecutor.java | 13 +- pom.xml | 12 ++ test/pom.xml | 12 ++ .../sqoop/test/hive/HiveServerRunner.java | 135 ++++++++++++++++++ .../test/hive/HiveServerRunnerFactory.java | 56 ++++++++ .../test/hive/InternalHiveServerRunner.java | 55 ++++++++ .../hive/InternalMetastoreServerRunner.java | 71 ++++++++++ .../sqoop/test/hive/MetastoreServerRunner.java | 137 +++++++++++++++++++ .../test/hive/MetastoreServerRunnerFactory.java | 56 ++++++++ .../minicluster/TomcatSqoopMiniCluster.java | 4 + .../test/testcases/HiveConnectorTestCase.java | 88 ++++++++++++ .../connector/hive/FromRDBMSToKiteHiveTest.java | 96 +++++++++++++ .../connector/kite/FromRDBMSToKiteTest.java | 4 +- 16 files changed, 873 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java new file mode 100644 index 0000000..dfe0f43 --- /dev/null +++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.common.test.db; + +/** + * MySQL Provider that will connect to remote MySQL server. + * + * JDBC can be configured via system properties. Default value is server running + * on the same box (localhost) that is access via sqoop/sqoop credentials. + */ +public class HiveProvider extends DatabaseProvider { + + public static final String DRIVER = "org.apache.hive.jdbc.HiveDriver"; + + private static final String CONNECTION = System.getProperties().getProperty( + "sqoop.provider.hive.jdbc", + "jdbc:hive2://" + ); + + private static final String USERNAME = System.getProperties().getProperty( + "sqoop.provider.hive.username", + "sqoop" + ); + + private static final String PASSWORD = System.getProperties().getProperty( + "sqoop.provider.hive.password", + "sqoop" + ); + + private String jdbcUrl; + + /** + * Use system properties to get JDBC URL. + */ + public HiveProvider() { + this.jdbcUrl = CONNECTION; + } + + /** + * Use JDBC URL provided. + * + * @param jdbcUrl hive server jdbc URL. + */ + public HiveProvider(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + @Override + public String getConnectionUrl() { + return jdbcUrl; + } + + @Override + public String getConnectionUsername() { + return USERNAME; + } + + @Override + public String getConnectionPassword() { + return PASSWORD; + } + + @Override + public String escapeColumnName(String columnName) { + return escape(columnName); + } + + @Override + public String escapeTableName(String tableName) { + return escape(tableName); + } + + @Override + public String escapeValueString(String value) { + return escape(value); + } + + @Override + public String getJdbcDriver() { + return DRIVER; + } + + public String escape(String entity) { + return entity; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java index 87534c7..7f0f750 100644 --- a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java +++ b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java @@ -17,14 +17,21 @@ */ package org.apache.sqoop.common.test.utils; +import org.apache.log4j.Logger; + import java.io.IOException; +import java.net.InetAddress; import java.net.ServerSocket; +import java.net.Socket; +import java.util.concurrent.TimeoutException; /** * Network related utilities. */ public class NetworkUtils { + private static final Logger LOG = Logger.getLogger(NetworkUtils.class); + /** * Create port number that is available on this machine for * subsequent use. @@ -58,6 +65,35 @@ public class NetworkUtils { } } + /** + * Create a socket and attempt to connect to ``hostname``:``port`` + * ``numberOfAttempts`` amount of times with ``sleepTime`` milliseconds + * between each attempt. + * + * @param hostname host name to connect to + * @param port port to connect on + * @param numberOfAttempts number of tries to connect + * @param sleepTime time in between connection attempts in milliseconds + * @throws InterruptedException + * @throws TimeoutException + */ + public static void waitForStartUp(String hostname, int port, int numberOfAttempts, long sleepTime) + throws InterruptedException, TimeoutException { + for (int i = 0; i < numberOfAttempts; ++i) { + try { + LOG.debug("Attempt " + (i + 1) + " to access " + hostname + ":" + port); + new Socket(InetAddress.getByName(hostname), port).close(); + return; + } catch (Exception e) { + LOG.debug("Failed to connect to " + hostname + ":" + port, e); + } + + Thread.sleep(sleepTime); + } + + throw new TimeoutException("Couldn't access new server: " + hostname + ":" + port); + } + private NetworkUtils() { // Instantiation is prohibited } http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java index db8d4e6..6aa28be 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java @@ -189,7 +189,7 @@ public class KiteDatasetExecutor { // Replace dataset name with temporary dataset name. uriParts[1] = uriParts[1].substring(0, uriParts[1].lastIndexOf("/")) + "/" + temporaryDatasetName; } else { - uriParts[1] = temporaryDatasetName; + uriParts[1] = ":" + temporaryDatasetName; } } else { uriParts[1] += "/" + temporaryDatasetName; http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java index 44721a8..0e797f8 100644 --- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java +++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java @@ -38,7 +38,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; -import static org.testng.AssertJUnit.assertTrue; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class TestKiteExecutor { @@ -173,6 +174,16 @@ public class TestKiteExecutor { assertTrue(suggestedUri.length() > subURI.length()); assertTrue(suggestedUri.contains(subURI)); assertTrue(suggestedUri.endsWith(endURI)); + + endURI = "auth:host=metastore&auth:port=9083"; + uri = "dataset:hive:sqoop?auth:host=metastore&auth:port=9083"; + subURI = "dataset:hive:"; + suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri); + assertTrue(suggestedUri.length() > subURI.length()); + assertTrue(suggestedUri.contains(subURI), suggestedUri); + assertTrue(suggestedUri.endsWith(endURI), suggestedUri); + assertFalse(suggestedUri.contains("sqoop")); + assertFalse(suggestedUri.contains("/")); } private static Schema createTwoFieldSchema() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e3431bd..c608ca7 100644 --- a/pom.xml +++ b/pom.xml @@ -368,6 +368,18 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${hive.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + <version>${hive.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> <version>${hive.version}</version> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index f743d25..d8fbfa2 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -113,6 +113,18 @@ limitations under the License. </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>org.codehaus.cargo</groupId> <artifactId>cargo-core-container-tomcat</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java new file mode 100644 index 0000000..8b355bd --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.utils.NetworkUtils; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + * Hive server runner for testing purpose. + * + * Runner provides methods for bootstrapping and using HiveServer2. This + * should allow providing a HiveServer2 minicluster or using a real server. + */ +public abstract class HiveServerRunner { + private static final Logger LOG = Logger.getLogger(HiveServerRunner.class); + + private final String hostname; + private final int port; + + public HiveServerRunner(String hostname, int port) throws Exception { + this.hostname = hostname; + + if (port == 0) { + this.port = NetworkUtils.findAvailablePort(); + } else { + this.port = port; + } + + LOG.info("Hive Server will bind to port " + getPort()); + } + + /** + * Configuration object. + */ + protected HiveConf config = null; + + /** + * Start Hive server. + * + * @throws Exception + */ + abstract public void start() throws Exception; + + /** + * Stop Hive server. + * + * @throws Exception + */ + abstract public void stop() throws Exception; + + /** + * Return JDBC URL to be used with HiveServer2 instance. + * + * @return String + */ + public String getUrl() { + return "jdbc:hive2://" + hostname + ":" + port + "/default"; + } + + /** + * Prepare configuration object. This method should be called once before the + * start method is called. + * + * @param config is the configuration object to prepare. + */ + public Configuration prepareConfiguration(Configuration config) throws Exception { + config.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, getHostName()); + config.setInt(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, getPort()); + return config; + } + + /** + * Get configuration. + * + * @return HiveConf + */ + public HiveConf getConfiguration() { + return config; + } + + /** + * Set the configuration object. + * + * @param config + */ + public void setConfiguration(Configuration config) { + this.config = new HiveConf(); + this.config.addResource(config); + this.printConfig(); + } + + /** + * Hostname used to start services on. + * + * @return String hostname + */ + public String getHostName() { + return hostname; + } + + /** + * Port hive service will be on. + * + * @return int port + */ + public int getPort() { + return this.port; + } + + private void printConfig() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + config.logVars(new PrintStream(baos)); + LOG.debug("Hive server runner configuration:\n" + baos.toString()); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java new file mode 100644 index 0000000..f6a2fa6 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import java.util.Properties; + +/** + * Create HiveServer2 runner. + */ +public class HiveServerRunnerFactory { + + public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.runner.class"; + + public static final String HOSTNAME_PROPERTY = "sqoop.hive.server.hostname"; + + public static final String PORT_PROPERTY = "sqoop.hive.server.port"; + + public static final String DEFAULT_HOSTNAME = "127.0.0.1"; + + public static final String DEFAULT_PORT = "0"; + + public static HiveServerRunner getRunner( + Properties properties, Class<? extends HiveServerRunner> defaultRunner) + throws Exception { + Class<?> klass; + + String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME); + int port = Integer.parseInt( + properties.getProperty(PORT_PROPERTY, DEFAULT_PORT)); + + String className = properties.getProperty(RUNNER_CLASS_PROPERTY); + if(className == null) { + klass = defaultRunner; + } else { + klass = Class.forName(className); + } + + return (HiveServerRunner)klass.getConstructor(String.class, int.class) + .newInstance(hostname, port); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java new file mode 100644 index 0000000..3418525 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import org.apache.hive.service.server.HiveServer2; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.utils.NetworkUtils; + +/** + * Use HiveServer2 JDBC client to run all operations. + * + * @see org.apache.sqoop.test.hive.HiveServerRunner + */ +public class InternalHiveServerRunner extends HiveServerRunner { + private static final Logger LOG = Logger.getLogger(InternalHiveServerRunner.class); + + private final HiveServer2 hiveServer2; + + public InternalHiveServerRunner(String hostname, int port) throws Exception { + super(hostname, port); + hiveServer2 = new HiveServer2(); + } + + @Override + public void start() throws Exception { + Long start = System.currentTimeMillis(); + hiveServer2.init(getConfiguration()); + hiveServer2.start(); + NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 100); + Long end = System.currentTimeMillis(); + LOG.debug("Hive service took " + (end - start) + "ms to start"); + } + + @Override + public void stop() throws Exception { + if (hiveServer2 != null) { + hiveServer2.stop(); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java new file mode 100644 index 0000000..a8282ec --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.utils.NetworkUtils; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Start a Hive Metastore service on the specified hostname and port. + * + * @see org.apache.sqoop.test.hive.MetastoreServerRunner + */ +public class InternalMetastoreServerRunner extends MetastoreServerRunner { + private static final Logger LOG = Logger.getLogger(InternalMetastoreServerRunner.class); + + private ExecutorService executor = Executors + .newSingleThreadExecutor(); + + public InternalMetastoreServerRunner(String hostname, int port) throws Exception { + super(hostname, port); + } + + @Override + public void start() throws Exception { + Long start = System.currentTimeMillis(); + final int metastorePort = getPort(); + final HiveConf conf = getConfiguration(); + Callable<Void> metastoreService = new Callable<Void>() { + public Void call() throws Exception { + try { + HiveMetaStore.startMetaStore(metastorePort, + ShimLoader.getHadoopThriftAuthBridge(), conf); + while(true); + } catch (Throwable e) { + throw new Exception("Error starting metastore", e); + } + } + }; + executor.submit(metastoreService); + NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 1000); + Long end = System.currentTimeMillis(); + LOG.debug("Metastore service took " + (end - start) + "ms to start"); + } + + @Override + public void stop() throws Exception { + executor.shutdownNow(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java new file mode 100644 index 0000000..32a6b35 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.utils.NetworkUtils; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; + +/** + * Metastore server runner for testing purpose. + * + * Runner provides methods for bootstrapping and using the Hive metastore. This + * should allow providing a Hive metastore minicluster or using a real server. + */ +public abstract class MetastoreServerRunner { + private static final Logger LOG = Logger.getLogger(MetastoreServerRunner.class); + + private final String hostname; + private final int port; + private final String warehouseDirectory; + + public MetastoreServerRunner(String hostname, int port) throws Exception { + this.hostname = hostname; + this.warehouseDirectory = "/user/hive/" + UUID.randomUUID(); + + if (port == 0) { + port = NetworkUtils.findAvailablePort(); + } + LOG.info("Hive Metastore will bind to port " + port); + + this.port = port; + } + + /** + * Configuration object. + */ + protected HiveConf config = null; + + /** + * Start Hive server. + * + * @throws Exception + */ + abstract public void start() throws Exception; + + /** + * Stop Hive server. + * + * @throws Exception + */ + abstract public void stop() throws Exception; + + /** + * Metastore URI authority (ex. hostname:port). + * + * @return String metastore authority. + */ + public String getAuthority() { + return getHostName() + ":" + getPort(); + } + + /** + * Prepare configuration object. This method should be called once before the + * start method is called. + * + * @param config is the configuration object to prepare. + */ + public Configuration prepareConfiguration(Configuration config) throws Exception { + config.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + getHostName() + ":" + getPort()); + config.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouseDirectory); + return config; + } + + /** + * Get configuration. + * + * @return HiveConf + */ + public HiveConf getConfiguration() { + return config; + } + + /** + * Set the configuration object. + * + * @param config + */ + public void setConfiguration(Configuration config) { + this.config = new HiveConf(); + this.config.addResource(config); + this.printConfig(); + } + + /** + * Hostname used to start services on. + * + * @return String hostname + */ + public String getHostName() { + return hostname; + } + + /** + * Port meta store service will be on. + * + * @return int port + */ + public int getPort() { + return this.port; + } + + private void printConfig() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + config.logVars(new PrintStream(baos)); + LOG.debug("Hive server runner configuration:\n" + baos.toString()); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java new file mode 100644 index 0000000..be3720f --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.hive; + +import java.util.Properties; + +/** + * Create HiveServer2 runner. + */ +public class MetastoreServerRunnerFactory { + + public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.metastore.runner.class"; + + public static final String HOSTNAME_PROPERTY = "sqoop.hive.metastore.server.hostname"; + + public static final String PORT_PROPERTY = "sqoop.hive.metastore.server.port"; + + public static final String DEFAULT_HOSTNAME = "127.0.0.1"; + + public static final String DEFAULT_PORT = "0"; + + public static MetastoreServerRunner getRunner( + Properties properties, Class<? extends MetastoreServerRunner> defaultRunner) + throws Exception { + Class<?> klass; + + String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME); + int port = Integer.parseInt( + properties.getProperty(PORT_PROPERTY, DEFAULT_PORT)); + + String className = properties.getProperty(RUNNER_CLASS_PROPERTY); + if(className == null) { + klass = defaultRunner; + } else { + klass = Class.forName(className); + } + + return (MetastoreServerRunner)klass.getConstructor(String.class, int.class) + .newInstance(hostname, port); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java index 648e2f6..4d27886 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java @@ -95,6 +95,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster { String []classpath = System.getProperty("java.class.path").split(":"); for(String jar : classpath) { if(jar.contains("hadoop-") || // Hadoop jars + jar.contains("hive-") || // Hive jars jar.contains("commons-") || // Apache Commons libraries jar.contains("httpcore-") || // Apache Http Core libraries jar.contains("httpclient-") || // Apache Http Client libraries @@ -106,6 +107,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster { jar.contains("jackson-") || // Jackson jar.contains("derby") || // Derby drivers jar.contains("avro-") || // Avro + jar.contains("parquet-") || // Parquet jar.contains("mysql") || // MySQL JDBC driver jar.contains("postgre") || // PostgreSQL JDBC driver jar.contains("oracle") || // Oracle driver @@ -113,6 +115,8 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster { jar.contains("tdgs") || // Teradata driver jar.contains("nzjdbc") || // Netezza driver jar.contains("sqljdbc") || // Microsoft SQL Server driver + jar.contains("libfb303") || // Facebook thrift lib + jar.contains("datanucleus-") || // Data nucleus libs jar.contains("google") // Google libraries (guava, ...) ) { extraClassPath.add(jar); http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java new file mode 100644 index 0000000..628f484 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.testcases; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.test.db.HiveProvider; +import org.apache.sqoop.test.hive.InternalHiveServerRunner; +import org.apache.sqoop.test.hive.HiveServerRunner; +import org.apache.sqoop.test.hive.HiveServerRunnerFactory; +import org.apache.sqoop.test.hive.InternalMetastoreServerRunner; +import org.apache.sqoop.test.hive.MetastoreServerRunner; +import org.apache.sqoop.test.hive.MetastoreServerRunnerFactory; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +public class HiveConnectorTestCase extends ConnectorTestCase { + private static final Logger LOG = Logger.getLogger(HiveConnectorTestCase.class); + + protected HiveServerRunner hiveServerRunner; + protected MetastoreServerRunner metastoreServerRunner; + protected HiveProvider hiveProvider; + + private void ensureWarehouseDirectory(Configuration conf) throws Exception { + String warehouseDirectory = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname); + StringBuilder dir = new StringBuilder(); + for (String part : warehouseDirectory.split("/")) { + dir.append(part).append("/"); + Path path = new Path(dir.toString()); + if (!hdfsClient.exists(path)) { + hdfsClient.mkdirs(path); + } + } + hdfsClient.setPermission(new Path(dir.toString()), new FsPermission((short)01777)); + } + + @BeforeMethod(alwaysRun = true) + public void startHive() throws Exception { + String databasePath = HdfsUtils.joinPathFragments(getTemporaryPath(), "metastore_db"); + metastoreServerRunner = MetastoreServerRunnerFactory.getRunner(System.getProperties(), InternalMetastoreServerRunner.class); + metastoreServerRunner.setConfiguration(metastoreServerRunner.prepareConfiguration(hadoopCluster.getConfiguration())); + metastoreServerRunner.getConfiguration().set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + "jdbc:derby:;databaseName=" + databasePath + ";create=true"); + ensureWarehouseDirectory(metastoreServerRunner.getConfiguration()); + LOG.info("Starting Metastore Server: " + metastoreServerRunner.getClass().getName()); + metastoreServerRunner.start(); + + hiveServerRunner = HiveServerRunnerFactory.getRunner(System.getProperties(), InternalHiveServerRunner.class); + hiveServerRunner.setConfiguration(hiveServerRunner.prepareConfiguration(metastoreServerRunner.getConfiguration())); + LOG.info("Starting Hive Server: " + hiveServerRunner.getClass().getName()); + hiveServerRunner.start(); + + LOG.info("Starting Hive Provider: " + provider.getClass().getName()); + hiveProvider = new HiveProvider(hiveServerRunner.getUrl()); + hiveProvider.start(); + } + + @AfterMethod(alwaysRun = true) + public void stopHive() throws Exception { + LOG.info("Stopping Hive Provider: " + provider.getClass().getName()); + hiveProvider.stop(); + + LOG.info("Stopping Hive Server: " + hiveServerRunner.getClass().getName()); + hiveServerRunner.stop(); + + LOG.info("Stopping Metastore Server: " + metastoreServerRunner.getClass().getName()); + metastoreServerRunner.stop(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java new file mode 100644 index 0000000..e789ce0 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.connector.hive; + +import org.apache.sqoop.common.Direction; +import org.apache.sqoop.common.test.asserts.ProviderAsserts; +import org.apache.sqoop.common.test.db.TableName; +import org.apache.sqoop.connector.common.FileFormat; +import org.apache.sqoop.model.MConfigList; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.testcases.HiveConnectorTestCase; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + */ +public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase { + private MLink rdbmsLink; + private MLink kiteLink; + + @BeforeMethod(alwaysRun = true) + public void createTable() { + createAndLoadTableCities(); + } + + @AfterMethod(alwaysRun = true) + public void dropTable() { + super.dropTable(); + } + + @BeforeMethod(alwaysRun = true) + public void createLinks() { + // RDBMS link + rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + saveLink(rdbmsLink); + + // Kite link + kiteLink = getClient().createLink("kite-connector"); + kiteLink.getConnectorLinkConfig().getStringInput("linkConfig.authority") + .setValue(metastoreServerRunner.getAuthority()); + saveLink(kiteLink); + } + + @Test + public void testCities() throws Exception { + // Job creation + MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId()); + + // Set rdbms "FROM" config + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fillRdbmsFromConfig(job, "id"); + // TODO: Kite have troubles with some data types, so we're limiting the columns to int only + fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); + + // Fill the Kite "TO" config + MConfigList toConfig = job.getJobConfig(Direction.TO); + toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable"); + toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO); + + // driver config + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1); + + saveJob(job); + + executeJob(job); + + // Assert correct output + ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1"); + ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2"); + ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3"); + ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4"); + + hiveProvider.dropTable(new TableName("testtable")); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java index 8ca1c3a..828d244 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java @@ -29,8 +29,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -/** - */ +@Test public class FromRDBMSToKiteTest extends ConnectorTestCase { @BeforeMethod(alwaysRun = true) public void createTable() { @@ -96,5 +95,4 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase { "\"4\"" ); } - }
