Repository: sqoop Updated Branches: refs/heads/sqoop2 b9d6a0528 -> 8f8205118
SQOOP-1952: Sqoop2: Load miniclusters at beginning of integration tests (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8f820511 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8f820511 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8f820511 Branch: refs/heads/sqoop2 Commit: 8f82051188cbaf824defa301e02d08468122827e Parents: b9d6a05 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Apr 23 15:04:41 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Apr 23 15:04:41 2015 -0700 ---------------------------------------------------------------------- .../sqoop/common/test/kafka/TestUtil.java | 7 +++- pom.xml | 10 ++++- test/pom.xml | 9 ++--- .../sqoop/test/testcases/ConnectorTestCase.java | 19 +-------- .../sqoop/test/testcases/TomcatTestCase.java | 38 +++++++++--------- .../connector/jdbc/generic/AllTypesTest.java | 36 ++++++++--------- .../jdbc/generic/IncrementalReadTest.java | 28 ++++++------- .../connector/kite/FromRDBMSToKiteTest.java | 2 +- .../test/resources/integration-tests-suite.xml | 41 ++++++++++++++++++++ 9 files changed, 112 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/common-test/src/main/java/org/apache/sqoop/common/test/kafka/TestUtil.java ---------------------------------------------------------------------- diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/kafka/TestUtil.java b/common-test/src/main/java/org/apache/sqoop/common/test/kafka/TestUtil.java index 09ddcc7..f9a3c30 100644 --- a/common-test/src/main/java/org/apache/sqoop/common/test/kafka/TestUtil.java +++ b/common-test/src/main/java/org/apache/sqoop/common/test/kafka/TestUtil.java @@ -91,7 +91,12 @@ public class TestUtil { public void tearDown() throws IOException { logger.info("Shutting down the Kafka Consumer."); - getKafkaConsumer().shutdown(); + + // Set kafkaConsumer to null so that a new one is created when + // getKafkaConsumer is called. + kafkaConsumer.shutdown(); + kafkaConsumer = null; + try { Thread.sleep(3 * 1000); // add this sleep time to // ensure that the server is fully started before proceeding with tests. http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c608ca7..bc2bec7 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ limitations under the License. <hive.version>0.13.1</hive.version> <guava.version>11.0.2</guava.version> <json-simple.version>1.1</json-simple.version> - <testng.version>6.8.13</testng.version> + <testng.version>6.8.17</testng.version> <mockito.version>1.9.5</mockito.version> <powermock.version>1.5.6</powermock.version> <log4j.version>1.2.16</log4j.version> @@ -154,6 +154,7 @@ limitations under the License. <artifactId>maven-surefire-plugin</artifactId> <configuration> <excludedGroups>slow</excludedGroups> + <reportNameSuffix>fast</reportNameSuffix> </configuration> </plugin> @@ -162,6 +163,7 @@ limitations under the License. <artifactId>maven-failsafe-plugin</artifactId> <configuration> <excludedGroups>slow</excludedGroups> + <reportNameSuffix>slow</reportNameSuffix> </configuration> </plugin> </plugins> @@ -184,6 +186,7 @@ limitations under the License. <artifactId>maven-surefire-plugin</artifactId> <configuration> <groups>slow</groups> + <reportNameSuffix>slow</reportNameSuffix> </configuration> </plugin> @@ -192,6 +195,7 @@ limitations under the License. <artifactId>maven-failsafe-plugin</artifactId> <configuration> <groups>slow</groups> + <reportNameSuffix>slow</reportNameSuffix> </configuration> </plugin> </plugins> @@ -665,6 +669,10 @@ limitations under the License. <name>usedefaultlisteners</name> <value>false</value> <!-- disabling default listeners is optional --> </property> + <property> + <name>listener</name> + <value>org.testng.reporters.JUnitReportReporter</value> + </property> </properties> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index d8fbfa2..98a60f6 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -193,12 +193,9 @@ limitations under the License. </goals> <phase>integration-test</phase> <configuration> - <excludes> - <exclude>none</exclude> - </excludes> - <includes> - <include>**/integration/**</include> - </includes> + <suiteXmlFiles> + <suiteXmlFile>src/test/resources/integration-tests-suite.xml</suiteXmlFile> + </suiteXmlFiles> </configuration> </execution> </executions> http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index c8fca68..7420739 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; import org.apache.log4j.Logger; import org.apache.sqoop.client.SubmissionCallback; -import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.test.asserts.ProviderAsserts; import org.apache.sqoop.common.test.db.DatabaseProvider; import org.apache.sqoop.common.test.db.DatabaseProviderFactory; @@ -42,8 +41,7 @@ import org.apache.sqoop.test.data.UbuntuReleases; import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner; import org.apache.sqoop.test.hadoop.HadoopRunnerFactory; import org.apache.sqoop.validation.Status; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; /** @@ -79,26 +77,13 @@ abstract public class ConnectorTestCase extends TomcatTestCase { }; @BeforeSuite(alwaysRun = true) - public static void startHadoop() throws Exception { - // Start Hadoop Clusters - hadoopCluster = HadoopRunnerFactory.getHadoopCluster(System.getProperties(), HadoopMiniClusterRunner.class); - hadoopCluster.setTemporaryPath(TMP_PATH_BASE); - hadoopCluster.setConfiguration( hadoopCluster.prepareConfiguration(new JobConf()) ); - hadoopCluster.start(); - - // Initialize Hdfs Client - hdfsClient = FileSystem.get(hadoopCluster.getConfiguration()); - LOG.debug("HDFS Client: " + hdfsClient); - } - - @BeforeClass(alwaysRun = true) public static void startProvider() throws Exception { provider = DatabaseProviderFactory.getProvider(System.getProperties()); LOG.info("Starting database provider: " + provider.getClass().getName()); provider.start(); } - @AfterClass(alwaysRun = true) + @AfterSuite(alwaysRun = true) public static void stopProvider() { LOG.info("Stopping database provider: " + provider.getClass().getName()); provider.stop(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java index 9416473..6729cc7 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java @@ -27,28 +27,26 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.log4j.Logger; import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.test.asserts.HdfsAsserts; +import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner; import org.apache.sqoop.test.hadoop.HadoopRunner; import org.apache.sqoop.test.hadoop.HadoopRunnerFactory; -import org.apache.sqoop.test.hadoop.HadoopLocalRunner; import org.apache.sqoop.test.minicluster.SqoopMiniCluster; import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; import org.apache.sqoop.test.utils.HdfsUtils; import org.testng.ITest; import org.testng.annotations.AfterMethod; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; /** * Basic test case that will bootstrap Sqoop server running in external Tomcat * process. */ -abstract public class TomcatTestCase { +abstract public class TomcatTestCase implements ITest { private static final Logger LOG = Logger.getLogger(TomcatTestCase.class); - public String name; + public String methodName; /** * Temporary base path that will be used for tests. @@ -59,7 +57,7 @@ abstract public class TomcatTestCase { * in case that no property is set. */ protected static final String TMP_PATH_BASE = - System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/"; + System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests"; /** * Temporary directory that will be used by the test. @@ -92,12 +90,19 @@ abstract public class TomcatTestCase { */ private SqoopClient client; + /** + * Use the method name as the test name + */ + public String getTestName() { + return methodName; + } + @BeforeSuite(alwaysRun = true) - public static void startHadoop() throws Exception { + public void startHadoop() throws Exception { // Start Hadoop Clusters - hadoopCluster = HadoopRunnerFactory.getHadoopCluster(System.getProperties(), HadoopLocalRunner.class); + hadoopCluster = HadoopRunnerFactory.getHadoopCluster(System.getProperties(), HadoopMiniClusterRunner.class); hadoopCluster.setTemporaryPath(TMP_PATH_BASE); - hadoopCluster.setConfiguration( hadoopCluster.prepareConfiguration(new JobConf()) ); + hadoopCluster.setConfiguration(hadoopCluster.prepareConfiguration(new JobConf())); hadoopCluster.start(); // Initialize Hdfs Client @@ -106,18 +111,11 @@ abstract public class TomcatTestCase { } @BeforeMethod(alwaysRun = true) - public void findMethodName(Method method) { - if(this instanceof ITest) { - name = ((ITest)this).getTestName(); - } else { - name = method.getName(); - } - } + public void startServer(Method method) throws Exception { + methodName = method.getName(); - @BeforeMethod(alwaysRun = true) - public void startServer() throws Exception { // Get and set temporary path in hadoop cluster. - tmpPath = HdfsUtils.joinPathFragments(TMP_PATH_BASE, getClass().getName(), name); + tmpPath = HdfsUtils.joinPathFragments(TMP_PATH_BASE, getClass().getName(), getTestName()); FileUtils.deleteDirectory(new File(tmpPath)); LOG.debug("Temporary Directory: " + tmpPath); @@ -187,7 +185,7 @@ abstract public class TomcatTestCase { * @return */ public String getMapreduceDirectory() { - return HdfsUtils.joinPathFragments(hadoopCluster.getTestDirectory(), getClass().getName(), name); + return HdfsUtils.joinPathFragments(hadoopCluster.getTestDirectory(), getClass().getName(), getTestName()); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java index 9c5770e..cdb6f81 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java @@ -18,7 +18,6 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import com.google.common.collect.Iterables; -import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.test.db.DatabaseProvider; import org.apache.sqoop.common.test.db.DatabaseProviderFactory; import org.apache.sqoop.common.test.db.types.DatabaseType; @@ -31,13 +30,11 @@ import org.apache.sqoop.model.MLink; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.test.utils.ParametrizedUtils; import org.testng.ITest; -import org.testng.annotations.BeforeMethod; +import org.testng.ITestContext; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; import org.testng.annotations.Test; -import java.lang.reflect.Method; - import static org.testng.Assert.assertEquals; /** @@ -46,8 +43,14 @@ import static org.testng.Assert.assertEquals; @Test(groups = "slow") public class AllTypesTest extends ConnectorTestCase implements ITest { - @DataProvider(name="all-types-test", parallel=true) - public static Object[][] data() throws Exception { + private static String testName; + + private DatabaseType type; + + @DataProvider(name="all-types-test", parallel=false) + public static Object[][] data(ITestContext context) throws Exception { + testName = context.getName(); + DatabaseProvider provider = DatabaseProviderFactory.getProvider(System.getProperties()); return Iterables.toArray(ParametrizedUtils.toArrayOfArrays(provider.getDatabaseTypes().getAllTypes()), Object[].class); } @@ -57,7 +60,14 @@ public class AllTypesTest extends ConnectorTestCase implements ITest { this.type = type; } - private DatabaseType type; + @Override + public String getTestName() { + if (methodName == null) { + return testName; + } else { + return methodName + "[" + type.name + "]"; + } + } @Test public void testFrom() throws Exception { @@ -150,16 +160,4 @@ public class AllTypesTest extends ConnectorTestCase implements ITest { // Clean up testing table dropTable(); } - - private String testName; - - @BeforeMethod(alwaysRun = true) - public void beforeMethod(Method aMethod) { - this.testName = aMethod.getName(); - } - - @Override - public String getTestName() { - return testName + "[" + type.name + "]"; - } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java index 0355a36..66c016d 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java @@ -25,6 +25,7 @@ import org.apache.sqoop.model.MLink; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.test.utils.ParametrizedUtils; import org.testng.ITest; +import org.testng.ITestContext; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; @@ -45,6 +46,8 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { {"release_date", "2008-10-18", "2013-10-17"}, }; + private static String testName; + private String checkColumn; private String lastValue; private String newMaxValue; @@ -57,10 +60,21 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { } @DataProvider(name="incremental-integration-test", parallel=true) - public static Object[][] data() { + public static Object[][] data(ITestContext context) { + testName = context.getName(); + return Iterables.toArray(ParametrizedUtils.toArrayOfArrays(COLUMNS), Object[].class); } + @Override + public String getTestName() { + if (methodName == null) { + return testName; + } else { + return methodName + "[" + checkColumn + "]"; + } + } + @Test public void testTable() throws Exception { createAndLoadTableUbuntuReleases(); @@ -167,16 +181,4 @@ public class IncrementalReadTest extends ConnectorTestCase implements ITest { // Clean up testing table dropTable(); } - - private String testName; - - @BeforeMethod(alwaysRun = true) - public void beforeMethod(Method aMethod) { - this.testName = aMethod.getName(); - } - - @Override - public String getTestName() { - return testName + "[" + checkColumn + "]"; - } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java index 5c90501..71e405b 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java @@ -49,7 +49,7 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase { */ @Override public String getMapreduceDirectory() { - return HdfsUtils.joinPathFragments(hadoopCluster.getTestDirectory(), getClass().getName(), "namespace", name).replaceAll("/$", ""); + return HdfsUtils.joinPathFragments(hadoopCluster.getTestDirectory(), getClass().getName(), "namespace", getTestName()).replaceAll("/$", ""); } @Test http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f820511/test/src/test/resources/integration-tests-suite.xml ---------------------------------------------------------------------- diff --git a/test/src/test/resources/integration-tests-suite.xml b/test/src/test/resources/integration-tests-suite.xml new file mode 100644 index 0000000..101b6ec --- /dev/null +++ b/test/src/test/resources/integration-tests-suite.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd"; > + +<suite name="IntegrationTests" verbose="2" parallel="false"> + + <test name="ConnectorTests"> + <packages> + <package name="org.apache.sqoop.integration.connector.*"/> + </packages> + </test> + + <test name="RepositoryTests"> + <packages> + <package name="org.apache.sqoop.integration.repository.derby.upgrade"/> + </packages> + </test> + + <test name="ServerTests"> + <packages> + <package name="org.apache.sqoop.integration.server"/> + </packages> + </test> + +</suite> \ No newline at end of file
