Updated Branches: refs/heads/sqoop2 9c7adb4e1 -> 3631b29e0
SQOOP-1114. Sqoop2: Integration: Add tests for various Generic JDBC Connector splitters (Jarek Jarcec Cecho via Hari Shreedharan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3631b29e Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3631b29e Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3631b29e Branch: refs/heads/sqoop2 Commit: 3631b29e06c5e78d666752a65b0fe01601dc3259 Parents: 9c7adb4 Author: Hari Shreedharan <[email protected]> Authored: Tue Jul 30 23:28:50 2013 -0700 Committer: Hari Shreedharan <[email protected]> Committed: Tue Jul 30 23:28:50 2013 -0700 ---------------------------------------------------------------------- pom.xml | 2 +- .../apache/sqoop/test/asserts/HdfsAsserts.java | 11 ++ .../apache/sqoop/test/data/UbuntuReleases.java | 72 +++++++++++ .../sqoop/test/testcases/ConnectorTestCase.java | 15 +++ .../sqoop/test/testcases/TomcatTestCase.java | 9 ++ .../sqoop/test/utils/ParametrizedUtils.java | 91 ++++++++++++++ .../jdbc/generic/imports/PartitionerTest.java | 125 +++++++++++++++++++ 7 files changed, 324 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 04cf479..c1ed306 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ limitations under the License. <hadoop.2.version>2.0.2-alpha</hadoop.2.version> <guava.version>11.0.2</guava.version> <json-simple.version>1.1</json-simple.version> - <junit.version>4.9</junit.version> + <junit.version>4.11</junit.version> <mockito.version>1.9.5</mockito.version> <log4j.version>1.2.16</log4j.version> <servlet.version>2.5</servlet.version> http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java index 056e612..c175272 100644 --- a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java +++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java @@ -30,6 +30,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** @@ -76,6 +77,16 @@ public class HdfsAsserts { } } + /** + * Verify number of output mapreduce files. + * + * @param directory Mapreduce output directory + * @param expectedFiles Expected number of files + */ + public static void assertMapreduceOutputFiles(String directory, int expectedFiles) { + String []files = HdfsUtils.getOutputMapreduceFiles(directory); + assertEquals(expectedFiles, files.length); + } private HdfsAsserts() { // Instantiation is prohibited http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java new file mode 100644 index 0000000..a4730c2 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.data; + +import org.apache.sqoop.test.db.DatabaseProvider; + +/** + * Releases of Ubuntu Linux. + * + * Purpose of this set is to cover most common data types (varchar, int, numeric, date, boolean). + */ +public class UbuntuReleases extends DataSet { + + public UbuntuReleases(DatabaseProvider provider, String tableBaseName) { + super(provider, tableBaseName); + } + + @Override + public DataSet createTables() { + provider.createTable( + tableBaseName, + "id", + "id", "int", + "code_name", "varchar(64)", + "version", "numeric(4,2)", + "release_date", "date", + "lts", "boolean" + ); + + return this; + } + + @Override + public DataSet loadBasicData() { + provider.insertRow(tableBaseName, 1, "Warty Warthog", 4.10, "2004-10-20", false); + provider.insertRow(tableBaseName, 2, "Hoary Hedgehog", 5.04, "2005-04-08", false); + provider.insertRow(tableBaseName, 3, "Breezy Badger", 5.10, "2005-10-13", false); + provider.insertRow(tableBaseName, 4, "Dapper Drake", 6.06, "2006-06-01", true); + provider.insertRow(tableBaseName, 5, "Edgy Eft", 6.10, "2006-10-26", false); + provider.insertRow(tableBaseName, 6, "Feisty Fawn", 7.04, "2007-04-19", false); + provider.insertRow(tableBaseName, 7, "Gutsy Gibbon", 7.10, "2007-10-18", false); + provider.insertRow(tableBaseName, 8, "Hardy Heron", 8.04, "2008-04-24", true); + provider.insertRow(tableBaseName, 9, "Intrepid Ibex", 8.10, "2008-10-18", false); + provider.insertRow(tableBaseName, 10, "Jaunty Jackalope", 9.04, "2009-04-23", false); + provider.insertRow(tableBaseName, 11, "Karmic Koala", 9.10, "2009-10-29", false); + provider.insertRow(tableBaseName, 12, "Lucid Lynx", 10.04, "2010-04-29", true); + provider.insertRow(tableBaseName, 13, "Maverick Meerkat",10.10, "2010-10-10", false); + provider.insertRow(tableBaseName, 14, "Natty Narwhal", 11.04, "2011-04-28", false); + provider.insertRow(tableBaseName, 15, "Oneiric Ocelot", 11.10, "2011-10-10", false); + provider.insertRow(tableBaseName, 16, "Precise Pangolin",12.04, "2012-04-26", true); + provider.insertRow(tableBaseName, 17, "Quantal Quetzal", 12.10, "2012-10-18", false); + provider.insertRow(tableBaseName, 18, "Raring Ringtail", 13.04, "2013-04-25", false); + provider.insertRow(tableBaseName, 19, "Saucy Salamander",13.10, "2013-10-17", false); + + return this; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index bf1c91f..d10b942 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -28,6 +28,7 @@ import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.test.asserts.ProviderAsserts; import org.apache.sqoop.test.data.Cities; +import org.apache.sqoop.test.data.UbuntuReleases; import org.apache.sqoop.test.db.DatabaseProvider; import org.apache.sqoop.test.db.DatabaseProviderFactory; import org.apache.sqoop.validation.Status; @@ -155,6 +156,20 @@ abstract public class ConnectorTestCase extends TomcatTestCase { } /** + * Create table for ubuntu releases. + */ + protected void createTableUbuntuReleases() { + new UbuntuReleases(provider, getTableName()).createTables(); + } + + /** + * Create table for ubuntu releases. + */ + protected void createAndLoadTableUbuntuReleases() { + new UbuntuReleases(provider, getTableName()).createTables().loadBasicData(); + } + + /** * Assert row in testing table. * * @param conditions Conditions in form that are expected by the database provider http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java index ca77e64..efdfed4 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java @@ -134,6 +134,15 @@ abstract public class TomcatTestCase { } /** + * Verify number of output mapreduce files. + * + * @param expectedFiles Expected number of files + */ + protected void assertMapreduceOutputFiles(int expectedFiles) { + HdfsAsserts.assertMapreduceOutputFiles(getMapreduceDirectory(), expectedFiles); + } + + /** * Create mapreduce input file with specified content. * * @param filename Input file name http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java new file mode 100644 index 0000000..656bc34 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.utils; + +import org.apache.commons.lang.ArrayUtils; + +import java.util.LinkedList; + +/** + * Various utils to help build expected structure for JUNIT Parametrized runner. + */ +public class ParametrizedUtils { + + /** + * Join two arrays of parameters by providing their cross product to make + * testing matrix for Parameterized runner easier. + * + * This method accepts both arguments as array of arrays. In such case + * the inner array will be flattened during the join to meet the Parametrized + * runner requirements. + * + * @param v1 First array of test values + * @param v2 Second array of test values + * @return Cross product of v1 and v2 + */ + public static Iterable<Object []> crossProduct(Object[] v1, Object[] v2) { + LinkedList<Object []> ret = new LinkedList<Object []>(); + + for(Object o1 : v1) { + for(Object o2 : v2) { + ret.add(mergeObjects(o1, o2)); + } + } + + return ret; + } + + /** + * Merge two objects into array. + * + * This method will flatten any argument that is already an array, e.g.: + * + * mergeObjects(1, 2) = [1, 2] + * mergeObjects([1], 2) = [1, 2] + * mergeObjects(1, [2]) = [1, 2] + * mergeObjects([1], [2]) = [1, 2] + * + * @param o1 + * @param o2 + * @return + */ + public static Object [] mergeObjects(Object o1, Object o2) { + if(!o1.getClass().isArray() && !o2.getClass().isArray()) { + return new Object[] { o1, o2 }; + } + + Object []a1 = toArray(o1); + Object []a2 = toArray(o2); + + return ArrayUtils.addAll(a1, a2); + } + + /** + * Convert object into one element array if it's not an array already. + * + * @param o + * @return + */ + public static Object[] toArray(Object o) { + if(o.getClass().isArray()) { + return (Object [])o; + } + + return new Object[] { o }; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java new file mode 100644 index 0000000..50fce74 --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.connector.jdbc.generic.imports; + +import org.apache.log4j.Logger; +import org.apache.sqoop.framework.configuration.OutputFormat; +import org.apache.sqoop.framework.configuration.StorageType; +import org.apache.sqoop.model.MConnection; +import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.test.testcases.ConnectorTestCase; +import org.apache.sqoop.test.utils.ParametrizedUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * + */ +@RunWith(Parameterized.class) +public class PartitionerTest extends ConnectorTestCase { + + private static final Logger LOG = Logger.getLogger(PartitionerTest.class); + + /** + * Columns that we will use as partition column with maximal number of + * partitions that can be created for such column. + */ + public static Object[] COLUMNS = new Object [][] { + {"id", 13}, + {"code_name", 13}, + {"version", 13}, + {"release_date", 13}, + {"lts", 2}, + }; + + /** + * Number of extractors that we will use to transfer the table. + */ + public static Object [] EXTRACTORS = new Object[] { + 3, 5, 10, 13, + }; + + @Parameterized.Parameters(name = "{0}-{1}-{2}") + public static Iterable<Object[]> data() { + return ParametrizedUtils.crossProduct(COLUMNS, EXTRACTORS); + } + + private String partitionColumn; + private int extractors; + private int maxOutputFiles; + + public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors) { + this.partitionColumn = partitionColumn; + this.maxOutputFiles = expectedOutputFiles; + this.extractors = extractors; + } + + @Test + public void testSplitter() throws Exception { + createAndLoadTableUbuntuReleases(); + + // Connection creation + MConnection connection = getClient().newConnection(1L); + fillConnectionForm(connection); + createConnection(connection); + + // Job creation + MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT); + + // Connector values + MFormList forms = job.getConnectorPart(); + forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName())); + forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); + // Framework values + fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE); + forms = job.getFrameworkPart(); + forms.getIntegerInput("throttling.extractors").setValue(extractors); + createJob(job); + + runJob(job); + + // Assert correct output + assertMapreduceOutputFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors); + assertMapreduceOutput( + "1,'Warty Warthog',4.10,2004-10-20,false", + "2,'Hoary Hedgehog',5.04,2005-04-08,false", + "3,'Breezy Badger',5.10,2005-10-13,false", + "4,'Dapper Drake',6.06,2006-06-01,true", + "5,'Edgy Eft',6.10,2006-10-26,false", + "6,'Feisty Fawn',7.04,2007-04-19,false", + "7,'Gutsy Gibbon',7.10,2007-10-18,false", + "8,'Hardy Heron',8.04,2008-04-24,true", + "9,'Intrepid Ibex',8.10,2008-10-18,false", + "10,'Jaunty Jackalope',9.04,2009-04-23,false", + "11,'Karmic Koala',9.10,2009-10-29,false", + "12,'Lucid Lynx',10.04,2010-04-29,true", + "13,'Maverick Meerkat',10.10,2010-10-10,false", + "14,'Natty Narwhal',11.04,2011-04-28,false", + "15,'Oneiric Ocelot',11.10,2011-10-10,false", + "16,'Precise Pangolin',12.04,2012-04-26,true", + "17,'Quantal Quetzal',12.10,2012-10-18,false", + "18,'Raring Ringtail',13.04,2013-04-25,false", + "19,'Saucy Salamander',13.10,2013-10-17,false" + ); + + // Clean up testing table + dropTable(); + } +}
