Updated Branches: refs/heads/sqoop2 fe54d473a -> f980e90fc
SQOOP-1087: Sqoop2: Integration: Abstract common functionality into src module (Jarek Jarcec Cecho via Kate Ting) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f980e90f Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f980e90f Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f980e90f Branch: refs/heads/sqoop2 Commit: f980e90fc9570b01305166b19d7381c2051bc49e Parents: fe54d47 Author: Kate Ting <kathl...@apache.org> Authored: Sun Jun 23 18:43:50 2013 -0400 Committer: Kate Ting <kathl...@apache.org> Committed: Sun Jun 23 18:43:50 2013 -0400 ---------------------------------------------------------------------- test/pom.xml | 1 - .../apache/sqoop/test/asserts/HdfsAsserts.java | 83 ++++++++ .../sqoop/test/asserts/ProviderAsserts.java | 74 +++++++ .../java/org/apache/sqoop/test/data/Cities.java | 53 +++++ .../org/apache/sqoop/test/data/DataSet.java | 66 ++++++ .../sqoop/test/testcases/ConnectorTestCase.java | 177 ++++++++++++++++ .../sqoop/test/testcases/TomcatTestCase.java | 146 +++++++++++++ .../org/apache/sqoop/test/utils/HdfsUtils.java | 69 ++++++ .../sqoop/integration/TomcatTestCase.java | 197 ----------------- .../connector/ConnectorTestCase.java | 209 ------------------- .../connector/jdbc/generic/TableExportTest.java | 10 +- .../connector/jdbc/generic/TableImportTest.java | 2 +- .../sqoop/integration/server/VersionTest.java | 2 +- 13 files changed, 675 insertions(+), 414 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/pom.xml ---------------------------------------------------------------------- diff --git a/test/pom.xml b/test/pom.xml index b7ea1ed..8001fce 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -33,7 +33,6 @@ limitations under the License. <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java new file mode 100644 index 0000000..056e612 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.asserts; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.sqoop.test.utils.HdfsUtils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.fail; + +/** + * Assert methods suitable for checking HDFS files and directories. + * + * TODO: This module will require clean up to work on MiniCluster/Real cluster. + */ +public class HdfsAsserts { + + private static final Logger LOG = Logger.getLogger(HdfsAsserts.class); + + /** + * Verify that mapreduce output (across all files) is as expected. + * + * @param directory Mapreduce output directory + * @param lines Expected lines + * @throws IOException + */ + public static void assertMapreduceOutput(String directory, String... lines) throws IOException { + Set<String> setLines = new HashSet<String>(Arrays.asList(lines)); + List<String> notFound = new LinkedList<String>(); + + String []files = HdfsUtils.getOutputMapreduceFiles(directory); + + for(String file : files) { + String filePath = directory + "/" + file; + BufferedReader br = new BufferedReader(new FileReader((filePath))); + + String line; + while ((line = br.readLine()) != null) { + if (!setLines.remove(line)) { + notFound.add(line); + } + } + br.close(); + } + + if(!setLines.isEmpty() || !notFound.isEmpty()) { + LOG.error("Expected lines that weren't present in the files:"); + LOG.error("\t" + StringUtils.join(setLines, "\n\t")); + LOG.error("Extra lines in files that weren't expected:"); + LOG.error("\t" + StringUtils.join(notFound, "\n\t")); + fail("Output do not match expectations."); + } + } + + + private HdfsAsserts() { + // Instantiation is prohibited + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java new file mode 100644 index 0000000..364f86d --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.asserts; + +import org.apache.sqoop.test.db.DatabaseProvider; +import org.apache.log4j.Logger; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Database provider related asserts. + */ +public class ProviderAsserts { + + private static final Logger LOG = Logger.getLogger(ProviderAsserts.class); + + /** + * Assert row in the table. + * + * @param provider Provider that should be used to query the database + * @param tableName Table name + * @param conditions Conditions for identifying the row + * @param values Values that should be present in the table + */ + public static void assertRow(DatabaseProvider provider, String tableName, Object []conditions, Object ...values) { + ResultSet rs = null; + try { + rs = provider.getRows(tableName, conditions); + + if(! rs.next()) { + fail("No rows found."); + } + + int i = 1; + for(Object expectedValue : values) { + Object actualValue = rs.getObject(i); + assertEquals("Columns do not match on position: " + i, expectedValue, actualValue); + i++; + } + + if(rs.next()) { + fail("Found more than one row."); + } + } catch (SQLException e) { + LOG.error("Unexpected SQLException", e); + fail("Unexpected SQLException: " + e); + } finally { + provider.closeResultSetWithStatement(rs); + } + } + + private ProviderAsserts() { + // Instantiation is prohibited + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/Cities.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/Cities.java b/test/src/main/java/org/apache/sqoop/test/data/Cities.java new file mode 100644 index 0000000..fb9be13 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/data/Cities.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.data; + +import org.apache.sqoop.test.db.DatabaseProvider; + +/** + * Simple listing of few world's cities to do basic sanity tests. + */ +public class Cities extends DataSet { + + public Cities(DatabaseProvider provider, String tableBaseName) { + super(provider, tableBaseName); + } + + @Override + public DataSet createTables() { + provider.createTable( + tableBaseName, + "id", + "id", "int", + "country", "varchar(50)", + "city", "varchar(50)" + ); + + return this; + } + + @Override + public DataSet loadBasicData() { + provider.insertRow(tableBaseName, 1, "USA", "San Francisco"); + provider.insertRow(tableBaseName, 2, "USA", "Sunnyvale"); + provider.insertRow(tableBaseName, 3, "Czech Republic", "Brno"); + provider.insertRow(tableBaseName, 4, "USA", "Palo Alto"); + + return this; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/DataSet.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java new file mode 100644 index 0000000..6999408 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.data; + +import org.apache.sqoop.test.db.DatabaseProvider; + +/** + * Abstract class for basic testing data sets. + * + * Each data set provides couple of generic methods that can be used to set up + * the tables and load example data. + */ +public abstract class DataSet { + + /** + * Database provider that will be used to populate the data. + */ + protected DatabaseProvider provider; + + /** + * Base name for created tables. + */ + protected String tableBaseName; + + public DataSet(DatabaseProvider provider, String tableBaseName) { + setProvider(provider); + setTableBaseName(tableBaseName); + } + + public DataSet setProvider(DatabaseProvider provider) { + this.provider = provider; + return this; + } + + public DataSet setTableBaseName(String tableBaseName) { + this.tableBaseName = tableBaseName; + return this; + } + + /** + * Crate all tables that this testing data set might need. + */ + public abstract DataSet createTables(); + + /** + * Load basic data. + * + * Basic data set should be small (around 10 rows) without any specialities. + */ + public abstract DataSet loadBasicData(); +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java new file mode 100644 index 0000000..6aeadd4 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.testcases; + +import org.apache.log4j.Logger; +import org.apache.sqoop.framework.configuration.OutputFormat; +import org.apache.sqoop.framework.configuration.StorageType; +import org.apache.sqoop.model.MConnection; +import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MPersistableEntity; +import org.apache.sqoop.test.asserts.ProviderAsserts; +import org.apache.sqoop.test.data.Cities; +import org.apache.sqoop.test.db.DatabaseProvider; +import org.apache.sqoop.test.db.DatabaseProviderFactory; +import org.apache.sqoop.validation.Status; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +/** + * Base test case suitable for connector testing. + * + * In addition to pure Tomcat based test case it will also create and initialize + * the database provider prior every test execution. + */ +abstract public class ConnectorTestCase extends TomcatTestCase { + + private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class); + + protected static DatabaseProvider provider; + + @BeforeClass + public static void startProvider() throws Exception { + provider = DatabaseProviderFactory.getProvider(System.getProperties()); + LOG.info("Starting database provider: " + provider.getClass().getName()); + provider.start(); + } + + @AfterClass + public static void stopProvider() { + LOG.info("Stopping database provider: " + provider.getClass().getName()); + provider.stop(); + } + + public String getTableName() { + return getClass().getSimpleName(); + } + + protected void createTable(String primaryKey, String ...columns) { + provider.createTable(getTableName(), primaryKey, columns); + } + + protected void dropTable() { + provider.dropTable(getTableName()); + } + + protected void insertRow(Object ...values) { + provider.insertRow(getTableName(), values); + } + + protected long rowCount() { + return provider.rowCount(getTableName()); + } + + /** + * Fill connection form based on currently active provider. + * + * @param connection MConnection object to fill + */ + protected void fillConnectionForm(MConnection connection) { + MFormList forms = connection.getConnectorPart(); + forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver()); + forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl()); + forms.getStringInput("connection.username").setValue(provider.getConnectionUsername()); + forms.getStringInput("connection.password").setValue(provider.getConnectionPassword()); + } + + /** + * Fill output form with specific storage and output type. Mapreduce output directory + * will be set to default test value. + * + * @param job MJOb object to fill + * @param storage Storage type that should be set + * @param output Output type that should be set + */ + protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) { + MFormList forms = job.getFrameworkPart(); + forms.getEnumInput("output.storageType").setValue(storage); + forms.getEnumInput("output.outputFormat").setValue(output); + forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory()); + } + + /** + * Fill input form. Mapreduce input directory will be set to default test value. + * + * @param job MJOb object to fill + */ + protected void fillInputForm(MJob job) { + MFormList forms = job.getFrameworkPart(); + forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory()); + } + + /** + * Create table cities. + */ + protected void createTableCities() { + new Cities(provider, getTableName()).createTables(); + } + + /** + * Create table cities and load few rows. + */ + protected void createAndLoadTableCities() { + new Cities(provider, getTableName()).createTables().loadBasicData(); + } + + /** + * Assert row in testing table. + * + * @param conditions Conditions in form that are expected by the database provider + * @param values Values that are expected in the table (with corresponding types) + */ + protected void assertRow(Object []conditions, Object ...values) { + ProviderAsserts.assertRow(provider, getTableName(), conditions, values); + } + + /** + * Assert row in table "cities". + * + * @param values Values that are expected + */ + protected void assertRowInCities(Object... values) { + assertRow(new Object[]{"id", values[0]}, values); + } + + /** + * Create connection. + * + * With asserts to make sure that it was created correctly. + * + * @param connection + */ + protected void createConnection(MConnection connection) { + assertEquals(Status.FINE, getClient().createConnection(connection)); + assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId()); + } + + /** + * Create job. + * + * With asserts to make sure that it was created correctly. + * + * @param job + */ + protected void createJob(MJob job) { + assertEquals(Status.FINE, getClient().createJob(job)); + assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId()); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java new file mode 100644 index 0000000..ca77e64 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.testcases; + +import org.apache.log4j.Logger; +import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.test.asserts.HdfsAsserts; +import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; +import org.apache.sqoop.test.utils.HdfsUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; + +import java.io.IOException; + +/** + * Basic test case that will bootstrap Sqoop server running in external Tomcat + * process. + */ +abstract public class TomcatTestCase { + + private static final Logger LOG = Logger.getLogger(TomcatTestCase.class); + + @Rule public TestName name = new TestName(); + + /** + * Temporary base path that will be used for tests. + * + * By default we will take a look for sqoop.integration.tmpdir property that is + * filled up by maven. If the test is not started from maven (IDE) we will + * pick up configured java.io.tmpdir value. The last results is /tmp/ directory + * in case that no property is set. + */ + private static final String TMP_PATH_BASE = + System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/"; + + /** + * Temporary directory that will be used by the test. + * + * We will take TMP_PATH_BASE and append two subdirectories. First will be named + * after fully qualified class name of current test class, second directory will + * be named after current test method name. For example: + * + * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/ + */ + private String tmpPath; + + /** + * Tomcat based Sqoop mini cluster + */ + private TomcatSqoopMiniCluster cluster; + + /** + * Sqoop client API. + */ + private SqoopClient client; + + @Before + public void startServer() throws Exception { + // Set up the temporary path + tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/"; + + // Set up and start server + cluster = new TomcatSqoopMiniCluster(getTemporaryPath()); + cluster.start(); + + // Initialize Sqoop Client API + client = new SqoopClient(getServerUrl()); + } + + @After + public void stopServer() throws Exception { + cluster.stop(); + } + + /** + * Return SqoopClient configured to talk to testing server. + * + * @return + */ + public SqoopClient getClient() { + return client; + } + + public String getTemporaryPath() { + return tmpPath; + } + + /** + * Return testing server URL + * + * @return + */ + public String getServerUrl() { + return cluster.getServerUrl(); + } + + /** + * Get input/output directory for mapreduce job. + * + * @return + */ + public String getMapreduceDirectory() { + return getTemporaryPath() + "/mapreduce-job-io"; + } + + /** + * Assert that mapreduce has generated following lines. + * + * As the lines can be spread between multiple files the ordering do not make + * a difference. + * + * @param lines + * @throws IOException + */ + protected void assertMapreduceOutput(String... lines) throws IOException { + HdfsAsserts.assertMapreduceOutput(getMapreduceDirectory(), lines); + } + + /** + * Create mapreduce input file with specified content. + * + * @param filename Input file name + * @param lines Individual lines that should be written into the file + * @throws IOException + */ + protected void createInputMapreduceFile(String filename, String...lines) throws IOException { + HdfsUtils.createFile(getMapreduceDirectory(), filename, lines); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java new file mode 100644 index 0000000..95dd177 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.utils; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; + +/** + * Handy utilities to work with HDFS + * + * TODO: This module will require clean up to work on MiniCluster/Real cluster. + */ +public class HdfsUtils { + + private static final Logger LOG = Logger.getLogger(HdfsUtils.class); + + /** + * Get list of mapreduce output files from given directory. + * + * @param directory Directory to be searched for files generated by MR + * @return + */ + public static String [] getOutputMapreduceFiles(String directory) { + File dir = new File(directory); + return dir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith("part-"); + } + }); + } + + /** + * Create HDFS file with given content. + * + * @param directory Directory where the file should be created + * @param filename File name + * @param lines Individual lines that should be written into the file + * @throws IOException + */ + public static void createFile(String directory, String filename, String ...lines) throws IOException { + File outputFile = new File(directory, filename); + FileUtils.writeLines(outputFile, Arrays.asList(lines)); + } + + private HdfsUtils() { + // Instantiation is not allowed + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java deleted file mode 100644 index fa2c2b4..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration; - -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; -import org.apache.commons.lang.StringUtils; -import org.apache.sqoop.client.SqoopClient; -import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestName; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FilenameFilter; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.fail; - -/** - * Basic test case that will bootstrap Sqoop server running in external Tomcat - * process. - */ -abstract public class TomcatTestCase { - - private static final Logger LOG = Logger.getLogger(TomcatTestCase.class); - - @Rule public TestName name = new TestName(); - - /** - * Temporary base path that will be used for tests. - * - * By default we will take a look for sqoop.integration.tmpdir property that is - * filled up by maven. If the test is not started from maven (IDE) we will - * pick up configured java.io.tmpdir value. The last results is /tmp/ directory - * in case that no property is set. - */ - private static final String TMP_PATH_BASE = - System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/"; - - /** - * Temporary directory that will be used by the test. - * - * We will take TMP_PATH_BASE and append two subdirectories. First will be named - * after fully qualified class name of current test class, second directory will - * be named after current test method name. For example: - * - * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/ - */ - private String tmpPath; - - /** - * Tomcat based Sqoop mini cluster - */ - private TomcatSqoopMiniCluster cluster; - - /** - * Sqoop client API. - */ - private SqoopClient client; - - @Before - public void startServer() throws Exception { - // Set up the temporary path - tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/"; - - // Set up and start server - cluster = new TomcatSqoopMiniCluster(getTemporaryPath()); - cluster.start(); - - // Initialize Sqoop Client API - client = new SqoopClient(getServerUrl()); - } - - @After - public void stopServer() throws Exception { - cluster.stop(); - } - - /** - * Return SqoopClient configured to talk to testing server. - * - * @return - */ - public SqoopClient getClient() { - return client; - } - - public String getTemporaryPath() { - return tmpPath; - } - - /** - * Return testing server URL - * - * @return - */ - public String getServerUrl() { - return cluster.getServerUrl(); - } - - /** - * Get input/output directory for mapreduce job. - * - * @return - */ - public String getMapreduceDirectory() { - return getTemporaryPath() + "/mapreduce-job-io"; - } - - /** - * Return list of file names that are outputs of mapreduce job. - * - * @return - */ - public String[] getOutputFilesMapreduce() { - File dir = new File(getMapreduceDirectory()); - return dir.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("part-"); - } - }); - } - - /** - * Assert that mapreduce has generated following lines. - * - * As the lines can be spread between multiple files the ordering do not make - * a difference. - * - * @param lines - * @throws IOException - */ - protected void assertMapreduceOutput(String... lines) throws IOException { - Set<String> setLines = new HashSet<String>(Arrays.asList(lines)); - List<String> notFound = new LinkedList<String>(); - - String []files = getOutputFilesMapreduce(); - - for(String file : files) { - String filePath = getMapreduceDirectory() + "/" + file; - BufferedReader br = new BufferedReader(new FileReader((filePath))); - - String line; - while ((line = br.readLine()) != null) { - if (!setLines.remove(line)) { - notFound.add(line); - } - } - br.close(); - } - - if(!setLines.isEmpty() || !notFound.isEmpty()) { - LOG.error("Expected lines that weren't present in the files:"); - LOG.error("\t" + StringUtils.join(setLines, "\n\t")); - LOG.error("Extra lines in files that weren't expected:"); - LOG.error("\t" + StringUtils.join(notFound, "\n\t")); - fail("Output do not match expectations."); - } - } - - /** - * Create mapreduce input file with specified content. - * - * @param filename Input file name - * @param lines Individual lines that should be written into the file - * @throws IOException - */ - protected void createInputMapreduceFile(String filename, String...lines) throws IOException { - File outputFile = new File(getMapreduceDirectory(), filename); - FileUtils.writeLines(outputFile, Arrays.asList(lines)); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java deleted file mode 100644 index d4e432d..0000000 --- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.integration.connector; - -import org.apache.log4j.Logger; -import org.apache.sqoop.framework.configuration.OutputFormat; -import org.apache.sqoop.framework.configuration.StorageType; -import org.apache.sqoop.integration.TomcatTestCase; -import org.apache.sqoop.model.MConnection; -import org.apache.sqoop.model.MFormList; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MPersistableEntity; -import org.apache.sqoop.test.db.DatabaseProvider; -import org.apache.sqoop.test.db.DatabaseProviderFactory; -import org.apache.sqoop.validation.Status; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.fail; - -/** - * Base test case for connector testing. - * - * It will create and initialize database provider prior every test execution. - */ -abstract public class ConnectorTestCase extends TomcatTestCase { - - private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class); - - protected static DatabaseProvider provider; - - @BeforeClass - public static void startProvider() throws Exception { - provider = DatabaseProviderFactory.getProvider(System.getProperties()); - LOG.info("Starting database provider: " + provider.getClass().getName()); - provider.start(); - } - - @AfterClass - public static void stopProvider() { - LOG.info("Stopping database provider: " + provider.getClass().getName()); - provider.stop(); - } - - public String getTableName() { - return getClass().getSimpleName(); - } - - protected void createTable(String primaryKey, String ...columns) { - provider.createTable(getTableName(), primaryKey, columns); - } - - protected void dropTable() { - provider.dropTable(getTableName()); - } - - protected void insertRow(Object ...values) { - provider.insertRow(getTableName(), values); - } - - protected long rowCount() { - return provider.rowCount(getTableName()); - } - - /** - * Fill connection form based on currently active provider. - * - * @param connection MConnection object to fill - */ - protected void fillConnectionForm(MConnection connection) { - MFormList forms = connection.getConnectorPart(); - forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver()); - forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl()); - forms.getStringInput("connection.username").setValue(provider.getConnectionUsername()); - forms.getStringInput("connection.password").setValue(provider.getConnectionPassword()); - } - - /** - * Fill output form with specific storage and output type. Mapreduce output directory - * will be set to default test value. - * - * @param job MJOb object to fill - * @param storage Storage type that should be set - * @param output Output type that should be set - */ - protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) { - MFormList forms = job.getFrameworkPart(); - forms.getEnumInput("output.storageType").setValue(storage); - forms.getEnumInput("output.outputFormat").setValue(output); - forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory()); - } - - /** - * Fill input form. Mapreduce input directory will be set to default test value. - * - * @param job MJOb object to fill - */ - protected void fillInputForm(MJob job) { - MFormList forms = job.getFrameworkPart(); - forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory()); - } - - /** - * Create table cities. - */ - protected void createTableCities() { - createTable("id", - "id", "int", - "country", "varchar(50)", - "city", "varchar(50)" - ); - } - - /** - * Create table cities and load few rows. - */ - protected void createAndLoadTableCities() { - createTableCities(); - insertRow(1, "USA", "San Francisco"); - insertRow(2, "USA", "Sunnyvale"); - insertRow(3, "Czech Republic", "Brno"); - insertRow(4, "USA", "Palo Alto"); - } - - /** - * Assert row in testing table. - * - * @param conditions Conditions in form that are expected by the database provider - * @param values Values that are expected in the table (with corresponding types) - */ - protected void assertRow(Object []conditions, Object ...values) { - ResultSet rs = provider.getRows(getTableName(), conditions); - - try { - if(! rs.next()) { - fail("No rows found."); - } - - int i = 1; - for(Object expectedValue : values) { - Object actualValue = rs.getObject(i); - assertEquals("Columns do not match on position: " + i, expectedValue, actualValue); - i++; - } - - if(rs.next()) { - fail("Found more than one row."); - } - } catch (SQLException e) { - LOG.error("Unexpected SQLException", e); - fail("Unexpected SQLException: " + e); - } finally { - provider.closeResultSetWithStatement(rs); - } - } - - /** - * Assert row in table "cities". - * - * @param values Values that are expected - */ - protected void assertRowInCitiesTable(Object ... values) { - assertRow(new Object[]{"id", values[0]}, values); - } - - /** - * Create connection. - * - * With asserts to make sure that it was created correctly. - * - * @param connection - */ - protected void createConnection(MConnection connection) { - assertEquals(Status.FINE, getClient().createConnection(connection)); - assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId()); - } - - /** - * Create job. - * - * With asserts to make sure that it was created correctly. - * - * @param job - */ - protected void createJob(MJob job) { - assertEquals(Status.FINE, getClient().createJob(job)); - assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId()); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java index b920655..fbf46e9 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java @@ -18,7 +18,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import org.apache.log4j.Logger; -import org.apache.sqoop.integration.connector.ConnectorTestCase; +import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; @@ -70,10 +70,10 @@ public class TableExportTest extends ConnectorTestCase { } while(submission.getStatus().isRunning()); assertEquals(4L, rowCount()); - assertRowInCitiesTable(1, "USA", "San Francisco"); - assertRowInCitiesTable(2, "USA", "Sunnyvale"); - assertRowInCitiesTable(3, "Czech Republic", "Brno"); - assertRowInCitiesTable(4, "USA", "Palo Alto"); + assertRowInCities(1, "USA", "San Francisco"); + assertRowInCities(2, "USA", "Sunnyvale"); + assertRowInCities(3, "Czech Republic", "Brno"); + assertRowInCities(4, "USA", "Palo Alto"); // Clean up testing table dropTable(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java index adcfbaf..639d9ad 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java @@ -20,7 +20,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import org.apache.log4j.Logger; import org.apache.sqoop.framework.configuration.OutputFormat; import org.apache.sqoop.framework.configuration.StorageType; -import org.apache.sqoop.integration.connector.ConnectorTestCase; +import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java index 0f48a8b..cea24b9 100644 --- a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java @@ -19,7 +19,7 @@ package org.apache.sqoop.integration.server; import org.apache.sqoop.client.request.VersionRequest; import org.apache.sqoop.common.VersionInfo; -import org.apache.sqoop.integration.TomcatTestCase; +import org.apache.sqoop.test.testcases.TomcatTestCase; import org.apache.sqoop.json.VersionBean; import org.junit.Test;