http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/CubridManagerImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/CubridManagerImportTest.java b/src/test/com/cloudera/sqoop/manager/CubridManagerImportTest.java deleted file mode 100644 index 03763ca..0000000 --- a/src/test/com/cloudera/sqoop/manager/CubridManagerImportTest.java +++ /dev/null @@ -1,296 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.sqoop.manager.CubridManager; -import org.apache.sqoop.manager.cubrid.CubridTestUtils; -import org.apache.sqoop.util.FileListing; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.testutil.ImportJobTestCase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test the CubridManager implementation. - * - * This uses JDBC to import data from an Cubrid database into HDFS. - * - * Since this requires an Cubrid installation, this class is named in such a way - * that Sqoop's default QA process does not run it. You need to run this - * manually with -Dtestcase=CubridManagerImportTest. - * - * You need to put Cubrid JDBC driver library (JDBC-9.2.2.0003-cubrid.jar) in a - * location where Sqoop will be able to access it (since this library cannot be - * checked into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install Cubrid 9.2.2 - * ref:http://www.cubrid.org/wiki_tutorials/entry/installing-cubrid-on-linux-using-shell-and-rpm - * Create a database SQOOPCUBRIDTEST - * $cubrid createdb SQOOPCUBRIDTEST en_us.utf8 - * Start cubrid and database - * $cubrid service start - * $cubrid server start SQOOPCUBRIDTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * $csql -u dba SQOOPCUBRIDTEST - * csql>CREATE USER SQOOPUSER password 'PASSWORD'; - */ -public class CubridManagerImportTest extends ImportJobTestCase { - - public static final Log LOG = LogFactory.getLog( - CubridManagerImportTest.class.getName()); - - static final String TABLE_NAME = "employees_cubrid"; - static final String NULL_TABLE_NAME = "null_employees_cubrid"; - - // instance variables populated during setUp, used during tests - private CubridManager manager; - - private Configuration conf = new Configuration(); - - @Override - protected Configuration getConf() { - return conf; - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Before - public void setUp() { - super.setUp(); - LOG.debug("Setting up another CubridImport test: " - + CubridTestUtils.getConnectString()); - setUpData(TABLE_NAME, false); - setUpData(NULL_TABLE_NAME, true); - LOG.debug("setUp complete."); - } - - public void setUpData(String tableName, boolean nullEntry) { - SqoopOptions options = new SqoopOptions( - CubridTestUtils.getConnectString(), tableName); - options.setUsername(CubridTestUtils.getCurrentUser()); - options.setPassword(CubridTestUtils.getPassword()); - - LOG.debug("Setting up another CubridImport test: " - + CubridTestUtils.getConnectString()); - - manager = new CubridManager(options); - - Connection connection = null; - Statement st = null; - - try { - connection = manager.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create the database table and populate it with data. - st.executeUpdate("DROP TABLE IF EXISTS " + tableName); - st.executeUpdate("CREATE TABLE " + tableName + " (" - + manager.escapeColName("id") - + " INT NOT NULL PRIMARY KEY, " - + manager.escapeColName("name") - + " VARCHAR(24) NOT NULL, " - + manager.escapeColName("start_date") + " DATE, " - + manager.escapeColName("Salary") + " FLOAT, " - + manager.escapeColName("dept") - + " VARCHAR(32));"); - - st.executeUpdate("INSERT INTO " + tableName - + " VALUES(1,'Aaron','2009-05-14'," - + "1000000.00,'engineering');"); - st.executeUpdate("INSERT INTO " + tableName - + " VALUES(2,'Bob','2009-04-20',400.00,'sales');"); - st.executeUpdate("INSERT INTO " + tableName - + " VALUES(3,'Fred','2009-01-23'," - + "15.00,'marketing');"); - if (nullEntry) { - st.executeUpdate("INSERT INTO " + tableName - + " VALUES(4,'Mike',NULL,NULL,NULL);"); - } - - connection.commit(); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: " + sqlE); - sqlE.printStackTrace(); - fail("SQLException when running test setUp(): " + sqlE); - } finally { - try { - if (null != st) { - st.close(); - } - - if (null != connection) { - connection.close(); - } - } catch (SQLException sqlE) { - LOG.warn("Got SQLException when closing connection: " - + sqlE); - } - } - } - - @After - public void tearDown() { - super.tearDown(); - try { - manager.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException: " + sqlE.toString()); - fail("Got SQLException: " + sqlE.toString()); - } - } - - @Test - public void testImportSimple() throws IOException { - String[] expectedResults = { - "1,Aaron,2009-05-14,1000000.0,engineering", - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", }; - - doImportAndVerify(TABLE_NAME, expectedResults); - } - - @Test - public void testListTables() throws IOException { - SqoopOptions options = new SqoopOptions(new Configuration()); - options.setConnectString(CubridTestUtils.getConnectString()); - options.setUsername(CubridTestUtils.getCurrentUser()); - options.setPassword(CubridTestUtils.getPassword()); - - ConnManager mgr = new CubridManager(options); - String[] tables = mgr.listTables(); - Arrays.sort(tables); - assertTrue(TABLE_NAME + " is not found!", - Arrays.binarySearch(tables, TABLE_NAME) >= 0); - } - - @Test - public void testNullEscapeCharacters() throws Exception { - String[] expectedResults = { - "1,Aaron,2009-05-14,1000000.0,engineering", - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", - "4,Mike,cubrid,cubrid,cubrid", }; - - String[] extraArgs = { - "--null-string", - "cubrid", - "--null-non-string", - "cubrid", }; - - doImportAndVerify(NULL_TABLE_NAME, expectedResults, extraArgs); - } - - private void doImportAndVerify(String tableName, - String[] expectedResults, - String... extraArgs) throws IOException { - Path warehousePath = new Path(this.getWarehouseDir()); - Path tablePath = new Path(warehousePath, tableName); - Path filePath = new Path(tablePath, "part-m-00000"); - - File tableFile = new File(tablePath.toString()); - if (tableFile.exists() && tableFile.isDirectory()) { - // remove the directory before running the import. - FileListing.recursiveDeleteDir(tableFile); - } - - String[] argv = getArgv(tableName, extraArgs); - try { - runImport(argv); - } catch (IOException ioe) { - LOG.error("Got IOException during import: " - + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } - - File f = new File(filePath.toString()); - assertTrue("Could not find imported data file", f.exists()); - BufferedReader r = null; - try { - // Read through the file and make sure it's all there. - r = new BufferedReader(new InputStreamReader( - new FileInputStream(f))); - for (String expectedLine : expectedResults) { - assertEquals(expectedLine, r.readLine()); - } - } catch (IOException ioe) { - LOG.error("Got IOException verifying results: " - + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } finally { - IOUtils.closeStream(r); - } - } - - private String[] getArgv(String tableName, String... extraArgs) { - ArrayList<String> args = new ArrayList<String>(); - - CommonArgs.addHadoopFlags(args); - - args.add("--table"); - args.add(tableName); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(CubridTestUtils.getConnectString()); - args.add("--username"); - args.add(CubridTestUtils.getCurrentUser()); - args.add("--password"); - args.add(CubridTestUtils.getPassword()); - args.add("--num-mappers"); - args.add("1"); - - if (extraArgs.length > 0) { - for (String arg : extraArgs) { - args.add(arg); - } - } - - return args.toArray(new String[0]); - } -}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/DB2ManagerImportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DB2ManagerImportManualTest.java b/src/test/com/cloudera/sqoop/manager/DB2ManagerImportManualTest.java deleted file mode 100644 index 2bc5c54..0000000 --- a/src/test/com/cloudera/sqoop/manager/DB2ManagerImportManualTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.cloudera.sqoop.manager; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.testutil.ImportJobTestCase; -import com.cloudera.sqoop.util.FileListing; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test the DB2Manager implementation. - * - * This uses JDBC to import data from an DB2 database into HDFS. - * - * Since this requires an DB2 Server installation, - * this class is named in such a way that Sqoop's default QA process does - * not run it. You need to run this manually with - * -Dtestcase=DB2ManagerImportManualTest - * - * You need to put DB2 JDBC driver library (db2jcc4.jar) in a location - * where Sqoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install DB2 Express 9.7 C server. - * Create a database SQOOP - * Create a login SQOOP with password PASSWORD and grant all - * access for database SQOOP to user SQOOP. - */ -public class DB2ManagerImportManualTest extends ImportJobTestCase { - - public static final Log LOG = LogFactory.getLog( - DB2ManagerImportManualTest.class.getName()); - - static final String HOST_URL = System.getProperty( - "sqoop.test.db2.connectstring.host_url", - "jdbc:db2://db2host:50000"); - - static final String DATABASE_NAME = System.getProperty( - "sqoop.test.db2.connectstring.database", - "SQOOP"); - static final String DATABASE_USER = System.getProperty( - "sqoop.test.db2.connectstring.username", - "SQOOP"); - static final String DATABASE_PASSWORD = System.getProperty( - "sqoop.test.db2.connectstring.password", - "SQOOP"); - static final String TABLE_NAME = "EMPLOYEES_DB2"; - static final String QUALIFIED_TABLE_NAME = DATABASE_USER + ".EMPLOYEES_DB2"; - static final String CONNECT_STRING = HOST_URL - + "/" + DATABASE_NAME - + ":currentSchema=" + DATABASE_USER +";"; - - static { - LOG.info("Using DB2 CONNECT_STRING: " + CONNECT_STRING); - } - - // instance variables populated during setUp, used during tests - private Db2Manager manager; - private boolean useQualifiedTableName; - private boolean useDefaultConnectManager; - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getTableName() { - return useQualifiedTableName ? QUALIFIED_TABLE_NAME : TABLE_NAME; - } - - @Before - public void setUp() { - super.setUp(); - - SqoopOptions options = new SqoopOptions(CONNECT_STRING, getTableName()); - options.setUsername(DATABASE_USER); - options.setPassword(DATABASE_PASSWORD); - - manager = new Db2Manager(options); - - // Drop the existing table, if there is one. - Connection conn = null; - Statement stmt = null; - try { - conn = manager.getConnection(); - stmt = conn.createStatement(); - stmt.execute("DROP TABLE " + getTableName()); - } catch (SQLException sqlE) { - LOG.info("Table was not dropped: " + sqlE.getMessage()); - } finally { - try { - if (null != stmt) { - stmt.close(); - } - } catch (Exception ex) { - LOG.warn("Exception while closing stmt", ex); - } - } - - // Create and populate table - try { - conn = manager.getConnection(); - conn.setAutoCommit(false); - stmt = conn.createStatement(); - - // create the database table and populate it with data. - stmt.executeUpdate("CREATE TABLE " + getTableName() + " (" - + "id INT NOT NULL, " - + "name VARCHAR(24) NOT NULL, " - + "salary FLOAT, " - + "dept VARCHAR(32), " - + "PRIMARY KEY (id))"); - - stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "1,'Aaron', " - + "1000000.00,'engineering')"); - stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "2,'Bob', " - + "400.00,'sales')"); - stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "3,'Fred', 15.00," - + "'marketing')"); - conn.commit(); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: ", sqlE); - sqlE.printStackTrace(); - fail("SQLException when running test setUp(): " + sqlE); - } finally { - try { - if (null != stmt) { - stmt.close(); - } - } catch (Exception ex) { - LOG.warn("Exception while closing connection/stmt", ex); - } - } - } - - @After - public void tearDown() { - super.tearDown(); - try { - manager.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException: " + sqlE.toString()); - fail("Got SQLException: " + sqlE.toString()); - } - } - - @Test - public void testDb2Import() throws IOException { - useQualifiedTableName = false; - - // Verify that GenericJdbcManager works. - useDefaultConnectManager = true; - runDb2Test(getExpectedResults()); - - // Verify that Db2Manager works. - useDefaultConnectManager = false; - runDb2Test(getExpectedResults()); - } - - @Test - public void testDb2ImportQualifiedTableName() throws IOException { - useQualifiedTableName = true; - - // Verify that GenericJdbcManager works. - useDefaultConnectManager = true; - runDb2Test(getExpectedResults()); - - // Verify that Db2Manager works. - useDefaultConnectManager = false; - runDb2Test(getExpectedResults()); - } - - private String [] getExpectedResults() { - return new String [] { - "1,Aaron,1000000.0,engineering", - "2,Bob,400.0,sales", - "3,Fred,15.0,marketing", - }; - } - - private String [] getArgv() { - ArrayList<String> args = new ArrayList<String>(); - - CommonArgs.addHadoopFlags(args); - - args.add("--table"); - args.add(getTableName()); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(CONNECT_STRING); - args.add("--username"); - args.add(DATABASE_USER); - args.add("--password"); - args.add(DATABASE_PASSWORD); - args.add("--num-mappers"); - args.add("1"); - - if (useDefaultConnectManager) { - // Specifying the driver class forces DefaultManagerFactory - // to instantiate GenericJdbcManager. - args.add("--driver"); - args.add("com.ibm.db2.jcc.DB2Driver"); - } - - return args.toArray(new String[0]); - } - - private void runDb2Test(String [] expectedResults) throws IOException { - - Path warehousePath = new Path(this.getWarehouseDir()); - Path tablePath = new Path(warehousePath, getTableName()); - Path filePath = new Path(tablePath, "part-m-00000"); - - File tableFile = new File(tablePath.toString()); - if (tableFile.exists() && tableFile.isDirectory()) { - // remove the directory before running the import. - FileListing.recursiveDeleteDir(tableFile); - } - - String [] argv = getArgv(); - try { - runImport(argv); - } catch (IOException ioe) { - LOG.error("Got IOException during import: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } - - File f = new File(filePath.toString()); - assertTrue("Could not find imported data file", f.exists()); - BufferedReader r = null; - try { - // Read through the file and make sure it's all there. - r = new BufferedReader(new InputStreamReader(new FileInputStream(f))); - for (String expectedLine : expectedResults) { - assertEquals(expectedLine, r.readLine()); - } - } catch (IOException ioe) { - LOG.error("Got IOException verifying results: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } finally { - IOUtils.closeStream(r); - } - } - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java deleted file mode 100644 index 9a48788..0000000 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java +++ /dev/null @@ -1,363 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Before; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.TestExport; -import com.cloudera.sqoop.mapreduce.MySQLExportMapper; -import org.junit.Ignore; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test the DirectMySQLManager implementation's exportJob() functionality. - */ -public class DirectMySQLExportTest extends TestExport { - - public static final Log LOG = LogFactory.getLog( - DirectMySQLExportTest.class.getName()); - - static final String TABLE_PREFIX = "EXPORT_MYSQL_"; - - // instance variables populated during setUp, used during tests. - private DirectMySQLManager manager; - private Connection conn; - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected Connection getConnection() { - return conn; - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return mySQLTestUtils.getMySqlConnectString(); - } - - @Override - protected String getTablePrefix() { - return TABLE_PREFIX; - } - - @Override - protected String getDropTableStatement(String tableName) { - return "DROP TABLE IF EXISTS " + tableName; - } - - @Before - public void setUp() { - super.setUp(); - - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(options); - this.manager = new DirectMySQLManager(options); - - try { - this.conn = manager.getConnection(); - this.conn.setAutoCommit(false); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: " + sqlE); - sqlE.printStackTrace(); - fail("SQLException when running test setUp(): " + sqlE); - } - } - - @After - public void tearDown() { - try { - Statement stmt = conn.createStatement(); - stmt.execute(getDropTableStatement(getTableName())); - } catch(SQLException e) { - LOG.error("Can't clean up the database:", e); - } - - super.tearDown(); - - if (null != this.conn) { - try { - this.conn.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException closing conn: " + sqlE.toString()); - } - } - } - - @Override - protected String [] getCodeGenArgv(String... extraArgs) { - return super.getCodeGenArgv(mySQLTestUtils.addUserNameAndPasswordToArgs(extraArgs)); - } - - @Override - protected String [] getArgv(boolean includeHadoopFlags, - int rowsPerStatement, int statementsPerTx, String... additionalArgv) { - - String [] subArgv = newStrArray(mySQLTestUtils.addUserNameAndPasswordToArgs(additionalArgv),"--direct"); - return super.getArgv(includeHadoopFlags, rowsPerStatement, - statementsPerTx, subArgv); - } - - /** - * Test a single mapper that runs several transactions serially. - */ - @Test - public void testMultiTxExport() throws IOException, SQLException { - multiFileTest(1, 20, 1, - "-D", MySQLExportMapper.MYSQL_CHECKPOINT_BYTES_KEY + "=10"); - } - - /** - * Test an authenticated export using mysqlimport. - */ - @Test - public void testAuthExport() throws IOException, SQLException { - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySQLTestUtils.getUserName()); - options.setPassword(mySQLTestUtils.getUserPass()); - - manager = new DirectMySQLManager(options); - - Connection connection = null; - Statement st = null; - - String tableName = getTableName(); - - try { - connection = manager.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create a target database table. - st.executeUpdate("DROP TABLE IF EXISTS " + tableName); - st.executeUpdate("CREATE TABLE " + tableName + " (" - + "id INT NOT NULL PRIMARY KEY, " - + "msg VARCHAR(24) NOT NULL)"); - connection.commit(); - - // Write a file containing a record to export. - Path tablePath = getTablePath(); - Path filePath = new Path(tablePath, "datafile"); - Configuration conf = new Configuration(); - conf.set("fs.default.name", "file:///"); - - FileSystem fs = FileSystem.get(conf); - fs.mkdirs(tablePath); - OutputStream os = fs.create(filePath); - BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os)); - w.write(getRecordLine(0)); - w.write(getRecordLine(1)); - w.write(getRecordLine(2)); - w.close(); - os.close(); - - // run the export and verify that the results are good. - runExport(getArgv(true, 10, 10, - "--username", mySQLTestUtils.getUserName(), - "--password", mySQLTestUtils.getUserPass(), - "--connect", mySQLTestUtils.getMySqlConnectString())); - verifyExport(3, connection); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: " + sqlE); - sqlE.printStackTrace(); - fail("SQLException when accessing target table. " + sqlE); - } finally { - try { - if (null != st) { - st.close(); - } - - if (null != connection) { - connection.close(); - } - } catch (SQLException sqlE) { - LOG.warn("Got SQLException when closing connection: " + sqlE); - } - } - } - - /** - * Test an authenticated export using mysqlimport. - */ - @Test - public void testEscapedByExport() throws IOException, SQLException { - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySQLTestUtils.getUserName()); - options.setPassword(mySQLTestUtils.getUserPass()); - - manager = new DirectMySQLManager(options); - - Connection connection = null; - Statement st = null; - - String tableName = getTableName(); - - try { - connection = manager.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create a target database table. - st.executeUpdate("DROP TABLE IF EXISTS " + tableName); - st.executeUpdate("CREATE TABLE " + tableName + " (" - + "id INT NOT NULL PRIMARY KEY, " - + "msg VARCHAR(24) NOT NULL, " - + "value VARCHAR(100) NOT NULL)"); - connection.commit(); - - // Write a file containing a record to export. - Path tablePath = getTablePath(); - Path filePath = new Path(tablePath, "datafile"); - Configuration conf = new Configuration(); - conf.set("fs.default.name", "file:///"); - - ColumnGenerator gen = new ColumnGenerator() { - public String getExportText(int rowNum) { - return "||" + rowNum; - } - public String getVerifyText(int rowNum) { - return "|" + rowNum; - } - public String getType() { - return "STRING"; - } - }; - - FileSystem fs = FileSystem.get(conf); - fs.mkdirs(tablePath); - OutputStream os = fs.create(filePath); - BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os)); - w.write(getRecordLine(0, gen)); - w.write(getRecordLine(1, gen)); - w.write(getRecordLine(2, gen)); - w.close(); - os.close(); - - // run the export and verify that the results are good. - runExport(getArgv(true, 10, 10, - "--username", mySQLTestUtils.getUserName(), - "--password", mySQLTestUtils.getUserPass(), - "--connect", mySQLTestUtils.getMySqlConnectString(), - "--escaped-by", "|")); - verifyExport(3, connection); - verifyTableColumnContents(connection, tableName, "value", gen); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: " + sqlE); - sqlE.printStackTrace(); - fail("SQLException when accessing target table. " + sqlE); - } finally { - try { - if (null != st) { - st.close(); - } - - if (null != connection) { - connection.close(); - } - } catch (SQLException sqlE) { - LOG.warn("Got SQLException when closing connection: " + sqlE); - } - } - } - - @Test(expected = IOException.class) - public void testExportInputNullStringFailsValidate() throws IOException { - runExport(getArgv(true, 10, 10, - "--username", mySQLTestUtils.getUserName(), - "--password", mySQLTestUtils.getUserPass(), - "--connect", mySQLTestUtils.getMySqlConnectString(), - "--input-null-string", "null")); - } - - @Test(expected = IOException.class) - public void testExportInputNullNonStringFailsValidate() throws IOException { - runExport(getArgv(true, 10, 10, - "--username", mySQLTestUtils.getUserName(), - "--password", mySQLTestUtils.getUserPass(), - "--connect", mySQLTestUtils.getMySqlConnectString(), - "--input-null-non-string", "null")); - } - - @Ignore("Ignoring this test as staging is not supported in direct mode.") - @Override - @Test - public void testMultiMapTextExportWithStaging() - throws IOException, SQLException { - } - - @Ignore("Ignoring this test as staging is not supported in direct mode.") - @Override - @Test - public void testMultiTransactionWithStaging() - throws IOException, SQLException { - } - - @Ignore("Ignoring this test as --input-null-non-string is not supported in direct mode.") - @Override - @Test - public void testLessColumnsInFileThanInTableInputNullIntPassed() throws IOException, SQLException { - } - - @Ignore("Ignoring this test as --input-null-string is not supported in direct mode.") - @Override - @Test - public void testLessColumnsInFileThanInTableInputNullStringPassed() throws IOException, SQLException { - } - - private void verifyTableColumnContents(Connection connection, - String table, String column, ColumnGenerator gen) - throws IOException, SQLException { - Statement st = connection.createStatement(); - - // create a target database table. - assertTrue(st.execute("SELECT " + column + " FROM " + table)); - ResultSet rs = st.getResultSet(); - - for (int row = 0; rs.next(); ++row) { - assertEquals(gen.getVerifyText(row), rs.getString(1)); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java deleted file mode 100644 index 247ce0b..0000000 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java +++ /dev/null @@ -1,422 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.FileInputStream; -import java.io.File; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.testutil.ImportJobTestCase; -import com.cloudera.sqoop.util.FileListing; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test the DirectMySQLManager implementation. - * This differs from MySQLManager only in its importTable() method, which - * uses mysqldump instead of mapreduce+DBInputFormat. - * - * Since this requires a MySQL installation on your local machine to use, this - * class is named in such a way that Hadoop's default QA process does not run - * it. You need to run this manually with -Dtestcase=DirectMySQLTest. - * - * You need to put MySQL's Connector/J JDBC driver library into a location - * where Hadoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * You should also create a database named 'sqooptestdb' and authorize yourself: - * - * CREATE DATABASE sqooptestdb; - * use mysql; - * GRANT ALL PRIVILEGES ON sqooptestdb.* TO 'yourusername'@'localhost'; - * flush privileges; - * - */ -public class DirectMySQLTest extends ImportJobTestCase { - - public static final Log LOG = LogFactory.getLog( - DirectMySQLTest.class.getName()); - - static final String TABLE_PREFIX = "EMPLOYEES_MYSQL_"; - - // instance variables populated during setUp, used during tests - private DirectMySQLManager manager; - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected String getTablePrefix() { - return TABLE_PREFIX; - } - - @Before - public void setUp() { - super.setUp(); - - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(options); - - LOG.debug("Setting up another DirectMySQLTest: " - + mySQLTestUtils.getMySqlConnectString()); - - manager = new DirectMySQLManager(options); - - Connection connection = null; - Statement st = null; - - try { - connection = manager.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create the database table and populate it with data. - st.executeUpdate("DROP TABLE IF EXISTS " + getTableName()); - st.executeUpdate("CREATE TABLE " + getTableName() + " (" - + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " - + "name VARCHAR(24) NOT NULL, " - + "overly_large_number INT UNSIGNED," - + "start_date DATE, " - + "salary FLOAT, " - + "dept VARCHAR(32))"); - - st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')"); - st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Bob',100,'2009-04-20',400.00,'sales')"); - st.executeUpdate("INSERT INTO " + getTableName() + " VALUES(" - + "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')"); - connection.commit(); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: " + sqlE); - sqlE.printStackTrace(); - fail("SQLException when running test setUp(): " + sqlE); - } - } - - @After - public void tearDown() { - try { - Statement stmt = manager.getConnection().createStatement(); - stmt.execute("DROP TABLE " + getTableName()); - } catch(SQLException e) { - LOG.error("Can't clean up the database:", e); - } - - super.tearDown(); - } - - private String [] getArgv(boolean mysqlOutputDelims, boolean isDirect, - String tableName, String... extraArgs) { - ArrayList<String> args = new ArrayList<String>(); - - CommonArgs.addHadoopFlags(args); - - args.add("--table"); - args.add(tableName); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(mySQLTestUtils.getMySqlConnectString()); - if (isDirect) { - args.add("--direct"); - } - args.add("--username"); - args.add(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(args); - args.add("--where"); - args.add("id > 1"); - args.add("--num-mappers"); - args.add("1"); - - if (mysqlOutputDelims) { - args.add("--mysql-delimiters"); - } - - if (null != extraArgs) { - for (String arg : extraArgs) { - args.add(arg); - } - } - - return args.toArray(new String[0]); - } - - private void doImport(boolean mysqlOutputDelims, boolean isDirect, - String tableName, String [] expectedResults, String [] extraArgs) - throws IOException { - - Path warehousePath = new Path(this.getWarehouseDir()); - Path tablePath = new Path(warehousePath, tableName); - - Path filePath = new Path(tablePath, "part-m-00000"); - - File tableFile = new File(tablePath.toString()); - if (tableFile.exists() && tableFile.isDirectory()) { - // remove the directory before running the import. - FileListing.recursiveDeleteDir(tableFile); - } - - String [] argv = getArgv(mysqlOutputDelims, isDirect, tableName, extraArgs); - runImport(argv); - - File f = new File(filePath.toString()); - assertTrue("Could not find imported data file: " + f, f.exists()); - BufferedReader r = null; - try { - // Read through the file and make sure it's all there. - r = new BufferedReader(new InputStreamReader(new FileInputStream(f))); - for (String expectedLine : expectedResults) { - assertEquals(expectedLine, r.readLine()); - } - } catch (IOException ioe) { - LOG.error("Got IOException verifying results: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } finally { - IOUtils.closeStream(r); - } - } - - @Test - public void testDirectBulkImportWithDefaultDelims() throws IOException { - // no quoting of strings allowed. - String [] expectedResults = { - "2,Bob,100,2009-04-20,400,sales", - "3,Fred,4000000000,2009-01-23,15,marketing", - }; - - doImport(false, true, getTableName(), expectedResults, null); - } - - @Test - public void testWithExtraParams() throws IOException { - // no quoting of strings allowed. - String [] expectedResults = { - "2,Bob,100,2009-04-20,400,sales", - "3,Fred,4000000000,2009-01-23,15,marketing", - }; - - String [] extraArgs = { "--", "--lock-tables" }; - - doImport(false, true, getTableName(), expectedResults, extraArgs); - } - - @Test - public void testMultiMappers() throws IOException { - // no quoting of strings allowed. - String [] expectedResults = { - "2,Bob,100,2009-04-20,400,sales", - "3,Fred,4000000000,2009-01-23,15,marketing", - }; - - String [] extraArgs = { "-m", "2" }; - - doImport(false, true, getTableName(), expectedResults, extraArgs); - } - - @Test - public void testJdbcColumnSubset() throws IOException { - // Test that column subsets work in JDBC mode. - LOG.info("Starting JDBC Column Subset test."); - - String [] expectedResults = { - "2,Bob,400.0", - "3,Fred,15.0", - }; - - String [] extraArgs = { "--columns", "id,name,salary" }; - doImport(false, false, getTableName(), expectedResults, extraArgs); - } - - @Test - public void testDirectColumnSubset() throws IOException { - // Using a column subset should actually force direct mode off, but this - // should just warn the user and do a normal import. - LOG.info("Starting Direct Column Subset test."); - - String [] expectedResults = { - "2,Bob,400.0", - "3,Fred,15.0", - }; - - String [] extraArgs = { "--columns", "id,name,salary" }; - doImport(false, true, getTableName(), expectedResults, extraArgs); - } - - @Test - public void testDirectBulkImportWithMySQLQuotes() throws IOException { - // mysql quotes all string-based output. - String [] expectedResults = { - "2,'Bob',100,'2009-04-20',400,'sales'", - "3,'Fred',4000000000,'2009-01-23',15,'marketing'", - }; - - doImport(true, true, getTableName(), expectedResults, null); - } - - @Test - public void testMySQLJdbcImport() throws IOException { - String [] expectedResults = { - "2,Bob,100,2009-04-20,400.0,sales", - "3,Fred,4000000000,2009-01-23,15.0,marketing", - }; - - doImport(false, false, getTableName(), expectedResults, null); - } - - @Test - public void testJdbcEscapedTableName() throws Exception { - // Test a JDBC-based import of a table whose name is - // a reserved sql keyword (and is thus `quoted`) - final String RESERVED_TABLE_NAME = "TABLE"; - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - RESERVED_TABLE_NAME); - options.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(options); - ConnManager mgr = new MySQLManager(options); - - Connection connection = null; - Statement st = null; - - try { - connection = mgr.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create the database table and populate it with data. - st.executeUpdate("DROP TABLE IF EXISTS `" + RESERVED_TABLE_NAME + "`"); - st.executeUpdate("CREATE TABLE `" + RESERVED_TABLE_NAME + "` (" - + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " - + "name VARCHAR(24) NOT NULL, " - + "start_date DATE, " - + "salary FLOAT, " - + "dept VARCHAR(32))"); - - st.executeUpdate("INSERT INTO `" + RESERVED_TABLE_NAME + "` VALUES(" - + "2,'Aaron','2009-05-14',1000000.00,'engineering')"); - st.close(); - connection.commit(); - - String [] expectedResults = { - "2,Aaron,2009-05-14,1000000.0,engineering", - }; - - doImport(false, false, RESERVED_TABLE_NAME, expectedResults, null); - - st = connection.createStatement(); - st.execute("DROP TABLE `" + RESERVED_TABLE_NAME + "`"); - } finally { - if (null != st) { - st.close(); - } - - if (null != connection) { - connection.close(); - } - } - - } - - @Test(expected = IOException.class) - public void testSqoopNullStringValueFailsValidate() throws Exception { - String [] expectedResults = {}; - String [] extraArgs = {"--null-string", "abc"}; - - doImport(false, true, getTableName(), expectedResults, extraArgs); - } - - @Test(expected = IOException.class) - public void testSqoopNullNonStringValueFailsValidate() throws Exception { - String [] expectedResults = {}; - String [] extraArgs = {"--null-non-string", "abc"}; - - doImport(false, true, getTableName(), expectedResults, extraArgs); - } - - @Test - public void testJdbcEscapedColumnName() throws Exception { - // Test a JDBC-based import of a table with a column whose name is - // a reserved sql keyword (and is thus `quoted`). - final String TABLE_NAME = "mysql_escaped_col_table"; - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - TABLE_NAME); - options.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(options); - ConnManager mgr = new MySQLManager(options); - - Connection connection = null; - Statement st = null; - - try { - connection = mgr.getConnection(); - connection.setAutoCommit(false); - st = connection.createStatement(); - - // create the database table and populate it with data. - st.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME); - st.executeUpdate("CREATE TABLE " + TABLE_NAME + " (" - + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " - + "`table` VARCHAR(24) NOT NULL, " - + "`CREATE` DATE, " - + "salary FLOAT, " - + "dept VARCHAR(32))"); - - st.executeUpdate("INSERT INTO " + TABLE_NAME + " VALUES(" - + "2,'Aaron','2009-05-14',1000000.00,'engineering')"); - st.close(); - connection.commit(); - - String [] expectedResults = { - "2,Aaron,2009-05-14,1000000.0,engineering", - }; - - doImport(false, false, TABLE_NAME, expectedResults, null); - - st = connection.createStatement(); - st.execute("DROP TABLE " + TABLE_NAME); - } finally { - if (null != st) { - st.close(); - } - - if (null != connection) { - connection.close(); - } - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java b/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java deleted file mode 100644 index b7c7416..0000000 --- a/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.PreparedStatement; -import java.util.Arrays; -import java.util.ArrayList; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.mapred.JobConf; -import com.cloudera.sqoop.TestExport; -import com.cloudera.sqoop.mapreduce.db.DBConfiguration; -import org.junit.Ignore; -import org.junit.Test; - - -/** - * Test the DirectPostgresqlManager implementations. - * DirectPostgresqlManager uses JDBC driver to facilitate it. - * - * Since this requires a Postgresql installation on your local machine to use, - * this class is named in such a way that Hadoop's default QA process does not - * run it. - * - * You need to run this manually with - * -Dtestcase=DirectPostgreSQLExportManualTest. - * - * You need to put Postgresql's JDBC driver library into lib dir. - * - * You need to create a sqooptest superuser and database and tablespace, - * - * $ sudo -u postgres createuser -U postgres -s sqooptest - * $ sudo -u postgres createdb -U sqooptest sqooptest - * $ psql -U sqooptest sqooptest - * - */ -public class DirectPostgreSQLExportManualTest extends TestExport { - - public static final Log LOG = - LogFactory.getLog(DirectPostgreSQLExportManualTest.class.getName()); - private DBConfiguration dbConf; - - static final String HOST_URL = - System.getProperty("sqoop.test.postgresql.connectstring.host_url", - "jdbc:postgresql://localhost/"); - static final String DATABASE = - System.getProperty("sqoop.test.postgresql.database", "sqooptest"); - static final String USERNAME = - System.getProperty("sqoop.test.postgresql.username", "sqooptest"); - static final String PASSWORD = System.getProperty( - "sqoop.test.postgresql.password"); - static final String CONNECT_STRING = HOST_URL + DATABASE; - - public DirectPostgreSQLExportManualTest() { - JobConf conf = new JobConf(getConf()); - DBConfiguration.configureDB(conf, - "org.postgresql.Driver", - getConnectString(), - getUserName(), - PASSWORD, (Integer) null); - dbConf = new DBConfiguration(conf); - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return CONNECT_STRING; - } - - protected String getUserName() { - return USERNAME; - } - - @Override - protected String getTablePrefix() { - return super.getTablePrefix().toLowerCase(); - } - - @Override - protected String getTableName() { - return super.getTableName().toLowerCase(); - } - - @Override - public String getStagingTableName() { - return super.getStagingTableName().toLowerCase(); - } - - @Override - protected Connection getConnection() { - try { - Connection conn = dbConf.getConnection(); - conn.setAutoCommit(false); - PreparedStatement stmt = - conn.prepareStatement("SET extra_float_digits TO 0"); - stmt.executeUpdate(); - conn.commit(); - return conn; - } catch (SQLException sqlE) { - LOG.error("Could not get connection to test server: " + sqlE); - return null; - } catch (ClassNotFoundException cnfE) { - LOG.error("Could not find driver class: " + cnfE); - return null; - } - } - - @Override - protected String getDropTableStatement(String tableName) { - return "DROP TABLE IF EXISTS " + tableName; - } - - @Override - protected String[] getArgv(boolean includeHadoopFlags, - int rowsPerStatement, - int statementsPerTx, - String... additionalArgv) { - ArrayList<String> args = - new ArrayList<String>(Arrays.asList(additionalArgv)); - args.add("--username"); - args.add(getUserName()); - args.add("--password"); - args.add(PASSWORD); - args.add("--direct"); - return super.getArgv(includeHadoopFlags, - rowsPerStatement, - statementsPerTx, - args.toArray(new String[0])); - } - - @Override - protected String [] getCodeGenArgv(String... extraArgs) { - ArrayList<String> args = new ArrayList<String>(Arrays.asList(extraArgs)); - args.add("--username"); - args.add(getUserName()); - args.add("--password"); - args.add(PASSWORD); - return super.getCodeGenArgv(args.toArray(new String[0])); - } - - @Ignore("Ignoring this test case as direct export does not support --columns option.") - @Override - @Test - public void testColumnsExport() throws IOException, SQLException { - } - - @Ignore("Ignoring this test case as the scenario is not supported with direct export.") - @Override - @Test - public void testLessColumnsInFileThanInTable() throws IOException, SQLException { - } - - @Ignore("Ignoring this test case as the scenario is not supported with direct export.") - @Override - @Test - public void testLessColumnsInFileThanInTableInputNullIntPassed() throws IOException, SQLException { - } - - @Ignore("Ignoring this test case as the scenario is not supported with direct export.") - @Override - @Test - public void testLessColumnsInFileThanInTableInputNullStringPassed() throws IOException, SQLException { - } - - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java deleted file mode 100644 index 6bf890b..0000000 --- a/src/test/com/cloudera/sqoop/manager/JdbcMySQLExportTest.java +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.StringUtils; -import org.junit.After; -import org.junit.Before; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.TestExport; -import org.junit.Test; - -import static org.junit.Assert.fail; - -/** - * Test the MySQLManager implementation's exportJob() functionality. - * This does a better test of ExportOutputFormat than TestExport does, - * because it supports multi-row INSERT statements. - */ -public class JdbcMySQLExportTest extends TestExport { - - public static final Log LOG = LogFactory.getLog( - JdbcMySQLExportTest.class.getName()); - - static final String TABLE_PREFIX = "EXPORT_MYSQL_J_"; - - // instance variables populated during setUp, used during tests. - private MySQLManager manager; - private Connection conn; - private MySQLTestUtils mySqlTestUtils = new MySQLTestUtils(); - - @Override - protected Connection getConnection() { - return conn; - } - - // MySQL allows multi-row INSERT statements. - @Override - protected int getMaxRowsPerStatement() { - return 1000; - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return mySqlTestUtils.getMySqlConnectString(); - } - - @Override - protected String getTablePrefix() { - return TABLE_PREFIX; - } - - @Override - protected String getDropTableStatement(String tableName) { - return "DROP TABLE IF EXISTS " + tableName; - } - - @Before - public void setUp() { - super.setUp(); - - SqoopOptions options = new SqoopOptions(mySqlTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySqlTestUtils.getUserName()); - mySqlTestUtils.addPasswordIfIsSet(options); - this.manager = new MySQLManager(options); - try { - this.conn = manager.getConnection(); - this.conn.setAutoCommit(false); - } catch (SQLException sqlE) { - LOG.error(StringUtils.stringifyException(sqlE)); - fail("Failed with sql exception in setup: " + sqlE); - } - } - - @After - public void tearDown() { - try { - Statement stmt = conn.createStatement(); - stmt.execute(getDropTableStatement(getTableName())); - stmt.execute(getDropTableStatement(getStagingTableName())); - } catch(SQLException e) { - LOG.error("Can't clean up the database:", e); - } - - super.tearDown(); - - if (null != this.conn) { - try { - this.conn.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException closing conn: " + sqlE.toString()); - } - } - } - - @Override - protected String [] getCodeGenArgv(String... extraArgs) { - return super.getCodeGenArgv(mySqlTestUtils.addUserNameAndPasswordToArgs(extraArgs)); - } - - @Override - protected String [] getArgv(boolean includeHadoopFlags, - int rowsPerStatement, int statementsPerTx, String... additionalArgv) { - - String [] subArgv = newStrArray(mySqlTestUtils.addUserNameAndPasswordToArgs(additionalArgv)); - return super.getArgv(includeHadoopFlags, rowsPerStatement, - statementsPerTx, subArgv); - } - - @Test - public void testIntColInBatchMode() throws IOException, SQLException { - final int TOTAL_RECORDS = 10; - - // generate a column equivalent to rownum. - ColumnGenerator gen = new ColumnGenerator() { - public String getExportText(int rowNum) { - return "" + rowNum; - } - public String getVerifyText(int rowNum) { - return "" + rowNum; - } - public String getType() { - return "INTEGER"; - } - }; - - createTextFile(0, TOTAL_RECORDS, false, gen); - createTable(gen); - runExport(getArgv(true, 10, 10, "--batch")); - verifyExport(TOTAL_RECORDS); - assertColMinAndMax(forIdx(0), gen); - } - - @Test - public void testUpsert() throws IOException, SQLException { - final int TOTAL_RECORDS = 10; - - createTextFile(0, TOTAL_RECORDS, false); - createTable(); - - // Insert only - runExport(getArgv(true, 10, 10, "--update-key", "id", - "--update-mode", "allowinsert")); - verifyExport(TOTAL_RECORDS); - - // Update only - runExport(getArgv(true, 10, 10, "--update-key", "id", - "--update-mode", "allowinsert")); - verifyExport(TOTAL_RECORDS); - - // Insert & update - createTextFile(0, TOTAL_RECORDS * 2, false); - runExport(getArgv(true, 10, 10, "--update-key", "id", - "--update-mode", "allowinsert")); - verifyExport(TOTAL_RECORDS * 2); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java b/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java deleted file mode 100644 index ce4af81..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.apache.hadoop.conf.Configuration; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.TestAllTables; - -/** - * Test the --all-tables functionality with MySQL. - */ -public class MySQLAllTablesTest extends TestAllTables { - - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return mySQLTestUtils.getMySqlConnectString(); - } - - @Override - protected SqoopOptions getSqoopOptions(Configuration conf) { - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(opts); - return opts; - } - - @Override - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - PreparedStatement statement = conn.prepareStatement( - "DROP TABLE IF EXISTS " + table, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLAuthTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLAuthTest.java b/src/test/com/cloudera/sqoop/manager/MySQLAuthTest.java deleted file mode 100644 index ed58c2b..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLAuthTest.java +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.FileInputStream; -import java.io.File; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.testutil.ImportJobTestCase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test authentication and remote access to direct mysqldump-based imports. - * - * Since this requires a MySQL installation with a properly configured database and user, this - * class is named in such a way that Hadoop's default QA process does not run - * it. You need to run this manually with -Dtestcase=MySQLAuthTest - * - * You need to put MySQL's Connector/J JDBC driver library into a location - * where Hadoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * If you don't have a database and a user which can be used by Sqoop, you can create them using - * the following MySQL commands: - * - * CREATE DATABASE sqooppasstest; - * use mysql; - * GRANT ALL PRIVILEGES on sqooppasstest.* TO 'sqooptest'@'localhost' - * IDENTIFIED BY '12345'; - * flush privileges; - * - * <br/> - * - * Ant command for running this test case: <br/> - * ant clean test - * -Dsqoop.thirdparty.lib.dir=mysql_driver_dir - * -Dsqoop.test.mysql.connectstring.host_url=jdbc:mysql://mysql_server_address/ - * -Dsqoop.test.mysql.username=sqooptest - * -Dsqoop.test.mysql.password=12345 - * -Dsqoop.test.mysql.databasename=sqooppasstest - * -Dtestcase=MySQLAuthTest - * - */ -public class MySQLAuthTest extends ImportJobTestCase { - - public static final Log LOG = LogFactory.getLog( - MySQLAuthTest.class.getName()); - - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - private List<String> createdTableNames = new ArrayList<>(); - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Before - public void setUp() { - super.setUp(); - SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(), - getTableName()); - options.setUsername(mySQLTestUtils.getUserName()); - options.setPassword(mySQLTestUtils.getUserPass()); - - LOG.debug("Setting up another MySQLAuthTest: " + mySQLTestUtils.getMySqlConnectString()); - - setManager(new DirectMySQLManager(options)); - } - - @After - public void tearDown() { - dropAllCreatedTables(); - super.tearDown(); - } - - private String [] getArgv(boolean includeHadoopFlags, - boolean useDirect, String connectString, String tableName) { - ArrayList<String> args = new ArrayList<String>(); - - if (includeHadoopFlags) { - CommonArgs.addHadoopFlags(args); - } - - args.add("--table"); - args.add(tableName); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(connectString); - if (useDirect) { - args.add("--direct"); - } - args.add("--username"); - args.add(mySQLTestUtils.getUserName()); - args.add("--password"); - args.add(mySQLTestUtils.getUserPass()); - args.add("--mysql-delimiters"); - args.add("--num-mappers"); - args.add("1"); - - return args.toArray(new String[0]); - } - - /** - * Connect to a db and ensure that password-based authentication - * succeeds. - */ - @Test - public void testAuthAccess() { - createAndPopulateAuthTable(); - String [] argv = getArgv(true, true, mySQLTestUtils.getMySqlConnectString(), getTableName()); - try { - runImport(argv); - } catch (IOException ioe) { - LOG.error("Got IOException during import: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } - - Path warehousePath = new Path(this.getWarehouseDir()); - Path tablePath = new Path(warehousePath, getTableName()); - Path filePath = new Path(tablePath, "part-m-00000"); - - File f = new File(filePath.toString()); - assertTrue("Could not find imported data file", f.exists()); - BufferedReader r = null; - try { - // Read through the file and make sure it's all there. - r = new BufferedReader(new InputStreamReader(new FileInputStream(f))); - assertEquals("1,'Aaron'", r.readLine()); - } catch (IOException ioe) { - LOG.error("Got IOException verifying results: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } finally { - IOUtils.closeStream(r); - } - } - - @Test - public void testZeroTimestamp() throws IOException, SQLException { - // MySQL timestamps can hold values whose range causes problems - // for java.sql.Timestamp. The MySQLManager adds settings to the - // connect string which configure the driver's handling of - // zero-valued timestamps. Check that all of these modifications - // to the connect string are successful. - - // A connect string with a null 'query' component. - doZeroTimestampTest(0, true, mySQLTestUtils.getMySqlConnectString()); - - // A connect string with a zero-length query component. - doZeroTimestampTest(1, true, mySQLTestUtils.getMySqlConnectString() + "?"); - - // A connect string with another argument - doZeroTimestampTest(2, true, mySQLTestUtils.getMySqlConnectString() + "?connectTimeout=0"); - doZeroTimestampTest(3, true, mySQLTestUtils.getMySqlConnectString() + "?connectTimeout=0&"); - - // A connect string with the zero-timestamp behavior already - // configured. - doZeroTimestampTest(4, true, mySQLTestUtils.getMySqlConnectString() - + "?zeroDateTimeBehavior=convertToNull"); - - // And finally, behavior already configured in such a way as to - // cause the timestamp import to fail. - doZeroTimestampTest(5, false, mySQLTestUtils.getMySqlConnectString() - + "?zeroDateTimeBehavior=exception"); - } - - public void doZeroTimestampTest(int testNum, boolean expectSuccess, - String connectString) throws IOException, SQLException { - - LOG.info("Beginning zero-timestamp test #" + testNum); - - final String tableName = "mysqlTimestampTable" + Integer.toString(testNum); - - createAndPopulateZeroTimestampTable(tableName); - - // Run the import. - String [] argv = getArgv(true, false, connectString, tableName); - try { - runImport(argv); - } catch (Exception e) { - if (expectSuccess) { - // This is unexpected. rethrow. - throw new RuntimeException(e); - } else { - // We expected an error. - LOG.info("Got exception running import (expected). msg: " + e); - } - } - - // Make sure the result file is there. - Path warehousePath = new Path(this.getWarehouseDir()); - Path tablePath = new Path(warehousePath, tableName); - Path filePath = new Path(tablePath, "part-m-00000"); - - File f = new File(filePath.toString()); - if (expectSuccess) { - assertTrue("Could not find imported data file", f.exists()); - BufferedReader r = new BufferedReader(new InputStreamReader( - new FileInputStream(f))); - assertEquals("1,null", r.readLine()); - IOUtils.closeStream(r); - } else { - assertFalse("Imported data when expected failure", f.exists()); - } - } - - private void createAndPopulateZeroTimestampTable(String tableName) { - String[] colNames = { "id", "ts" }; - String[] colTypes = { "INT NOT NULL PRIMARY KEY AUTO_INCREMENT", "TIMESTAMP NOT NULL" }; - String[] colValues = { "NULL", "'0000-00-00 00:00:00.0'" }; - createTableWithColTypesAndNames(tableName, colNames, colTypes, colValues); - createdTableNames.add(tableName); - } - - private void dropAllCreatedTables() { - try { - for (String createdTableName : createdTableNames) { - dropTableIfExists(createdTableName); - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - private void createAndPopulateAuthTable() { - String[] colNames = { "id", "name" }; - String[] colTypes = { "INT NOT NULL PRIMARY KEY AUTO_INCREMENT", "VARCHAR(24) NOT NULL" }; - String[] colValues = { "NULL", "'Aaron'" }; - - createTableWithColTypesAndNames(colNames, colTypes, colValues); - createdTableNames.add(getTableName()); - } - - protected String dropTableIfExistsCommand(String tableName) { - return String.format("DROP TABLE IF EXISTS %s", tableName); - } - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java b/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java deleted file mode 100644 index 6539972..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.ManagerCompatTestCase; - -/** - * Test the basic mysql connection manager with the various column types. - */ -public class MySQLCompatTest extends ManagerCompatTestCase { - - public static final Log LOG = LogFactory.getLog( - MySQLCompatTest.class.getName()); - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected Log getLogger() { - return LOG; - } - - @Override - protected String getDbFriendlyName() { - return "MySQL"; - } - - @Override - protected String getConnectString() { - return mySQLTestUtils.getMySqlConnectString(); - } - - @Override - protected SqoopOptions getSqoopOptions(Configuration conf) { - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(opts); - return opts; - - } - - @Override - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - PreparedStatement statement = conn.prepareStatement( - "DROP TABLE IF EXISTS " + table, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } - - @Override - protected String getLongVarCharType() { - return "MEDIUMTEXT"; - } - - @Override - protected String getTimestampType() { - // return a nullable timestamp type. - return "TIMESTAMP NULL"; - } - - @Override - protected String getClobType() { - return "MEDIUMTEXT"; - } - - @Override - protected String getBlobType() { - return "MEDIUMBLOB"; - } - - @Override - protected String getRealSeqOutput(String realAsInserted) { - return withDecimalZero(realAsInserted); - } - - @Override - protected String getFloatSeqOutput(String floatAsInserted) { - return withDecimalZero(floatAsInserted); - } - - @Override - protected String getDoubleSeqOutput(String doubleAsInserted) { - return withDecimalZero(doubleAsInserted); - } - - @Override - protected String getTimestampSeqOutput(String tsAsInserted) { - // We trim timestamps to exactly one tenth of a second. - if ("null".equals(tsAsInserted)) { - return tsAsInserted; - } - - int dotPos = tsAsInserted.indexOf("."); - if (-1 == dotPos) { - return tsAsInserted + ".0"; - } else { - return tsAsInserted.substring(0, dotPos + 2); - } - } - - @Override - protected String getNumericSeqOutput(String numAsInserted) { - // We always pad to exactly the number of digits in - // getNumericDecPartDigits(). - - int totalDecPartSize = getNumericDecPartDigits(); - int numPad; // number of digits to pad by. - - int dotPos = numAsInserted.indexOf("."); - if (-1 == dotPos) { - numAsInserted = numAsInserted + "."; - numPad = totalDecPartSize; - } else { - int existingDecimalSize = numAsInserted.length() - dotPos; - numPad = totalDecPartSize - existingDecimalSize; - } - - if (numPad < 0) { - // We actually have to trim the value. - return numAsInserted.substring(0, numAsInserted.length() + numPad + 1); - } else { - String zeros = ""; - for (int i = 0; i < numPad; i++) { - zeros = zeros + "0"; - } - return numAsInserted + zeros; - } - } - - @Override - protected String getDecimalSeqOutput(String numAsInserted) { - return getNumericSeqOutput(numAsInserted); - } - - @Test - public void testYear() { - verifyType("YEAR", "2012", "2012"); - } -} - http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java b/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java deleted file mode 100644 index 22547f2..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLFreeFormQueryTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.TestFreeFormQueryImport; - -/** - * Test free form query import with the MySQL db. - */ -public class MySQLFreeFormQueryTest extends TestFreeFormQueryImport { - - public static final Log LOG = LogFactory.getLog( - MySQLFreeFormQueryTest.class.getName()); - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected Log getLogger() { - return LOG; - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return mySQLTestUtils.getMySqlConnectString(); - } - - @Override - protected SqoopOptions getSqoopOptions(Configuration conf) { - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(opts); - return opts; - } - - @Override - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - PreparedStatement statement = conn.prepareStatement( - "DROP TABLE IF EXISTS " + table, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java b/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java deleted file mode 100644 index cd146f4..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLLobAvroImportTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.LobAvroImportTestCase; - -/** - * Tests BLOB/CLOB import for Avro with MySQL Db. - */ -public class MySQLLobAvroImportTest extends LobAvroImportTestCase { - - public static final Log LOG = LogFactory.getLog( - OracleCompatTest.class.getName()); - private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils(); - - @Override - protected Log getLogger() { - return LOG; - } - - @Override - protected String getDbFriendlyName() { - return "MySQL"; - } - - @Override - protected String getConnectString() { - return mySQLTestUtils.getMySqlConnectString(); - } - - @Override - protected SqoopOptions getSqoopOptions(Configuration conf) { - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(mySQLTestUtils.getUserName()); - mySQLTestUtils.addPasswordIfIsSet(opts); - return opts; - } - - @Override - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - PreparedStatement statement = conn.prepareStatement( - "DROP TABLE IF EXISTS " + table, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } - - @Override - protected String getBlobType() { - return "MEDIUMBLOB"; - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java b/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java deleted file mode 100644 index 77aefde..0000000 --- a/src/test/com/cloudera/sqoop/manager/MySQLTestUtils.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import com.cloudera.sqoop.SqoopOptions; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; - -/** - * Utilities for mysql-based tests. - */ -public final class MySQLTestUtils { - - public static final Log LOG = LogFactory.getLog( - MySQLTestUtils.class.getName()); - - private String hostUrl; - - private String userName; - private String userPass; - - private String mysqlDbNAme; - private String mySqlConnectString; - - public MySQLTestUtils() { - hostUrl = System.getProperty( - "sqoop.test.mysql.connectstring.host_url", - "jdbc:mysql://localhost/"); - userName = System.getProperty("sqoop.test.mysql.username", getCurrentUser()); - userPass = System.getProperty("sqoop.test.mysql.password"); - - mysqlDbNAme = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb"); - mySqlConnectString = getHostUrl() + getMysqlDbNAme(); - } - - public String getHostUrl() { - return hostUrl; - } - - public String getUserName() { - return userName; - } - - public String getUserPass() { - return userPass; - } - - public String getMysqlDbNAme() { - return mysqlDbNAme; - } - - - public String getMySqlConnectString() { - return mySqlConnectString; - } - - public String[] addUserNameAndPasswordToArgs(String[] extraArgs) { - int extraLength = isSet(getUserPass()) ? 4 : 2; - String[] moreArgs = new String[extraArgs.length + extraLength]; - int i = 0; - for (i = 0; i < extraArgs.length; i++) { - moreArgs[i] = extraArgs[i]; - } - - // Add username argument for mysql. - moreArgs[i++] = "--username"; - moreArgs[i++] = getUserName(); - if (isSet(userPass)) { - moreArgs[i++] = "--password"; - moreArgs[i++] = getUserPass(); - } - return moreArgs; - } - - public static String getCurrentUser() { - // First, check the $USER environment variable. - String envUser = System.getenv("USER"); - if (null != envUser) { - return envUser; - } - // Try `whoami` - String[] whoamiArgs = new String[1]; - whoamiArgs[0] = "whoami"; - Process p = null; - BufferedReader r = null; - try { - p = Runtime.getRuntime().exec(whoamiArgs); - InputStream is = p.getInputStream(); - r = new BufferedReader(new InputStreamReader(is)); - return r.readLine(); - } catch (IOException ioe) { - LOG.error("IOException reading from `whoami`: " + ioe.toString()); - return null; - } finally { - // close our stream. - if (null != r) { - try { - r.close(); - } catch (IOException ioe) { - LOG.warn("IOException closing input stream from `whoami`: " - + ioe.toString()); - } - } - // wait for whoami to exit. - while (p != null) { - try { - int ret = p.waitFor(); - if (0 != ret) { - LOG.error("whoami exited with error status " + ret); - // suppress original return value from this method. - return null; - } - } catch (InterruptedException ie) { - continue; // loop around. - } - } - - } - } - - public void addPasswordIfIsSet(ArrayList<String> args) { - if (isSet(userPass)) { - args.add("--password"); - args.add(getUserPass()); - } - } - - private boolean isSet(String userPass) { - return !StringUtils.isBlank(userPass); - } - - public void addPasswordIfIsSet(SqoopOptions opts) { - if (isSet(userPass)) { - opts.setPassword(getUserPass()); - } - } - -} \ No newline at end of file
