Repository: sqoop Updated Branches: refs/heads/trunk afd6a8610 -> 0ca73d4e7
SQOOP-3169: Evaluate and fix SQLServer Manual tests (Boglarka Egyed via Anna Szonyi) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0ca73d4e Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0ca73d4e Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0ca73d4e Branch: refs/heads/trunk Commit: 0ca73d4e71bf4724cd7dd15faa108e6ee56ee121 Parents: afd6a86 Author: Anna Szonyi <[email protected]> Authored: Thu Apr 13 11:33:45 2017 +0200 Committer: Anna Szonyi <[email protected]> Committed: Thu Apr 13 11:33:45 2017 +0200 ---------------------------------------------------------------------- build.xml | 6 + .../com/cloudera/sqoop/hive/TestHiveImport.java | 12 +- .../SQLServerManagerExportManualTest.java | 31 ++++- .../SQLServerManagerImportManualTest.java | 27 ++++- .../sqoop/manager/sqlserver/MSSQLTestUtils.java | 21 ++-- .../manager/sqlserver/ManagerCompatExport.java | 118 +++++-------------- ...erDatatypeExportDelimitedFileManualTest.java | 25 +++- ...verDatatypeExportSequenceFileManualTest.java | 32 +++-- ...erDatatypeImportDelimitedFileManualTest.java | 88 +++++++++++--- ...verDatatypeImportSequenceFileManualTest.java | 23 +++- .../SQLServerHiveImportManualTest.java | 41 +++++-- .../sqlserver/SQLServerManagerManualTest.java | 24 +++- .../sqlserver/SQLServerMultiColsManualTest.java | 36 +++++- .../sqlserver/SQLServerMultiMapsManualTest.java | 26 ++-- .../SQLServerParseMethodsManualTest.java | 38 ++++-- .../sqlserver/SQLServerQueryManualTest.java | 26 ++-- .../sqlserver/SQLServerSplitByManualTest.java | 26 ++-- .../sqlserver/SQLServerWhereManualTest.java | 28 +++-- 18 files changed, 432 insertions(+), 196 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 73db28b..10deb83 100644 --- a/build.xml +++ b/build.xml @@ -224,6 +224,9 @@ <property name="sqoop.test.sqlserver.connectstring.host_url" value="jdbc:sqlserver://sqlserverhost:1433"/> + <property name="sqoop.test.sqlserver.database" + value="sqooptest"/> + <property name="java.security.krb5.realm" value="OX.AC.UK"/> @@ -850,6 +853,9 @@ <sysproperty key="sqoop.test.sqlserver.connectstring.host_url" value="${sqoop.test.sqlserver.connectstring.host_url}"/> + <sysproperty key="sqoop.test.sqlserver.database" + value="${sqoop.test.sqlserver.database}"/> + <sysproperty key="sqoop.test.msserver.connector.factory" value="${sqoop.test.msserver.connector.factory}"/> http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index 33e0cc4..a624f52 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -104,7 +104,7 @@ public class TestHiveImport extends ImportJobTestCase { setColNames(cols); } - protected String[] getTypesNewLineTest() { + protected String[] getTypes() { String[] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; return types; } @@ -295,7 +295,7 @@ public class TestHiveImport extends ImportJobTestCase { final String TABLE_NAME = "NORMAL_HIVE_IMPORT_AS_PARQUET"; setCurTableName(TABLE_NAME); setNumCols(3); - String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] types = getTypes(); String [] vals = { "'test'", "42", "'somestring'" }; String [] extraArgs = {"--as-parquetfile"}; @@ -382,7 +382,7 @@ public class TestHiveImport extends ImportJobTestCase { final String TABLE_NAME = "CREATE_OVERWRITE_HIVE_IMPORT_AS_PARQUET"; setCurTableName(TABLE_NAME); setNumCols(3); - String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] types = getTypes(); String [] vals = { "'test'", "42", "'somestring'" }; String [] extraArgs = {"--as-parquetfile"}; ImportTool tool = new ImportTool(); @@ -441,7 +441,7 @@ public class TestHiveImport extends ImportJobTestCase { final String TABLE_NAME = "APPEND_HIVE_IMPORT_AS_PARQUET"; setCurTableName(TABLE_NAME); setNumCols(3); - String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] types = getTypes(); String [] vals = { "'test'", "42", "'somestring'" }; String [] extraArgs = {"--as-parquetfile"}; String [] args = getArgv(false, extraArgs); @@ -560,7 +560,7 @@ public class TestHiveImport extends ImportJobTestCase { LOG.info("Doing import of single row into FIELD_WITH_NL_HIVE_IMPORT table"); setCurTableName(TABLE_NAME); setNumCols(3); - String[] types = getTypesNewLineTest(); + String[] types = getTypes(); String[] vals = { "'test with \n new lines \n'", "42", "'oh no " + '\01' + " field delims " + '\01' + "'", }; String[] moreArgs = { "--"+ BaseSqoopTool.HIVE_DROP_DELIMS_ARG }; @@ -609,7 +609,7 @@ public class TestHiveImport extends ImportJobTestCase { + "FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT table"); setCurTableName(TABLE_NAME); setNumCols(3); - String[] types = getTypesNewLineTest(); + String[] types = getTypes(); String[] vals = { "'test with\nnew lines\n'", "42", "'oh no " + '\01' + " field delims " + '\01' + "'", }; String[] moreArgs = { "--"+BaseSqoopTool.HIVE_DELIMS_REPLACEMENT_ARG, " "}; http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java index 9a92479..668a3a9 100644 --- a/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java +++ b/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java @@ -53,10 +53,16 @@ public class SQLServerManagerExportManualTest extends ExportJobTestCase { static final String HOST_URL = System.getProperty( "sqoop.test.sqlserver.connectstring.host_url", "jdbc:sqlserver://sqlserverhost:1433"); + static final String DATABASE_NAME = System.getProperty( + "sqoop.test.sqlserver.database", + "sqooptest"); + static final String DATABASE_USER = System.getProperty( + "ms.sqlserver.username", + "sqoopuser"); + static final String DATABASE_PASSWORD = System.getProperty( + "ms.sqlserver.password", + "password"); - static final String DATABASE_NAME = "SQOOPTEST"; - static final String DATABASE_USER = "SQOOPUSER"; - static final String DATABASE_PASSWORD = "PASSWORD"; static final String SCHEMA_DBO = "dbo"; static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL"; static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL"; @@ -84,6 +90,11 @@ public class SQLServerManagerExportManualTest extends ExportJobTestCase { return false; } + private String getDropTableStatement(String schema, String tableName) { + return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema) + + "." + manager.escapeObjectName(tableName); + } + @Before public void setUp() { super.setUp(); @@ -247,6 +258,14 @@ public class SQLServerManagerExportManualTest extends ExportJobTestCase { @After public void tearDown() { + try { + Statement stmt = conn.createStatement(); + stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME)); + stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME)); + } catch (SQLException e) { + LOG.error("Can't clean up the database:", e); + } + super.tearDown(); try { conn.close(); @@ -382,11 +401,11 @@ public class SQLServerManagerExportManualTest extends ExportJobTestCase { }); // first time will be insert. runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id", - "--update-mode", "allowinsert")); + "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH)); // second time will be update. runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id", - "--update-mode", "allowinsert")); - assertRowCount(2, escapeObjectName(SCH_TABLE_NAME), conn); + "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH)); + assertRowCount(2, escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME), conn); } public static void checkSQLBinaryTableContent(String[] expected, String tableName, Connection connection){ http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java index 1f69725..abc0479 100644 --- a/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java +++ b/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java @@ -75,10 +75,16 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase { static final String HOST_URL = System.getProperty( "sqoop.test.sqlserver.connectstring.host_url", "jdbc:sqlserver://sqlserverhost:1433"); + static final String DATABASE_NAME = System.getProperty( + "sqoop.test.sqlserver.database", + "sqooptest"); + static final String DATABASE_USER = System.getProperty( + "ms.sqlserver.username", + "sqoopuser"); + static final String DATABASE_PASSWORD = System.getProperty( + "ms.sqlserver.password", + "password"); - static final String DATABASE_NAME = "SQOOPTEST"; - static final String DATABASE_USER = "SQOOPUSER"; - static final String DATABASE_PASSWORD = "PASSWORD"; static final String SCHEMA_DBO = "dbo"; static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL"; static final String SCHEMA_SCH = "sch"; @@ -94,6 +100,7 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase { private SQLServerManager manager; private Configuration conf = new Configuration(); + private Connection conn = null; @Override protected Configuration getConf() { @@ -105,6 +112,11 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase { return false; } + private String getDropTableStatement(String schema, String tableName) { + return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema) + + "." + manager.escapeObjectName(tableName); + } + @Before public void setUp() { super.setUp(); @@ -130,7 +142,6 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase { String fulltableName = manager.escapeObjectName(schema) + "." + manager.escapeObjectName(table); - Connection conn = null; Statement stmt = null; // Create schema if needed @@ -208,6 +219,14 @@ public class SQLServerManagerImportManualTest extends ImportJobTestCase { @After public void tearDown() { + try { + Statement stmt = conn.createStatement(); + stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME)); + stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME)); + } catch (SQLException e) { + LOG.error("Can't clean up the database:", e); + } + super.tearDown(); try { manager.close(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java b/src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java index 851bf49..2220b7d 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/MSSQLTestUtils.java @@ -17,9 +17,6 @@ */ package org.apache.sqoop.manager.sqlserver; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -27,9 +24,6 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -47,10 +41,15 @@ public class MSSQLTestUtils { "ms.sqlserver.username", "SQOOPUSER"); static final String DATABASE_PASSWORD = System.getProperty( "ms.sqlserver.password", "PASSWORD"); + static final String DATABASE_NAME = System.getProperty( + "sqoop.test.sqlserver.database", + "sqooptest"); public static final String HOST_URL = System.getProperty( "sqoop.test.sqlserver.connectstring.host_url", "jdbc:sqlserver://sqlserverhost:1433"); + public static final String CONNECT_STRING = HOST_URL + ";database=" + DATABASE_NAME; + public static final String CREATE_TALBE_LINEITEM = "CREATE TABLE TPCH1M_LINEITEM" + "( [L_ORDERKEY] [int] NULL, [L_PARTKEY] " @@ -70,7 +69,7 @@ public class MSSQLTestUtils { if (conn == null) { try { - Connection con = DriverManager.getConnection(HOST_URL, + Connection con = DriverManager.getConnection(CONNECT_STRING, DATABASE_USER, DATABASE_PASSWORD); conn = con; return con; @@ -158,6 +157,14 @@ public class MSSQLTestUtils { return DATABASE_PASSWORD; } + public static String getDBDatabaseName() { + return DATABASE_NAME; + } + + public static String getDBConnectString() { + return CONNECT_STRING; + } + public void dropTableIfExists(String table) throws SQLException { conn = getConnection(); System.out.println("Dropping table : " + table); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java b/src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java index 8c5176a..be2b22c 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/ManagerCompatExport.java @@ -40,7 +40,6 @@ import com.cloudera.sqoop.testutil.ExportJobTestCase; import com.cloudera.sqoop.tool.ExportTool; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @@ -117,162 +116,162 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { public abstract void createFile(DATATYPES dt, String data) throws Exception; @Test - public void testVarBinary() { + public void testVarBinary() throws Exception { exportTestMethod(DATATYPES.VARBINARY); } @Test - public void testTime() { + public void testTime() throws Exception { exportTestMethod(DATATYPES.TIME); } @Test - public void testSmalldatetime() { + public void testSmalldatetime() throws Exception { exportTestMethod(DATATYPES.SMALLDATETIME); } @Test - public void testdatetime2() { + public void testdatetime2() throws Exception { exportTestMethod(DATATYPES.DATETIME2); } @Test - public void testdatetime() { + public void testdatetime() throws Exception { exportTestMethod(DATATYPES.DATETIME); } @Test - public void testdatetimeoffset() { + public void testdatetimeoffset() throws Exception { exportTestMethod(DATATYPES.DATETIMEOFFSET); } @Test - public void testDecimal() { + public void testDecimal() throws Exception { exportTestMethod(DATATYPES.DECIMAL); } @Test - public void testNumeric() { + public void testNumeric() throws Exception { exportTestMethod(DATATYPES.NUMERIC); } @Test - public void testBigInt() { + public void testBigInt() throws Exception { exportTestMethod(DATATYPES.BIGINT); } @Test - public void testInt() { + public void testInt() throws Exception { exportTestMethod(DATATYPES.INT); } @Test - public void testSmallInt() { + public void testSmallInt() throws Exception { exportTestMethod(DATATYPES.SMALLINT); } @Test - public void testTinyint() { + public void testTinyint() throws Exception { exportTestMethod(DATATYPES.TINYINT); } @Test - public void testFloat() { + public void testFloat() throws Exception { exportTestMethod(DATATYPES.FLOAT); } @Test - public void testReal() { + public void testReal() throws Exception { exportTestMethod(DATATYPES.REAL); } @Test - public void testDate() { + public void testDate() throws Exception { exportTestMethod(DATATYPES.DATE); } @Test - public void testMoney() { + public void testMoney() throws Exception { exportTestMethod(DATATYPES.MONEY); } @Test - public void testSmallMoney() { + public void testSmallMoney() throws Exception { exportTestMethod(DATATYPES.SMALLMONEY); } @Test - public void testText() { + public void testText() throws Exception { exportTestMethod(DATATYPES.TEXT); } @Test - public void testVarchar() { + public void testVarchar() throws Exception { exportTestMethod(DATATYPES.VARCHAR); } @Test - public void testChar() { + public void testChar() throws Exception { exportTestMethod(DATATYPES.CHAR); } @Test - public void testNText() { + public void testNText() throws Exception { exportTestMethod(DATATYPES.NTEXT); } @Test - public void testNChar() { + public void testNChar() throws Exception { exportTestMethod(DATATYPES.NCHAR); } @Test - public void testNVarchar() { + public void testNVarchar() throws Exception { exportTestMethod(DATATYPES.NVARCHAR); } @Test - public void testImage() { + public void testImage() throws Exception { exportTestMethod(DATATYPES.IMAGE); } @Test - public void testBinary() { + public void testBinary() throws Exception { exportTestMethod(DATATYPES.BINARY); } - public void exportTestMethod(DATATYPES dt) { + public void exportTestMethod(DATATYPES dt) throws SQLException { int exceptionCount = 0; List testdata = tdfs.getTestdata(dt); @@ -370,6 +369,8 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { } catch (Error e) { addToReport(current, e); exceptionCount++; + } finally { + dropTableIfExists(getTableName(dt)); } } if (exceptionCount > 0) { @@ -377,7 +378,7 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { System.out.println("There were failures for :" + dt.toString()); System.out.println("Failed for " + exceptionCount + "/" + testdata.size() + " test data samples\n"); - System.out.println("Sroll up for detailed errors"); + System.out.println("Scroll up for detailed errors"); System.out .println("----------------------------------------------------------" + "-"); @@ -387,25 +388,6 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { } - /* - - */ - - public String[] extractData(List data, String negPosFlag, - KEY_STRINGS readBackType) { - List<String> filtered = new ArrayList<String>(); - for (Iterator<MSSQLTestData> itr = data.iterator(); itr.hasNext();) { - MSSQLTestData current = itr.next(); - if (current.getData(KEY_STRINGS.NEG_POS_FLAG).toString().equals( - negPosFlag)) { - filtered.add(current.getData(readBackType)); - } - } - String[] ret = new String[filtered.size()]; - filtered.toArray(ret); - return ret; - } - public void verifyExport(DATATYPES dt, String[] data) throws SQLException { LOG.info("Verifying export: " + getTableName()); // Check that we got back the correct number of records. @@ -446,43 +428,6 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { } - public void verifyNegativeExport(DATATYPES dt, String[] data) - throws SQLException { - LOG.info("Verifying export: " + getTableName()); - // Check that we got back the correct number of records. - Connection conn = getManager().getConnection(); - - PreparedStatement statement = conn.prepareStatement("SELECT " - + getColName() + " FROM " + getTableName(dt), - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - System.out.println("data samples being compared : " + data.length); - - ResultSet rs = null; - try { - rs = statement.executeQuery(); - int cnt = 0; - try { - while (rs.next()) { - String tmp = rs.getString(1); - String expected = data[cnt++]; - System.out.println("Readback, expected" + tmp + " :" - + expected); - if (tmp == null) { - assertNull("Must be null", expected); - } else { - assertNotSame("Data must match", expected, tmp); - } - } - - } finally { - rs.close(); - } - } finally { - statement.close(); - } - - } - public void verifyExport(DATATYPES dt, String data) throws SQLException { verifyExport(dt, new String[] { data }); @@ -568,10 +513,9 @@ public abstract class ManagerCompatExport extends ExportJobTestCase { return false; } + @Override protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java index 099d734..539eeb3 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; import java.io.OutputStream; @@ -30,11 +29,29 @@ import java.io.OutputStreamWriter; import java.io.BufferedWriter; /** -* Export delimited file SQL Server. -*/ + * Test to export delimited file to SQL Server. + * + * This uses JDBC to export data to an SQLServer database from HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeExportDelimitedFileManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. + */ public class SQLServerDatatypeExportDelimitedFileManualTest extends ManagerCompatExport { + @Override public void createFile(DATATYPES dt, String[] data) throws IOException { Path tablePath = getTablePath(dt); Path filePath = new Path(tablePath, "part0000"); @@ -60,10 +77,12 @@ public class SQLServerDatatypeExportDelimitedFileManualTest os.close(); } + @Override public void createFile(DATATYPES dt, String data) throws IOException { createFile(dt, new String[] { data }); } + @Override public String getOutputFileName() { return "ManagerCompatExportDelim.txt"; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java index 21676f0..0f206d0 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java @@ -29,7 +29,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; import com.cloudera.sqoop.SqoopOptions; @@ -43,14 +42,31 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** -* Export sequence file to SQL Server test. -*/ + * Test to export sequence file to SQL Server. + * + * This uses JDBC to export data to an SQLServer database from HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeExportSequenceFileManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. + */ public class SQLServerDatatypeExportSequenceFileManualTest extends ManagerCompatExport { private static Map jars = new HashMap(); - @Override + @Override public void createFile(DATATYPES dt, String[] data) throws Exception { try { codeGen(dt); @@ -142,11 +158,9 @@ public class SQLServerDatatypeExportSequenceFileManualTest jars.put(dt, jarFileName); return (getArgv(dt, "--class-name", className, "--jar-file", jarFileName)); - - - } + @Override protected String[] getArgv(DATATYPES dt) { String[] args = super.getArgv(dt); @@ -177,7 +191,7 @@ public class SQLServerDatatypeExportSequenceFileManualTest codeGenArgv.add("--table"); codeGenArgv.add(getTableName(dt)); codeGenArgv.add("--connect"); - codeGenArgv.add(getConnectString()); + codeGenArgv.add(MSSQLTestUtils.getDBConnectString()); codeGenArgv.add("--fields-terminated-by"); codeGenArgv.add("\\t"); codeGenArgv.add("--lines-terminated-by"); @@ -225,7 +239,7 @@ public class SQLServerDatatypeExportSequenceFileManualTest args.add("--export-dir"); args.add(getTablePath(dt).toString()); args.add("--connect"); - args.add(getConnectString()); + args.add(MSSQLTestUtils.getDBConnectString()); args.add("--fields-terminated-by"); args.add("\\t"); args.add("--lines-terminated-by"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java index 519fb52..9c20bca 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; import com.cloudera.sqoop.Sqoop; import com.cloudera.sqoop.SqoopOptions; @@ -38,13 +37,31 @@ import com.cloudera.sqoop.orm.CompilationManager; import com.cloudera.sqoop.testutil.CommonArgs; import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.util.ClassLoaderStack; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test import delimited file from SQL Server. + * Test to import delimited file from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeImportDelimitedFileManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerDatatypeImportDelimitedFileManualTest extends SQLServerDatatypeImportSequenceFileManualTest { @@ -88,7 +105,7 @@ public class SQLServerDatatypeImportDelimitedFileManualTest args.add("--warehouse-dir"); args.add(getWarehouseDir()); args.add("--connect"); - args.add(getConnectString()); + args.add(MSSQLTestUtils.getDBConnectString()); args.add("--num-mappers"); args.add("2"); @@ -190,13 +207,13 @@ public class SQLServerDatatypeImportDelimitedFileManualTest } finally { IOUtils.closeStream(reader); } - LOG.info("Read back from sequencefile: " + line); + LOG.info("Read back from delimited file: " + line); foundRecord = true; // Add trailing '\n' to expected value since // SqoopRecord.toString() // encodes the record delim. if (null == expectedVal) { - assertEquals("Error validating result from SeqFile", + assertEquals("Error validating result from delimited file", "null\n", line); } } catch (EOFException eoe) { @@ -209,7 +226,7 @@ public class SQLServerDatatypeImportDelimitedFileManualTest } if (!foundRecord) { - fail("Couldn't read any records from SequenceFiles"); + fail("Couldn't read any records from delimited file"); } } catch (IOException ioe) { LOG.error(StringUtils.stringifyException(ioe)); @@ -221,15 +238,6 @@ public class SQLServerDatatypeImportDelimitedFileManualTest } } - - @Test - public void testVarBinary() { - if (!supportsVarBinary()) { - return; - } - dataTypeTest(DATATYPES.VARBINARY); - } - @Test public void testTime() { if (!supportsTime()) { @@ -240,6 +248,56 @@ public class SQLServerDatatypeImportDelimitedFileManualTest dataTypeTest(DATATYPES.TIME); } + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testVarBinary() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit2() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit3() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testNChar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testChar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testVarchar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testNVarchar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBinary() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testTimestamp3() { + } + public String getResportFileName(){ return this.getClass().toString()+".txt"; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java index a0dad8a..9cd3176 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java @@ -42,7 +42,24 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * Testing import of a sequence file to SQL Server. + * Test importing sequence file from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeImportSequenceFileManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerDatatypeImportSequenceFileManualTest extends ManagerCompatTestCase { @@ -89,9 +106,7 @@ public class SQLServerDatatypeImportSequenceFileManualTest extends } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java index 1999272..ab3dd08 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java @@ -23,22 +23,37 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.hive.TestHiveImport; import com.cloudera.sqoop.testutil.CommonArgs; import com.cloudera.sqoop.tool.SqoopTool; +import org.junit.After; import org.junit.Before; import static org.junit.Assert.fail; /** * Test import to Hive from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerHiveImportManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerHiveImportManualTest extends TestHiveImport { @@ -47,18 +62,28 @@ public class SQLServerHiveImportManualTest extends TestHiveImport { super.setUp(); } + @After + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException sqle) { + LOG.info("Table clean-up failed: " + sqle); + } finally { + super.tearDown(); + } + } + protected boolean useHsqldbTestServer() { return false; } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } //SQL Server pads out - protected String[] getTypesNewLineTest() { + @Override + protected String[] getTypes() { String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" }; return types; } @@ -115,7 +140,7 @@ public class SQLServerHiveImportManualTest extends TestHiveImport { protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) { ArrayList<String> args = new ArrayList<String>(); - System.out.println("Ovverdien getArgv is called.."); + System.out.println("Overridden getArgv is called.."); if (includeHadoopFlags) { CommonArgs.addHadoopFlags(args); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java index 1178e3c..8cc5a0b 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java @@ -30,7 +30,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -49,8 +48,25 @@ import static org.junit.Assert.fail; /** * Test methods of the generic SqlManager implementation. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerManagerManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ -public class SQLServerManagerManualTest { +public class SQLServerManagerManualTest { public static final Log LOG = LogFactory.getLog( SQLServerManagerManualTest.class.getName()); @@ -282,9 +298,7 @@ public class SQLServerManagerManualTest { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java index 6a8ab51..51d5f75 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java @@ -22,14 +22,31 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import org.apache.hadoop.conf.Configuration; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.TestMultiCols; +import org.junit.After; import org.junit.Test; /** - * Test multiple columns SQL Server. + * Test multiple columns in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerMultiColsManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerMultiColsManualTest extends TestMultiCols { @@ -38,9 +55,7 @@ public class SQLServerMultiColsManualTest extends TestMultiCols { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** @@ -77,6 +92,17 @@ public class SQLServerMultiColsManualTest extends TestMultiCols { } + @After + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException sqle) { + LOG.info("Table clean-up failed: " + sqle); + } finally { + super.tearDown(); + } + } + @Test public void testMixed4() { // Overridden to bypass test case invalid for MSSQL server http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java index c9a5b5e..fc9e20d 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java @@ -26,8 +26,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -37,7 +35,6 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.mapred.Utils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; @@ -56,7 +53,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test that using multiple mapper splits works. + * Test multiple mapper splits in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerMultiMapsManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerMultiMapsManualTest extends ImportJobTestCase { @@ -247,9 +261,7 @@ public class SQLServerMultiMapsManualTest extends ImportJobTestCase { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java index cd05aec..b28c165 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java @@ -25,8 +25,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -37,7 +35,6 @@ import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; @@ -48,6 +45,7 @@ import com.cloudera.sqoop.testutil.ImportJobTestCase; import com.cloudera.sqoop.testutil.ReparseMapper; import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.util.ClassLoaderStack; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -55,7 +53,24 @@ import static org.junit.Assert.fail; /** * Test that the parse() methods generated in user SqoopRecord implementations - * work. + * work in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerParseMethodsManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerParseMethodsManualTest extends ImportJobTestCase { @@ -72,6 +87,17 @@ public class SQLServerParseMethodsManualTest extends ImportJobTestCase { } } + @After + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException sqle) { + LOG.info("Table clean-up failed: " + sqle); + } finally { + super.tearDown(); + } + } + /** * Create the argv to pass to Sqoop. * @@ -220,9 +246,7 @@ public class SQLServerParseMethodsManualTest extends ImportJobTestCase { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java index 0057ac9..d891c2b 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java @@ -25,14 +25,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; @@ -50,7 +47,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test that --query works in Sqoop. + * Test that --query works in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerQueryManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerQueryManualTest extends ImportJobTestCase { @@ -244,9 +258,7 @@ public class SQLServerQueryManualTest extends ImportJobTestCase { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java index f85245a..67e2cae 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java @@ -25,14 +25,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; @@ -50,7 +47,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test that --split-by works. + * Test that --split-by works in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerSplitByManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerSplitByManualTest extends ImportJobTestCase { @@ -213,9 +227,7 @@ public class SQLServerSplitByManualTest extends ImportJobTestCase { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } protected String getTableName() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/0ca73d4e/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java index 10ae03b..700fbba 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java @@ -25,18 +25,14 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestUtils.*; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; -import com.cloudera.sqoop.manager.JdbcMySQLExportTest; import com.cloudera.sqoop.orm.CompilationManager; import com.cloudera.sqoop.testutil.CommonArgs; import com.cloudera.sqoop.testutil.ImportJobTestCase; @@ -51,8 +47,24 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test that --where works in Sqoop. Methods essentially copied out of the other - * Test* classes. + * Test that --where works in SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerWhereManualTest. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons). + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. */ public class SQLServerWhereManualTest extends ImportJobTestCase { @@ -235,9 +247,7 @@ public class SQLServerWhereManualTest extends ImportJobTestCase { } protected String getConnectString() { - return System.getProperty( - "sqoop.test.sqlserver.connectstring.host_url", - "jdbc:sqlserver://sqlserverhost:1433"); + return MSSQLTestUtils.getDBConnectString(); } /**
