http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java deleted file mode 100644 index 9c20bca..0000000 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java +++ /dev/null @@ -1,304 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.manager.sqlserver; - -import java.io.BufferedReader; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; -import com.cloudera.sqoop.Sqoop; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.orm.CompilationManager; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.tool.ImportTool; -import com.cloudera.sqoop.util.ClassLoaderStack; -import org.junit.Ignore; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Test to import delimited file from SQL Server. - * - * This uses JDBC to import data from an SQLServer database to HDFS. - * - * Since this requires an SQLServer installation, - * this class is named in such a way that Sqoop's default QA process does - * not run it. You need to run this manually with - * -Dtestcase=SQLServerDatatypeImportDelimitedFileManualTest. - * - * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location - * where Sqoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install SQL Server Express 2012 - * Create a database SQOOPTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * access for SQOOPTEST to SQOOPUSER. - */ -public class SQLServerDatatypeImportDelimitedFileManualTest - extends SQLServerDatatypeImportSequenceFileManualTest { - -/** - * Create the argv to pass to Sqoop. - * - * @param includeHadoopFlags - * if true, then include -D various.settings=values - * @param colNames - * the columns to import. If null, all columns are used. - * @param conf - * a Configuration specifying additional properties to use when - * determining the arguments. - * @return the argv as an array of strings. -*/ - protected String[] getArgv(boolean includeHadoopFlags, String[] colNames, - Configuration conf) { - if (null == colNames) { - colNames = getColNames(); - } - - String splitByCol = colNames[0]; - String columnsString = ""; - for (String col : colNames) { - columnsString += col + ","; - } - - ArrayList<String> args = new ArrayList<String>(); - - if (includeHadoopFlags) { - CommonArgs.addHadoopFlags(args); - } - - args.add("--table"); - args.add(getTableName()); - args.add("--columns"); - args.add(columnsString); - args.add("--split-by"); - args.add(splitByCol); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(MSSQLTestUtils.getDBConnectString()); - - args.add("--num-mappers"); - args.add("2"); - - args.addAll(getExtraArgs(conf)); - - return args.toArray(new String[0]); - } - - - private void runSqoopImport(String[] importCols) { - Configuration conf = getConf(); - SqoopOptions opts = getSqoopOptions(conf); - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - opts.setUsername(username); - opts.setPassword(password); - - // run the tool through the normal entry-point. - int ret; - try { - Sqoop importer = new Sqoop(new ImportTool(), conf, opts); - ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf)); - } catch (Exception e) { - LOG.error("Got exception running Sqoop: " + e.toString()); - throw new RuntimeException(e); - } - - // expect a successful return. - assertEquals("Failure during job", 0, ret); - } - - /** - * Do a MapReduce-based import of the table and verify that the results were - * imported as expected. (tests readFields(ResultSet) and toString()) - * - * @param expectedVal - * the value we injected into the table. - * @param importCols - * the columns to import. If null, all columns are used. - */ - protected void verifyImport(String expectedVal, String[] importCols) { - - // paths to where our output file will wind up. - Path tableDirPath = getTablePath(); - - removeTableDir(); - - runSqoopImport(importCols); - Configuration conf = getConf(); - - SqoopOptions opts = getSqoopOptions(conf); - try { - ImportTool importTool = new ImportTool(); - opts = importTool.parseArguments(getArgv(false, importCols, conf), - conf, opts, true); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - fail(e.toString()); - } - - CompilationManager compileMgr = new CompilationManager(opts); - String jarFileName = compileMgr.getJarFilename(); - ClassLoader prevClassLoader = null; - try { - prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, - getTableName()); - - // Now open and check all part-files in the table path until we find - // a non-empty one that we can verify contains the value. - - FileSystem fs = FileSystem.getLocal(conf); - FileStatus[] stats = fs.listStatus(tableDirPath); - - if (stats == null || stats.length == 0) { - fail("Error: no files in " + tableDirPath); - } - - boolean foundRecord = false; - for (FileStatus stat : stats) { - if (!stat.getPath().getName().startsWith("part-") - && !stat.getPath().getName().startsWith("data-")) { - // This isn't a data file. Ignore it. - continue; - } - - try { - String line; - String fname = stat.getPath().toString(); - fname = fname.substring(5, fname.length()); - - BufferedReader reader = new BufferedReader( - new InputStreamReader(new FileInputStream(new File( - fname)))); - try { - line = reader.readLine(); - assertEquals(" expected a different string", - expectedVal, line); - } finally { - IOUtils.closeStream(reader); - } - LOG.info("Read back from delimited file: " + line); - foundRecord = true; - // Add trailing '\n' to expected value since - // SqoopRecord.toString() - // encodes the record delim. - if (null == expectedVal) { - assertEquals("Error validating result from delimited file", - "null\n", line); - } - } catch (EOFException eoe) { - // EOF in a file isn't necessarily a problem. We may have - // some - // empty sequence files, which will throw this. Just - // continue - // in the loop. - } - } - - if (!foundRecord) { - fail("Couldn't read any records from delimited file"); - } - } catch (IOException ioe) { - LOG.error(StringUtils.stringifyException(ioe)); - fail("IOException: " + ioe.toString()); - } finally { - if (null != prevClassLoader) { - ClassLoaderStack.setCurrentClassLoader(prevClassLoader); - } - } - } - - @Test - public void testTime() { - if (!supportsTime()) { - skipped = true; - return; - } - - dataTypeTest(DATATYPES.TIME); - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testVarBinary() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testBit() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testBit2() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testBit3() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testNChar() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testChar() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testVarchar() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testNVarchar() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testBinary() { - } - - @Ignore("Ignored as used type is not supported for table splitting.") - @Test - public void testTimestamp3() { - } - - public String getResportFileName(){ - return this.getClass().toString()+".txt"; - } -}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java new file mode 100644 index 0000000..a4d1822 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java @@ -0,0 +1,306 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.manager.sqlserver; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; +import com.cloudera.sqoop.Sqoop; +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.orm.CompilationManager; +import com.cloudera.sqoop.testutil.CommonArgs; +import com.cloudera.sqoop.tool.ImportTool; +import com.cloudera.sqoop.util.ClassLoaderStack; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Test to import delimited file from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeImportDelimitedFileTest or -Dthirdparty=true. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir. + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. + * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and + * -Dms.sqlserver.password + */ +public class SQLServerDatatypeImportDelimitedFileTest + extends SQLServerDatatypeImportSequenceFileTest { + +/** + * Create the argv to pass to Sqoop. + * + * @param includeHadoopFlags + * if true, then include -D various.settings=values + * @param colNames + * the columns to import. If null, all columns are used. + * @param conf + * a Configuration specifying additional properties to use when + * determining the arguments. + * @return the argv as an array of strings. +*/ + protected String[] getArgv(boolean includeHadoopFlags, String[] colNames, + Configuration conf) { + if (null == colNames) { + colNames = getColNames(); + } + + String splitByCol = colNames[0]; + String columnsString = ""; + for (String col : colNames) { + columnsString += col + ","; + } + + ArrayList<String> args = new ArrayList<String>(); + + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + } + + args.add("--table"); + args.add(getTableName()); + args.add("--columns"); + args.add(columnsString); + args.add("--split-by"); + args.add(splitByCol); + args.add("--warehouse-dir"); + args.add(getWarehouseDir()); + args.add("--connect"); + args.add(MSSQLTestUtils.getDBConnectString()); + + args.add("--num-mappers"); + args.add("2"); + + args.addAll(getExtraArgs(conf)); + + return args.toArray(new String[0]); + } + + + private void runSqoopImport(String[] importCols) { + Configuration conf = getConf(); + SqoopOptions opts = getSqoopOptions(conf); + String username = MSSQLTestUtils.getDBUserName(); + String password = MSSQLTestUtils.getDBPassWord(); + opts.setUsername(username); + opts.setPassword(password); + + // run the tool through the normal entry-point. + int ret; + try { + Sqoop importer = new Sqoop(new ImportTool(), conf, opts); + ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf)); + } catch (Exception e) { + LOG.error("Got exception running Sqoop: " + e.toString()); + throw new RuntimeException(e); + } + + // expect a successful return. + assertEquals("Failure during job", 0, ret); + } + + /** + * Do a MapReduce-based import of the table and verify that the results were + * imported as expected. (tests readFields(ResultSet) and toString()) + * + * @param expectedVal + * the value we injected into the table. + * @param importCols + * the columns to import. If null, all columns are used. + */ + protected void verifyImport(String expectedVal, String[] importCols) { + + // paths to where our output file will wind up. + Path tableDirPath = getTablePath(); + + removeTableDir(); + + runSqoopImport(importCols); + Configuration conf = getConf(); + + SqoopOptions opts = getSqoopOptions(conf); + try { + ImportTool importTool = new ImportTool(); + opts = importTool.parseArguments(getArgv(false, importCols, conf), + conf, opts, true); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + fail(e.toString()); + } + + CompilationManager compileMgr = new CompilationManager(opts); + String jarFileName = compileMgr.getJarFilename(); + ClassLoader prevClassLoader = null; + try { + prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, + getTableName()); + + // Now open and check all part-files in the table path until we find + // a non-empty one that we can verify contains the value. + + FileSystem fs = FileSystem.getLocal(conf); + FileStatus[] stats = fs.listStatus(tableDirPath); + + if (stats == null || stats.length == 0) { + fail("Error: no files in " + tableDirPath); + } + + boolean foundRecord = false; + for (FileStatus stat : stats) { + if (!stat.getPath().getName().startsWith("part-") + && !stat.getPath().getName().startsWith("data-")) { + // This isn't a data file. Ignore it. + continue; + } + + try { + String line; + String fname = stat.getPath().toString(); + fname = fname.substring(5, fname.length()); + + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(new File( + fname)))); + try { + line = reader.readLine(); + assertEquals(" expected a different string", + expectedVal, line); + } finally { + IOUtils.closeStream(reader); + } + LOG.info("Read back from delimited file: " + line); + foundRecord = true; + // Add trailing '\n' to expected value since + // SqoopRecord.toString() + // encodes the record delim. + if (null == expectedVal) { + assertEquals("Error validating result from delimited file", + "null\n", line); + } + } catch (EOFException eoe) { + // EOF in a file isn't necessarily a problem. We may have + // some + // empty sequence files, which will throw this. Just + // continue + // in the loop. + } + } + + if (!foundRecord) { + fail("Couldn't read any records from delimited file"); + } + } catch (IOException ioe) { + LOG.error(StringUtils.stringifyException(ioe)); + fail("IOException: " + ioe.toString()); + } finally { + if (null != prevClassLoader) { + ClassLoaderStack.setCurrentClassLoader(prevClassLoader); + } + } + } + + @Test + public void testTime() { + if (!supportsTime()) { + skipped = true; + return; + } + + dataTypeTest(DATATYPES.TIME); + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testVarBinary() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit2() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBit3() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testNChar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testChar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testVarchar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testNVarchar() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testBinary() { + } + + @Ignore("Ignored as used type is not supported for table splitting.") + @Test + public void testTimestamp3() { + } + + public String getResportFileName(){ + return this.getClass().toString()+".txt"; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java deleted file mode 100644 index 9cd3176..0000000 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java +++ /dev/null @@ -1,845 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.manager.sqlserver; - -import java.io.FileWriter; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.ManagerCompatTestCase; -import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; -import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test importing sequence file from SQL Server. - * - * This uses JDBC to import data from an SQLServer database to HDFS. - * - * Since this requires an SQLServer installation, - * this class is named in such a way that Sqoop's default QA process does - * not run it. You need to run this manually with - * -Dtestcase=SQLServerDatatypeImportSequenceFileManualTest. - * - * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location - * where Sqoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install SQL Server Express 2012 - * Create a database SQOOPTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * access for SQOOPTEST to SQOOPUSER. - */ -public class SQLServerDatatypeImportSequenceFileManualTest extends - ManagerCompatTestCase { - - public static final Log LOG = LogFactory.getLog( - SQLServerDatatypeImportSequenceFileManualTest.class.getName()); - private static MSSQLTestDataFileParser tdfs; - private static Map report; - - static { - try { - - String testfile = null; - testfile = System.getProperty("test.data.dir") - + "/" + System.getProperty("ms.datatype.test.data.file.import"); - String delim = System.getProperty("ms.datatype.test.data.file.delim", ","); - System.out.println("Using data file : " + testfile); - LOG.info("Using data file : " + testfile); - tdfs = new MSSQLTestDataFileParser(testfile); - tdfs.setDelim(delim); - tdfs.parse(); - report = new HashMap(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - System.out - .println("Error with test data file, check stack trace for cause" - + ".\nTests cannont continue."); - System.exit(0); - } - } - - @Override - protected String getDbFriendlyName() { - return "MSSQL"; - } - - @Override - protected Log getLogger() { - return LOG; - } - - protected boolean useHsqldbTestServer() { - return false; - } - - protected String getConnectString() { - return MSSQLTestUtils.getDBConnectString(); - } - - /** - * Drop a table if it already exists in the database. - * - * @param table - * the name of the table to drop. - * @throws SQLException - * if something goes wrong. - */ - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - String sqlStmt = "IF OBJECT_ID('" + table - + "') IS NOT NULL DROP TABLE " + table; - - PreparedStatement statement = conn.prepareStatement(sqlStmt, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } - - protected SqoopOptions getSqoopOptions(Configuration conf) { - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(username); - opts.setPassword(password); - return opts; - } - - @Before - public void setUp() { - try { - super.setUp(); - } catch (Exception e) { - try { - FileWriter fr = new FileWriter(getResportFileName(), true); - String res = removeNewLines(e.getMessage()); - fr.append("Error\t" + res + "\n"); - fr.close(); - } catch (Exception e2) { - LOG.error(StringUtils.stringifyException(e2)); - fail(e2.toString()); - } - } catch (Error e) { - try { - FileWriter fr = new FileWriter(getResportFileName(), true); - - String res = removeNewLines(e.getMessage()); - - fr.append("Error\t" + res + "\n"); - fr.close(); - fail(res); - } catch (Exception e2) { - LOG.error(StringUtils.stringifyException(e2)); - fail(e2.toString()); - } - } - } - - @After - public void tearDown() { - try { - super.tearDown(); - } catch (Exception e) { - try { - FileWriter fr = new FileWriter(getResportFileName(), true); - String res = removeNewLines(e.getMessage()); - fr.append("Error\t" + res + "\n"); - fr.close(); - } catch (Exception e2) { - LOG.error(StringUtils.stringifyException(e2)); - fail(e2.toString()); - } - } catch (Error e) { - try { - FileWriter fr = new FileWriter(getResportFileName(), true); - String res = removeNewLines(e.getMessage()); - fr.append("Error\t" + res + "\n"); - fr.close(); - fail(res); - } catch (Exception e2) { - LOG.error(StringUtils.stringifyException(e2)); - fail(e2.toString()); - } - } - } - - protected boolean supportsBoolean() { - return true; - } - - @Test - public void testBit() { - if (!supportsBoolean()) { - skipped = true; - return; - } - verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput()); - } - - @Test - public void testBit2() { - if (!supportsBoolean()) { - skipped = true; - return; - } - verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput()); - } - - @Test - public void testBit3() { - if (!supportsBoolean()) { - skipped = true; - return; - } - verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput()); - } - - @Test - public void testBoolean() { - try { - super.testBoolean(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testBoolean2() { - try { - super.testBoolean2(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testBoolean3() { - try { - super.testBoolean3(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testDouble1() { - try { - super.testDouble1(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testDouble2() { - try { - super.testDouble2(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testClob1() { - try { - super.testClob1(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testBlob1() { - try { - super.testBlob1(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testLongVarChar() { - try { - super.testLongVarChar(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testTimestamp1() { - try { - super.testTimestamp1(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testTimestamp2() { - try { - super.testTimestamp2(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testTimestamp3() { - try { - super.testTimestamp3(); - assertTrue("This test should not pass on sql server", false); - } catch (AssertionError a) { - System.out.println("Test failed, this was expected"); - } - } - - @Test - public void testVarBinary() { - if (!supportsVarBinary()) { - return; - } - dataTypeTest(DATATYPES.VARBINARY); - } - - @Test - public void testTime() { - if (!supportsTime()) { - skipped = true; - return; - } - dataTypeTest(DATATYPES.TIME); - } - - @Test - public void testSmalldatetime() { - if (!supportsTime()) { - skipped = true; - return; - } - dataTypeTest(DATATYPES.SMALLDATETIME); - } - - @Test - public void testdatetime2() { - if (!supportsTime()) { - skipped = true; - return; - } - dataTypeTest(DATATYPES.DATETIME2); - } - - @Test - public void testdatetime() { - if (!supportsTime()) { - skipped = true; - return; - } - dataTypeTest(DATATYPES.DATETIME); - } - - @Test - public void testdatetimeoffset() { - if (!supportsTime()) { - skipped = true; - return; - } - dataTypeTest(DATATYPES.DATETIMEOFFSET); - } - - @Test - public void testDecimal() { - dataTypeTest(DATATYPES.DECIMAL); - } - - @Test - public void testNumeric() { - dataTypeTest(DATATYPES.NUMERIC); - } - - @Test - public void testNumeric1() { - } - - @Test - public void testNumeric2() { - } - - @Test - public void testDecimal1() { - } - - @Test - public void testDecimal2() { - } - - @Test - public void testBigInt() { - dataTypeTest(DATATYPES.BIGINT); - } - - @Test - public void testBigInt1() { - } - - @Test - public void testInt() { - dataTypeTest(DATATYPES.INT); - } - - @Test - public void testSmallInt() { - dataTypeTest(DATATYPES.SMALLINT); - } - - @Test - public void testSmallInt1() { - } - - @Test - public void testSmallInt2() { - } - - @Test - public void testTinyint() { - dataTypeTest(DATATYPES.TINYINT); - - } - - @Test - public void testTinyInt1() { - } - - @Test - public void testTinyInt2() { - } - - @Test - public void testFloat() { - dataTypeTest(DATATYPES.FLOAT); - } - - @Test - public void testReal() { - dataTypeTest(DATATYPES.REAL); - } - - @Test - public void testDate() { - dataTypeTest(DATATYPES.DATE); - } - - @Test - public void testMoney() { - dataTypeTest(DATATYPES.MONEY); - } - - @Test - public void testSmallMoney() { - dataTypeTest(DATATYPES.SMALLMONEY); - } - - @Test - public void testText() { - dataTypeTest(DATATYPES.TEXT); - } - - @Test - public void testVarchar() { - dataTypeTest(DATATYPES.VARCHAR); - } - - @Test - public void testChar() { - dataTypeTest(DATATYPES.CHAR); - } - - @Test - public void testNText() { - dataTypeTest(DATATYPES.NTEXT); - } - - @Test - public void testNChar() { - dataTypeTest(DATATYPES.NCHAR); - } - - @Test - public void testNVarchar() { - dataTypeTest(DATATYPES.NVARCHAR); - } - - @Test - public void testImage() { - dataTypeTest(DATATYPES.IMAGE); - } - - @Test - public void testBinary() { - dataTypeTest(DATATYPES.BINARY); - } - - //---------------disabled tests----- - @Test - public void testTime1() { - } - - @Test - public void testTime2() { - } - - @Test - public void testTime3() { - } - - @Test - public void testTime4() { - } - - @Test - public void testStringCol1() { - - } - - @Test - public void testStringCol2() { - - } - - @Test - public void testEmptyStringCol() { - - } - - @Test - public void testNullStringCol() { - - } - - @Test - public void testNullInt() { - - } - - @Test - public void testReal1() { - - } - - @Test - public void testReal2() { - - } - - @Test - public void testFloat1() { - - } - - @Test - public void testFloat2() { - - } - - @Test - public void testDate1() { - - } - - @Test - public void testDate2() { - - } - - - @Test - public void testNumeric3() { - - } - - @Test - public void testNumeric4() { - - } - - @Test - public void testNumeric5() { - - - } - - @Test - public void testNumeric6() { - - } - - - - @Test - public void testDecimal3() { - - } - - @Test - public void testDecimal4() { - - } - - @Test - public void testDecimal5() { - - - } - - @Test - public void testDecimal6() { - - } - - - - //end disabled tests---------------------------- - - public String getTrueBoolDbOutput() { - return "1"; - } - - public String getFalseBoolDbOutput() { - return "0"; - } - - protected String getFalseBoolSeqOutput() { - return "false"; - } - - protected String getFalseBoolLiteralSqlInput() { - return "0"; - } - - protected String getFixedCharSeqOut(int len, String val) { - return val + nSpace(len - val.length()); - } - - protected String getFixedCharDbOut(int len, String val) { - return val + nSpace(len - val.length()); - } - - public String nSpace(int n) { - String tmp = ""; - for (int i = 0; i < n; i++) { - tmp += " "; - } - - return tmp; - } - - public String nZeros(int n) { - String tmp = ""; - for (int i = 0; i < n; i++) { - tmp += "0"; - } - - return tmp; - } - - public void dataTypeTest(DATATYPES datatype) { - int exceptionCount = 0; - - List testdata = tdfs.getTestdata(datatype); - - for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) { - MSSQLTestData current = itr.next(); - System.out.println("Testing with : \n" + current); - - try { - if (datatype == DATATYPES.DECIMAL - || datatype == DATATYPES.NUMERIC) { - - verifyType(current.getDatatype() + "(" - + current.getData(KEY_STRINGS.SCALE) + "," - + current.getData(KEY_STRINGS.PREC) + ")", current - .getData(KEY_STRINGS.TO_INSERT), current - .getData(KEY_STRINGS.HDFS_READBACK)); - - } else if (datatype == DATATYPES.TIME - || datatype == DATATYPES.SMALLDATETIME - || datatype == DATATYPES.DATETIME2 - || datatype == DATATYPES.DATETIME - || datatype == DATATYPES.DATETIMEOFFSET - || datatype == DATATYPES.TEXT - || datatype == DATATYPES.NTEXT - || datatype == DATATYPES.DATE) { - verifyType(current.getDatatype(), "'" - + current.getData(KEY_STRINGS.TO_INSERT) + "'", current - .getData(KEY_STRINGS.HDFS_READBACK)); - } else if (datatype == DATATYPES.VARBINARY) { - verifyType( - current.getDatatype() + "(" - + current.getData(KEY_STRINGS.SCALE) + ")", - "cast('" + current.getData(KEY_STRINGS.TO_INSERT) - + "' as varbinary(" - + current.getData(KEY_STRINGS.SCALE) + "))", - current.getData(KEY_STRINGS.HDFS_READBACK)); - } else if (datatype == DATATYPES.BINARY) { - verifyType( - current.getDatatype() + "(" - + current.getData(KEY_STRINGS.SCALE) + ")", - "cast('" + current.getData(KEY_STRINGS.TO_INSERT) - + "' as binary(" - + current.getData(KEY_STRINGS.SCALE) + "))", - current.getData(KEY_STRINGS.HDFS_READBACK)); - } else if (datatype == DATATYPES.NCHAR - || datatype == DATATYPES.VARCHAR - || datatype == DATATYPES.CHAR - || datatype == DATATYPES.NVARCHAR) { - System.out.println("------>" - + current.getData(KEY_STRINGS.DB_READBACK) - + "<----"); - verifyType(current.getDatatype() + "(" - + current.getData(KEY_STRINGS.SCALE) + ")", "'" - + current.getData(KEY_STRINGS.TO_INSERT) + "'", - current.getData(KEY_STRINGS.HDFS_READBACK)); - } else if (datatype == DATATYPES.IMAGE) { - verifyType(current.getDatatype(), "cast('" - + current.getData(KEY_STRINGS.TO_INSERT) - + "' as image )", - current.getData(KEY_STRINGS.HDFS_READBACK)); - } else { - verifyType(current.getDatatype(), current - .getData(KEY_STRINGS.TO_INSERT), current - .getData(KEY_STRINGS.HDFS_READBACK)); - } - - addToReport(current, null); - - } catch (AssertionError ae) { - if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) { - System.out.println("failure was expected, PASS"); - addToReport(current, null); - } else { - System.out - .println("----------------------------------------------------------" - + "-"); - System.out.println("Failure for following Test Data :\n" - + current.toString()); - System.out - .println("----------------------------------------------------------" - + "-"); - System.out.println("Exception details : \n"); - System.out.println(ae.getMessage()); - System.out - .println("----------------------------------------------------------" - + "-"); - addToReport(current, ae); - exceptionCount++; - } - } catch (Exception e) { - addToReport(current, e); - exceptionCount++; - } - } - - if (exceptionCount > 0) { - System.out.println("There were failures for :" - + datatype.toString()); - System.out.println("Failed for " + exceptionCount - + " test data samples\n"); - System.out.println("Sroll up for detailed errors"); - System.out - .println("-----------------------------------------------------------"); - throw new AssertionError("Failed for " + exceptionCount - + " test data sample"); - } - } - - public synchronized void addToReport(MSSQLTestData td, Object result) { - System.out.println("called"); - try { - FileWriter fr = new FileWriter(getResportFileName(), true); - String offset = td.getData(KEY_STRINGS.OFFSET); - String res = "_"; - if (result == null) { - res = "Success"; - } else { - try { - res = "FAILED " - + removeNewLines(((AssertionError) result) - .getMessage()); - } catch (Exception ae) { - if (result instanceof Exception - && ((Exception) result) != null) { - res = "FAILED " - + removeNewLines(((Exception) result) - .getMessage()); - } else { - res = "FAILED " + result.toString(); - } - } - } - - fr.append(offset + "\t" + res + "\n"); - fr.close(); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } - } - - public static String removeNewLines(String str) { - String[] tmp = str.split("\n"); - String result = ""; - for (String a : tmp) { - result += " " + a; - } - return result; - } - - public String getResportFileName(){ - return this.getClass().toString()+".txt"; - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java new file mode 100644 index 0000000..409c4ad --- /dev/null +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java @@ -0,0 +1,847 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.manager.sqlserver; + +import java.io.FileWriter; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.testutil.ManagerCompatTestCase; +import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES; +import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test importing sequence file from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerDatatypeImportSequenceFileTest or -Dthirdparty=true. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir. + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. + * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and + * -Dms.sqlserver.password + */ +public class SQLServerDatatypeImportSequenceFileTest extends + ManagerCompatTestCase { + + public static final Log LOG = LogFactory.getLog( + SQLServerDatatypeImportSequenceFileTest.class.getName()); + private static MSSQLTestDataFileParser tdfs; + private static Map report; + + static { + try { + + String testfile = null; + testfile = System.getProperty("test.data.dir") + + "/" + System.getProperty("ms.datatype.test.data.file.import"); + String delim = System.getProperty("ms.datatype.test.data.file.delim", ","); + System.out.println("Using data file : " + testfile); + LOG.info("Using data file : " + testfile); + tdfs = new MSSQLTestDataFileParser(testfile); + tdfs.setDelim(delim); + tdfs.parse(); + report = new HashMap(); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + System.out + .println("Error with test data file, check stack trace for cause" + + ".\nTests cannont continue."); + System.exit(0); + } + } + + @Override + protected String getDbFriendlyName() { + return "MSSQL"; + } + + @Override + protected Log getLogger() { + return LOG; + } + + protected boolean useHsqldbTestServer() { + return false; + } + + protected String getConnectString() { + return MSSQLTestUtils.getDBConnectString(); + } + + /** + * Drop a table if it already exists in the database. + * + * @param table + * the name of the table to drop. + * @throws SQLException + * if something goes wrong. + */ + protected void dropTableIfExists(String table) throws SQLException { + Connection conn = getManager().getConnection(); + String sqlStmt = "IF OBJECT_ID('" + table + + "') IS NOT NULL DROP TABLE " + table; + + PreparedStatement statement = conn.prepareStatement(sqlStmt, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + try { + statement.executeUpdate(); + conn.commit(); + } finally { + statement.close(); + } + } + + protected SqoopOptions getSqoopOptions(Configuration conf) { + String username = MSSQLTestUtils.getDBUserName(); + String password = MSSQLTestUtils.getDBPassWord(); + SqoopOptions opts = new SqoopOptions(conf); + opts.setUsername(username); + opts.setPassword(password); + return opts; + } + + @Before + public void setUp() { + try { + super.setUp(); + } catch (Exception e) { + try { + FileWriter fr = new FileWriter(getResportFileName(), true); + String res = removeNewLines(e.getMessage()); + fr.append("Error\t" + res + "\n"); + fr.close(); + } catch (Exception e2) { + LOG.error(StringUtils.stringifyException(e2)); + fail(e2.toString()); + } + } catch (Error e) { + try { + FileWriter fr = new FileWriter(getResportFileName(), true); + + String res = removeNewLines(e.getMessage()); + + fr.append("Error\t" + res + "\n"); + fr.close(); + fail(res); + } catch (Exception e2) { + LOG.error(StringUtils.stringifyException(e2)); + fail(e2.toString()); + } + } + } + + @After + public void tearDown() { + try { + super.tearDown(); + } catch (Exception e) { + try { + FileWriter fr = new FileWriter(getResportFileName(), true); + String res = removeNewLines(e.getMessage()); + fr.append("Error\t" + res + "\n"); + fr.close(); + } catch (Exception e2) { + LOG.error(StringUtils.stringifyException(e2)); + fail(e2.toString()); + } + } catch (Error e) { + try { + FileWriter fr = new FileWriter(getResportFileName(), true); + String res = removeNewLines(e.getMessage()); + fr.append("Error\t" + res + "\n"); + fr.close(); + fail(res); + } catch (Exception e2) { + LOG.error(StringUtils.stringifyException(e2)); + fail(e2.toString()); + } + } + } + + protected boolean supportsBoolean() { + return true; + } + + @Test + public void testBit() { + if (!supportsBoolean()) { + skipped = true; + return; + } + verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput()); + } + + @Test + public void testBit2() { + if (!supportsBoolean()) { + skipped = true; + return; + } + verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput()); + } + + @Test + public void testBit3() { + if (!supportsBoolean()) { + skipped = true; + return; + } + verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput()); + } + + @Test + public void testBoolean() { + try { + super.testBoolean(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testBoolean2() { + try { + super.testBoolean2(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testBoolean3() { + try { + super.testBoolean3(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testDouble1() { + try { + super.testDouble1(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testDouble2() { + try { + super.testDouble2(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testClob1() { + try { + super.testClob1(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testBlob1() { + try { + super.testBlob1(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testLongVarChar() { + try { + super.testLongVarChar(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testTimestamp1() { + try { + super.testTimestamp1(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testTimestamp2() { + try { + super.testTimestamp2(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testTimestamp3() { + try { + super.testTimestamp3(); + assertTrue("This test should not pass on sql server", false); + } catch (AssertionError a) { + System.out.println("Test failed, this was expected"); + } + } + + @Test + public void testVarBinary() { + if (!supportsVarBinary()) { + return; + } + dataTypeTest(DATATYPES.VARBINARY); + } + + @Test + public void testTime() { + if (!supportsTime()) { + skipped = true; + return; + } + dataTypeTest(DATATYPES.TIME); + } + + @Test + public void testSmalldatetime() { + if (!supportsTime()) { + skipped = true; + return; + } + dataTypeTest(DATATYPES.SMALLDATETIME); + } + + @Test + public void testdatetime2() { + if (!supportsTime()) { + skipped = true; + return; + } + dataTypeTest(DATATYPES.DATETIME2); + } + + @Test + public void testdatetime() { + if (!supportsTime()) { + skipped = true; + return; + } + dataTypeTest(DATATYPES.DATETIME); + } + + @Test + public void testdatetimeoffset() { + if (!supportsTime()) { + skipped = true; + return; + } + dataTypeTest(DATATYPES.DATETIMEOFFSET); + } + + @Test + public void testDecimal() { + dataTypeTest(DATATYPES.DECIMAL); + } + + @Test + public void testNumeric() { + dataTypeTest(DATATYPES.NUMERIC); + } + + @Test + public void testNumeric1() { + } + + @Test + public void testNumeric2() { + } + + @Test + public void testDecimal1() { + } + + @Test + public void testDecimal2() { + } + + @Test + public void testBigInt() { + dataTypeTest(DATATYPES.BIGINT); + } + + @Test + public void testBigInt1() { + } + + @Test + public void testInt() { + dataTypeTest(DATATYPES.INT); + } + + @Test + public void testSmallInt() { + dataTypeTest(DATATYPES.SMALLINT); + } + + @Test + public void testSmallInt1() { + } + + @Test + public void testSmallInt2() { + } + + @Test + public void testTinyint() { + dataTypeTest(DATATYPES.TINYINT); + + } + + @Test + public void testTinyInt1() { + } + + @Test + public void testTinyInt2() { + } + + @Test + public void testFloat() { + dataTypeTest(DATATYPES.FLOAT); + } + + @Test + public void testReal() { + dataTypeTest(DATATYPES.REAL); + } + + @Test + public void testDate() { + dataTypeTest(DATATYPES.DATE); + } + + @Test + public void testMoney() { + dataTypeTest(DATATYPES.MONEY); + } + + @Test + public void testSmallMoney() { + dataTypeTest(DATATYPES.SMALLMONEY); + } + + @Test + public void testText() { + dataTypeTest(DATATYPES.TEXT); + } + + @Test + public void testVarchar() { + dataTypeTest(DATATYPES.VARCHAR); + } + + @Test + public void testChar() { + dataTypeTest(DATATYPES.CHAR); + } + + @Test + public void testNText() { + dataTypeTest(DATATYPES.NTEXT); + } + + @Test + public void testNChar() { + dataTypeTest(DATATYPES.NCHAR); + } + + @Test + public void testNVarchar() { + dataTypeTest(DATATYPES.NVARCHAR); + } + + @Test + public void testImage() { + dataTypeTest(DATATYPES.IMAGE); + } + + @Test + public void testBinary() { + dataTypeTest(DATATYPES.BINARY); + } + + //---------------disabled tests----- + @Test + public void testTime1() { + } + + @Test + public void testTime2() { + } + + @Test + public void testTime3() { + } + + @Test + public void testTime4() { + } + + @Test + public void testStringCol1() { + + } + + @Test + public void testStringCol2() { + + } + + @Test + public void testEmptyStringCol() { + + } + + @Test + public void testNullStringCol() { + + } + + @Test + public void testNullInt() { + + } + + @Test + public void testReal1() { + + } + + @Test + public void testReal2() { + + } + + @Test + public void testFloat1() { + + } + + @Test + public void testFloat2() { + + } + + @Test + public void testDate1() { + + } + + @Test + public void testDate2() { + + } + + + @Test + public void testNumeric3() { + + } + + @Test + public void testNumeric4() { + + } + + @Test + public void testNumeric5() { + + + } + + @Test + public void testNumeric6() { + + } + + + + @Test + public void testDecimal3() { + + } + + @Test + public void testDecimal4() { + + } + + @Test + public void testDecimal5() { + + + } + + @Test + public void testDecimal6() { + + } + + + + //end disabled tests---------------------------- + + public String getTrueBoolDbOutput() { + return "1"; + } + + public String getFalseBoolDbOutput() { + return "0"; + } + + protected String getFalseBoolSeqOutput() { + return "false"; + } + + protected String getFalseBoolLiteralSqlInput() { + return "0"; + } + + protected String getFixedCharSeqOut(int len, String val) { + return val + nSpace(len - val.length()); + } + + protected String getFixedCharDbOut(int len, String val) { + return val + nSpace(len - val.length()); + } + + public String nSpace(int n) { + String tmp = ""; + for (int i = 0; i < n; i++) { + tmp += " "; + } + + return tmp; + } + + public String nZeros(int n) { + String tmp = ""; + for (int i = 0; i < n; i++) { + tmp += "0"; + } + + return tmp; + } + + public void dataTypeTest(DATATYPES datatype) { + int exceptionCount = 0; + + List testdata = tdfs.getTestdata(datatype); + + for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) { + MSSQLTestData current = itr.next(); + System.out.println("Testing with : \n" + current); + + try { + if (datatype == DATATYPES.DECIMAL + || datatype == DATATYPES.NUMERIC) { + + verifyType(current.getDatatype() + "(" + + current.getData(KEY_STRINGS.SCALE) + "," + + current.getData(KEY_STRINGS.PREC) + ")", current + .getData(KEY_STRINGS.TO_INSERT), current + .getData(KEY_STRINGS.HDFS_READBACK)); + + } else if (datatype == DATATYPES.TIME + || datatype == DATATYPES.SMALLDATETIME + || datatype == DATATYPES.DATETIME2 + || datatype == DATATYPES.DATETIME + || datatype == DATATYPES.DATETIMEOFFSET + || datatype == DATATYPES.TEXT + || datatype == DATATYPES.NTEXT + || datatype == DATATYPES.DATE) { + verifyType(current.getDatatype(), "'" + + current.getData(KEY_STRINGS.TO_INSERT) + "'", current + .getData(KEY_STRINGS.HDFS_READBACK)); + } else if (datatype == DATATYPES.VARBINARY) { + verifyType( + current.getDatatype() + "(" + + current.getData(KEY_STRINGS.SCALE) + ")", + "cast('" + current.getData(KEY_STRINGS.TO_INSERT) + + "' as varbinary(" + + current.getData(KEY_STRINGS.SCALE) + "))", + current.getData(KEY_STRINGS.HDFS_READBACK)); + } else if (datatype == DATATYPES.BINARY) { + verifyType( + current.getDatatype() + "(" + + current.getData(KEY_STRINGS.SCALE) + ")", + "cast('" + current.getData(KEY_STRINGS.TO_INSERT) + + "' as binary(" + + current.getData(KEY_STRINGS.SCALE) + "))", + current.getData(KEY_STRINGS.HDFS_READBACK)); + } else if (datatype == DATATYPES.NCHAR + || datatype == DATATYPES.VARCHAR + || datatype == DATATYPES.CHAR + || datatype == DATATYPES.NVARCHAR) { + System.out.println("------>" + + current.getData(KEY_STRINGS.DB_READBACK) + + "<----"); + verifyType(current.getDatatype() + "(" + + current.getData(KEY_STRINGS.SCALE) + ")", "'" + + current.getData(KEY_STRINGS.TO_INSERT) + "'", + current.getData(KEY_STRINGS.HDFS_READBACK)); + } else if (datatype == DATATYPES.IMAGE) { + verifyType(current.getDatatype(), "cast('" + + current.getData(KEY_STRINGS.TO_INSERT) + + "' as image )", + current.getData(KEY_STRINGS.HDFS_READBACK)); + } else { + verifyType(current.getDatatype(), current + .getData(KEY_STRINGS.TO_INSERT), current + .getData(KEY_STRINGS.HDFS_READBACK)); + } + + addToReport(current, null); + + } catch (AssertionError ae) { + if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) { + System.out.println("failure was expected, PASS"); + addToReport(current, null); + } else { + System.out + .println("----------------------------------------------------------" + + "-"); + System.out.println("Failure for following Test Data :\n" + + current.toString()); + System.out + .println("----------------------------------------------------------" + + "-"); + System.out.println("Exception details : \n"); + System.out.println(ae.getMessage()); + System.out + .println("----------------------------------------------------------" + + "-"); + addToReport(current, ae); + exceptionCount++; + } + } catch (Exception e) { + addToReport(current, e); + exceptionCount++; + } + } + + if (exceptionCount > 0) { + System.out.println("There were failures for :" + + datatype.toString()); + System.out.println("Failed for " + exceptionCount + + " test data samples\n"); + System.out.println("Sroll up for detailed errors"); + System.out + .println("-----------------------------------------------------------"); + throw new AssertionError("Failed for " + exceptionCount + + " test data sample"); + } + } + + public synchronized void addToReport(MSSQLTestData td, Object result) { + System.out.println("called"); + try { + FileWriter fr = new FileWriter(getResportFileName(), true); + String offset = td.getData(KEY_STRINGS.OFFSET); + String res = "_"; + if (result == null) { + res = "Success"; + } else { + try { + res = "FAILED " + + removeNewLines(((AssertionError) result) + .getMessage()); + } catch (Exception ae) { + if (result instanceof Exception + && ((Exception) result) != null) { + res = "FAILED " + + removeNewLines(((Exception) result) + .getMessage()); + } else { + res = "FAILED " + result.toString(); + } + } + } + + fr.append(offset + "\t" + res + "\n"); + fr.close(); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + } + } + + public static String removeNewLines(String str) { + String[] tmp = str.split("\n"); + String result = ""; + for (String a : tmp) { + result += " " + a; + } + return result; + } + + public String getResportFileName(){ + return this.getClass().toString()+".txt"; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java deleted file mode 100644 index ab3dd08..0000000 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.manager.sqlserver; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.hive.TestHiveImport; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.tool.SqoopTool; -import org.junit.After; -import org.junit.Before; - -import static org.junit.Assert.fail; - -/** - * Test import to Hive from SQL Server. - * - * This uses JDBC to import data from an SQLServer database to HDFS. - * - * Since this requires an SQLServer installation, - * this class is named in such a way that Sqoop's default QA process does - * not run it. You need to run this manually with - * -Dtestcase=SQLServerHiveImportManualTest. - * - * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location - * where Sqoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install SQL Server Express 2012 - * Create a database SQOOPTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * access for SQOOPTEST to SQOOPUSER. - */ -public class SQLServerHiveImportManualTest extends TestHiveImport { - - @Before - public void setUp() { - super.setUp(); - } - - @After - public void tearDown() { - try { - dropTableIfExists(getTableName()); - } catch (SQLException sqle) { - LOG.info("Table clean-up failed: " + sqle); - } finally { - super.tearDown(); - } - } - - protected boolean useHsqldbTestServer() { - return false; - } - - protected String getConnectString() { - return MSSQLTestUtils.getDBConnectString(); - } - - //SQL Server pads out - @Override - protected String[] getTypes() { - String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" }; - return types; - } - - /** - * Drop a table if it already exists in the database. - * @param table - * the name of the table to drop. - * @throws SQLException - * if something goes wrong. - */ - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - String sqlStmt = "IF OBJECT_ID('" + table - + "') IS NOT NULL DROP TABLE " + table; - PreparedStatement statement = conn.prepareStatement(sqlStmt, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } - - protected SqoopOptions getSqoopOptions(Configuration conf) { - - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(username); - opts.setPassword(password); - - return opts; - - } - - SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) { - SqoopOptions opts = null; - try { - opts = tool.parseArguments(args, null, null, true); - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - opts.setUsername(username); - opts.setPassword(password); - - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - fail("Invalid options: " + e.toString()); - } - - return opts; - } - - protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) { - ArrayList<String> args = new ArrayList<String>(); - System.out.println("Overridden getArgv is called.."); - if (includeHadoopFlags) { - CommonArgs.addHadoopFlags(args); - } - - if (null != moreArgs) { - for (String arg : moreArgs) { - args.add(arg); - } - } - - args.add("--table"); - args.add(getTableName()); - args.add("--warehouse-dir"); - args.add(getWarehouseDir()); - args.add("--connect"); - args.add(getConnectString()); - args.add("--hive-import"); - String[] colNames = getColNames(); - if (null != colNames) { - args.add("--split-by"); - args.add(colNames[0]); - } else { - fail("Could not determine column names."); - } - - args.add("--num-mappers"); - args.add("1"); - - for (String a : args) { - LOG.debug("ARG : " + a); - } - - return args.toArray(new String[0]); - } - - protected String[] getCodeGenArgs() { - ArrayList<String> args = new ArrayList<String>(); - - args.add("--table"); - args.add(getTableName()); - args.add("--connect"); - args.add(getConnectString()); - args.add("--hive-import"); - - return args.toArray(new String[0]); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java new file mode 100644 index 0000000..535e599 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.manager.sqlserver; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.hive.TestHiveImport; +import com.cloudera.sqoop.testutil.CommonArgs; +import com.cloudera.sqoop.tool.SqoopTool; +import org.junit.After; +import org.junit.Before; + +import static org.junit.Assert.fail; + +/** + * Test import to Hive from SQL Server. + * + * This uses JDBC to import data from an SQLServer database to HDFS. + * + * Since this requires an SQLServer installation, + * this class is named in such a way that Sqoop's default QA process does + * not run it. You need to run this manually with + * -Dtestcase=SQLServerHiveImportTest or -Dthirdparty=true. + * + * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location + * where Sqoop will be able to access it (since this library cannot be checked + * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir. + * + * To set up your test environment: + * Install SQL Server Express 2012 + * Create a database SQOOPTEST + * Create a login SQOOPUSER with password PASSWORD and grant all + * access for SQOOPTEST to SQOOPUSER. + * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and + * -Dms.sqlserver.password + */ +public class SQLServerHiveImportTest extends TestHiveImport { + + @Before + public void setUp() { + super.setUp(); + } + + @After + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException sqle) { + LOG.info("Table clean-up failed: " + sqle); + } finally { + super.tearDown(); + } + } + + protected boolean useHsqldbTestServer() { + return false; + } + + protected String getConnectString() { + return MSSQLTestUtils.getDBConnectString(); + } + + //SQL Server pads out + @Override + protected String[] getTypes() { + String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" }; + return types; + } + + /** + * Drop a table if it already exists in the database. + * @param table + * the name of the table to drop. + * @throws SQLException + * if something goes wrong. + */ + protected void dropTableIfExists(String table) throws SQLException { + Connection conn = getManager().getConnection(); + String sqlStmt = "IF OBJECT_ID('" + table + + "') IS NOT NULL DROP TABLE " + table; + PreparedStatement statement = conn.prepareStatement(sqlStmt, + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + try { + statement.executeUpdate(); + conn.commit(); + } finally { + statement.close(); + } + } + + protected SqoopOptions getSqoopOptions(Configuration conf) { + + String username = MSSQLTestUtils.getDBUserName(); + String password = MSSQLTestUtils.getDBPassWord(); + SqoopOptions opts = new SqoopOptions(conf); + opts.setUsername(username); + opts.setPassword(password); + + return opts; + + } + + SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) { + SqoopOptions opts = null; + try { + opts = tool.parseArguments(args, null, null, true); + String username = MSSQLTestUtils.getDBUserName(); + String password = MSSQLTestUtils.getDBPassWord(); + opts.setUsername(username); + opts.setPassword(password); + + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + fail("Invalid options: " + e.toString()); + } + + return opts; + } + + protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) { + ArrayList<String> args = new ArrayList<String>(); + System.out.println("Overridden getArgv is called.."); + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + } + + if (null != moreArgs) { + for (String arg : moreArgs) { + args.add(arg); + } + } + + args.add("--table"); + args.add(getTableName()); + args.add("--warehouse-dir"); + args.add(getWarehouseDir()); + args.add("--connect"); + args.add(getConnectString()); + args.add("--hive-import"); + String[] colNames = getColNames(); + if (null != colNames) { + args.add("--split-by"); + args.add(colNames[0]); + } else { + fail("Could not determine column names."); + } + + args.add("--num-mappers"); + args.add("1"); + + for (String a : args) { + LOG.debug("ARG : " + a); + } + + return args.toArray(new String[0]); + } + + protected String[] getCodeGenArgs() { + ArrayList<String> args = new ArrayList<String>(); + + args.add("--table"); + args.add(getTableName()); + args.add("--connect"); + args.add(getConnectString()); + args.add("--hive-import"); + + return args.toArray(new String[0]); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java deleted file mode 100644 index 8cc5a0b..0000000 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java +++ /dev/null @@ -1,366 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sqoop.manager.sqlserver; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Types; -import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import com.cloudera.sqoop.ConnFactory; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.manager.ConnManager; -import com.cloudera.sqoop.metastore.JobData; -import com.cloudera.sqoop.testutil.HsqldbTestServer; -import com.cloudera.sqoop.tool.ImportTool; -import com.cloudera.sqoop.tool.SqoopTool; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * Test methods of the generic SqlManager implementation. - * - * This uses JDBC to import data from an SQLServer database to HDFS. - * - * Since this requires an SQLServer installation, - * this class is named in such a way that Sqoop's default QA process does - * not run it. You need to run this manually with - * -Dtestcase=SQLServerManagerManualTest. - * - * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location - * where Sqoop will be able to access it (since this library cannot be checked - * into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install SQL Server Express 2012 - * Create a database SQOOPTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * access for SQOOPTEST to SQOOPUSER. - */ -public class SQLServerManagerManualTest { - - public static final Log LOG = LogFactory.getLog( - SQLServerManagerManualTest.class.getName()); - - /** the name of a table that doesn't exist. */ - static final String MISSING_TABLE = "MISSING_TABLE"; - - // instance variables populated during setUp, used during tests - private HsqldbTestServer testServer; - private ConnManager manager; - - @Before - public void setUp() { - MSSQLTestUtils utils = new MSSQLTestUtils(); - try { - utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM); - utils.populateLineItem(); - } catch (SQLException e) { - LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e)); - fail("Setup fail with SQLException: " + e.toString()); - } - Configuration conf = getConf(); - SqoopOptions opts = getSqoopOptions(conf); - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - opts.setUsername(username); - opts.setPassword(password); - opts.setConnectString(getConnectString()); - ConnFactory f = new ConnFactory(conf); - try { - this.manager = f.getManager(new JobData(opts, new ImportTool())); - System.out.println("Manger : " + this.manager); - } catch (IOException ioe) { - LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(ioe)); - fail("IOException instantiating manager: " - + StringUtils.stringifyException(ioe)); - } - } - - @After - public void tearDown() { - try { - - MSSQLTestUtils utils = new MSSQLTestUtils(); - utils.dropTableIfExists("TPCH1M_LINEITEM"); - manager.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException: " + sqlE.toString()); - fail("Got SQLException: " + sqlE.toString()); - } - } - - @Test - public void testListColNames() { - String[] colNames = manager.getColumnNames(getTableName()); - assertNotNull("manager returned no colname list", colNames); - assertEquals("Table list should be length 2", 16, colNames.length); - String[] knownFields = MSSQLTestUtils.getColumns(); - for (int i = 0; i < colNames.length; i++) { - assertEquals(knownFields[i], colNames[i]); - } - } - - @Test - public void testListColTypes() { - Map<String, Integer> types = manager.getColumnTypes(getTableName()); - - assertNotNull("manager returned no types map", types); - assertEquals("Map should be size=16", 16, types.size()); - assertEquals(types.get("L_ORDERKEY").intValue(), Types.INTEGER); - assertEquals(types.get("L_COMMENT").intValue(), Types.VARCHAR); - } - - @Test - public void testMissingTableColNames() { - // SQL Server returns an empty column list which gets translated as a - // zero length array - // how ever also check in case it returns null, which is also correct - String[] colNames = manager.getColumnNames(MISSING_TABLE); - if (colNames == null) { - assertNull("No column names should be returned for missing table", - colNames); - } - int numItems = colNames.length; - assertEquals(0, numItems); - } - - @Test - public void testMissingTableColTypes() { - Map<String, Integer> colTypes = manager.getColumnTypes(MISSING_TABLE); - assertNull("No column types should be returned for missing table", - colTypes); - } - - // constants related to testReadTable() - static final int EXPECTED_NUM_ROWS = 4; - static final int EXPECTED_COL1_SUM = 10; - static final int EXPECTED_COL2_SUM = 14; - - @Test - public void testReadTable() { - ResultSet results = null; - try { - results = manager.readTable(getTableName(), MSSQLTestUtils - .getColumns()); - - assertNotNull("ResultSet from readTable() is null!", results); - - ResultSetMetaData metaData = results.getMetaData(); - assertNotNull("ResultSetMetadata is null in readTable()", metaData); - - // ensure that we get the correct number of columns back - assertEquals("Number of returned columns was unexpected!", metaData - .getColumnCount(), 16); - - // should get back 4 rows. They are: - // 1 2 - // 3 4 - // 5 6 - // 7 8 - // .. so while order isn't guaranteed, we should get back 16 on the - // left - // and 20 on the right. - int sumCol1 = 0, sumCol2 = 0, rowCount = 0; - while (results.next()) { - rowCount++; - sumCol1 += results.getInt(1); - sumCol2 += results.getInt(2); - } - - assertEquals("Expected 4 rows back", EXPECTED_NUM_ROWS, rowCount); - assertEquals("Expected left sum of 10", EXPECTED_COL1_SUM, sumCol1); - assertEquals("Expected right sum of 14", EXPECTED_COL2_SUM, sumCol2); - } catch (SQLException sqlException) { - LOG.error(StringUtils.stringifyException(sqlException)); - fail("SQL Exception: " + sqlException.toString()); - } finally { - if (null != results) { - try { - results.close(); - } catch (SQLException sqlE) { - LOG.error(StringUtils.stringifyException(sqlE)); - fail("SQL Exception in ResultSet.close(): " - + sqlE.toString()); - } - } - - manager.release(); - } - } - - @Test - public void testReadMissingTable() { - ResultSet results = null; - try { - String[] colNames = { "*" }; - results = manager.readTable(MISSING_TABLE, colNames); - assertNull("Expected null resultset from readTable(MISSING_TABLE)", - results); - } catch (SQLException sqlException) { - // we actually expect this pass. - } finally { - if (null != results) { - try { - results.close(); - } catch (SQLException sqlE) { - fail("SQL Exception in ResultSet.close(): " - + sqlE.toString()); - } - } - - manager.release(); - } - } - - @Test - public void testgetPrimaryKeyFromMissingTable() { - String primaryKey = manager.getPrimaryKey(MISSING_TABLE); - assertNull("Expected null pkey for missing table", primaryKey); - } - - @Test - public void testgetPrimaryKeyFromTableWithoutKey() { - String primaryKey = manager.getPrimaryKey(getTableName()); - assertNull("Expected null pkey for table without key", primaryKey); - } - - // constants for getPrimaryKeyFromTable() - static final String TABLE_WITH_KEY = "TABLE_WITH_KEY"; - static final String KEY_FIELD_NAME = "KEYFIELD"; - - @Test - public void testgetPrimaryKeyFromTable() { - // first, create a table with a primary key - Connection conn = null; - try { - conn = getManager().getConnection(); - dropTableIfExists(TABLE_WITH_KEY); - PreparedStatement statement = conn.prepareStatement("CREATE TABLE " - + TABLE_WITH_KEY + "(" + KEY_FIELD_NAME - + " INT NOT NULL PRIMARY KEY, foo INT)", - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - statement.executeUpdate(); - statement.close(); - - String primaryKey = getManager().getPrimaryKey(TABLE_WITH_KEY); - assertEquals("Expected null pkey for table without key", - primaryKey, KEY_FIELD_NAME); - - } catch (SQLException sqlException) { - LOG.error(StringUtils.stringifyException(sqlException)); - fail("Could not create table with primary key: " - + sqlException.toString()); - } finally { - if (null != conn) { - try { - conn.close(); - } catch (SQLException sqlE) { - LOG.warn("Got SQLException during close: " - + sqlE.toString()); - } - } - } - - } - - protected boolean useHsqldbTestServer() { - return false; - } - - protected String getConnectString() { - return MSSQLTestUtils.getDBConnectString(); - } - - /** - * Drop a table if it already exists in the database. - * - * @param table - * the name of the table to drop. - * @throws SQLException - * if something goes wrong. - */ - protected void dropTableIfExists(String table) throws SQLException { - Connection conn = getManager().getConnection(); - String sqlStmt = "IF OBJECT_ID('" + table - + "') IS NOT NULL DROP TABLE " + table; - PreparedStatement statement = conn.prepareStatement(sqlStmt, - ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - try { - statement.executeUpdate(); - conn.commit(); - } finally { - statement.close(); - } - } - - protected SqoopOptions getSqoopOptions(Configuration conf) { - SqoopOptions opt = new SqoopOptions(conf); - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - SqoopOptions opts = new SqoopOptions(conf); - opts.setUsername(username); - opts.setPassword(password); - - return opt; - } - - SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) { - SqoopOptions opts = null; - try { - opts = tool.parseArguments(args, null, null, true); - String username = MSSQLTestUtils.getDBUserName(); - String password = MSSQLTestUtils.getDBPassWord(); - opts.setUsername(username); - opts.setPassword(password); - - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - fail("Invalid options: " + e.toString()); - } - - return opts; - } - - protected String getTableName() { - return "tpch1m_lineitem"; - } - - protected ConnManager getManager() { - return manager; - } - - protected Configuration getConf() { - return new Configuration(); - } - -}
