Repository: sqoop Updated Branches: refs/heads/trunk ac42f2ad5 -> b0f391e75
SQOOP-3158: Columns added to Mysql after initial sqoop import, export back to table with same schema fails (Eric Lin via Anna Szonyi) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b0f391e7 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b0f391e7 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b0f391e7 Branch: refs/heads/trunk Commit: b0f391e75154be86f95378ab141f6dd1b3b59475 Parents: ac42f2a Author: Anna Szonyi <[email protected]> Authored: Thu Jun 8 20:45:50 2017 +0200 Committer: Anna Szonyi <[email protected]> Committed: Thu Jun 8 20:45:50 2017 +0200 ---------------------------------------------------------------------- src/java/org/apache/sqoop/orm/ClassWriter.java | 15 +- src/test/com/cloudera/sqoop/TestExport.java | 189 +++++++++++++++++++ .../apache/sqoop/TestExportUsingProcedure.java | 45 +++++ 3 files changed, 248 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0f391e7/src/java/org/apache/sqoop/orm/ClassWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index eaa9123..cdb2364 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -1409,9 +1409,22 @@ public class ClassWriter { private void parseColumn(String colName, int colType, StringBuilder sb) { // assume that we have __it and __cur_str vars, based on // __loadFromFields() code. - sb.append(" __cur_str = __it.next();\n"); + String javaType = toJavaType(colName, colType); + String nullValue; + if (javaType.equals("String")) { + nullValue = this.options.getInNullStringValue(); + } else { + nullValue = this.options.getInNullNonStringValue(); + } + + sb.append(" if (__it.hasNext()) {\n"); + sb.append(" __cur_str = __it.next();\n"); + sb.append(" } else {\n"); + sb.append(" __cur_str = \"" + nullValue + "\";\n"); + sb.append(" }\n"); + parseNullVal(javaType, colName, sb); if (javaType.equals("String")) { // TODO(aaron): Distinguish between 'null' and null. Currently they both http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0f391e7/src/test/com/cloudera/sqoop/TestExport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestExport.java b/src/test/com/cloudera/sqoop/TestExport.java index b2edc53..8b468c8 100644 --- a/src/test/com/cloudera/sqoop/TestExport.java +++ b/src/test/com/cloudera/sqoop/TestExport.java @@ -895,4 +895,193 @@ public class TestExport extends ExportJobTestCase { verifyExport(TOTAL_RECORDS); } + /** + * When we have less columns in the export file than in the export table, without changes in SQOOP-3158 + * it will fail with errors. After SQOOP-3158, it should succeed without errors + * + * @throws IOException + * @throws SQLException + */ + @Test + public void testLessColumnsInFileThanInTable() throws IOException, SQLException { + final int TOTAL_RECORDS = 10; + + ColumnGenerator genDate = getDateColumnGenerator(); + ColumnGenerator genTime = getTimeColumnGenerator(); + + createTextFile(0, TOTAL_RECORDS, false, genDate); + createTable(genDate, genTime); + runExport(getArgv(true, 10, 10)); + verifyExport(TOTAL_RECORDS); + assertColMinAndMax(forIdx(0), genDate); + + // test that the Time column is with NULL values + class NullColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + return null; + } + public String getVerifyText(int rowNum) { + return null; + } + public String getType() { + return "Timestamp"; + } + } + + ColumnGenerator genNull = new NullColumnGenerator(); + assertColMinAndMax(forIdx(1), genNull); + } + + /** + * This test case covers the following: + * + * - two files under export directory + * - first file has one missing column + * - second file has correct number of columns + * - second file has last column with the value defined by parameter --input-null-string + * - last column is of STRING data type + * + * This case will happen when destination table has been modified to add a new column and + * user has data for both before and after the table modification, some files has less columns + * and some files has new columns + * + * @throws IOException + * @throws SQLException + */ + @Test + public void testLessColumnsInFileThanInTableInputNullStringPassed() throws IOException, SQLException { + final int TOTAL_RECORDS = 10; + + // a column that contains string value of "STRING_NULL", + // this should be treated as NULL value during test import + // due to --input-null-string is defined using "STRING_NULL" + class StringNULLValueColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + return "STRING_NULL"; + } + public String getVerifyText(int rowNum) { + return "STRING_NULL"; + } + public String getType() { + return "VARCHAR(255)"; + } + } + + // a normal string column + class StringColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + int day = rowNum + 1; + return "ROW_" + pad(day); + } + public String getVerifyText(int rowNum) { + int day = rowNum + 1; + return "ROW_" + pad(day); + } + public String getType() { + return "VARCHAR(255)"; + } + } + + // test that the second column is with NULL values after import + class NullColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + return null; + } + public String getVerifyText(int rowNum) { + return null; + } + public String getType() { + return "VARCHAR(255)"; + } + } + + ColumnGenerator genString = new StringColumnGenerator(); + ColumnGenerator genNullString = new StringNULLValueColumnGenerator(); + + createTextFile(0, TOTAL_RECORDS, false, genString); + createTextFile(1, TOTAL_RECORDS, false, genString, genNullString); + createTable(genString, genNullString); + runExport(getArgv(true, 10, 10, "--input-null-string", "STRING_NULL")); + + verifyExport(TOTAL_RECORDS * 2); + assertColMinAndMax(forIdx(0), genString); + + ColumnGenerator genNull = new NullColumnGenerator(); + assertColMinAndMax(forIdx(1), genNull); + } + + /** + * This test case covers the following: + * + * - two files under export directory + * - first file has one missing column + * - second file has correct number of columns + * - second file has last column with the value defined by parameter --input-null-non-string + * - last column is of INTEGER data type + * + * This case will happen when destination table has been modified to add a new column and + * user has data for both before and after the table modification, some files has less columns + * and some files has new columns + * + * @throws IOException + * @throws SQLException + */ + @Test + public void testLessColumnsInFileThanInTableInputNullIntPassed() throws IOException, SQLException { + final int TOTAL_RECORDS = 10; + + // a column that contains string value of "INT_NULL", + // this should be treated as NULL value during test import + // due to --input-null-non-string is defined using "INT_NULL" + class IntNULLValueColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + return "INT_NULL"; + } + public String getVerifyText(int rowNum) { + return "INT_NULL"; + } + public String getType() { + return "INT"; + } + } + + // a normal string column + class IntColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + int day = rowNum + 1; + return String.valueOf(day); + } + public String getVerifyText(int rowNum) { + int day = rowNum + 1; + return String.valueOf(day); + } + public String getType() { + return "INT"; + } + } + + // test that the second column is with NULL values after import + class NullColumnGenerator implements ColumnGenerator { + public String getExportText(int rowNum) { + return null; + } + public String getVerifyText(int rowNum) { + return null; + } + public String getType() { + return "INT"; + } + } + + ColumnGenerator genInt = new IntColumnGenerator(); + ColumnGenerator genNullInt = new IntNULLValueColumnGenerator(); + + createTextFile(0, TOTAL_RECORDS, false, genInt); + createTextFile(1, TOTAL_RECORDS, false, genInt, genNullInt); + createTable(genInt, genNullInt); + runExport(getArgv(true, 10, 10, "--input-null-non-string", "INT_NULL")); + verifyExport(TOTAL_RECORDS * 2); + assertColMinAndMax(forIdx(0), genInt); + assertColMinAndMax(forIdx(1), new NullColumnGenerator()); + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0f391e7/src/test/org/apache/sqoop/TestExportUsingProcedure.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/TestExportUsingProcedure.java b/src/test/org/apache/sqoop/TestExportUsingProcedure.java index 519305c..68e79f1 100644 --- a/src/test/org/apache/sqoop/TestExportUsingProcedure.java +++ b/src/test/org/apache/sqoop/TestExportUsingProcedure.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.Date; +import java.sql.Timestamp; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -337,4 +338,48 @@ public class TestExportUsingProcedure extends TestExport { }); } + /** + * This test case is special - we're only inserting into a subset of the + * columns in the table. + */ + public static void insertFunctiontestLessColumnsInFileThanInTable(int id, String msg, + final Date date1, final Timestamp t2) throws SQLException { + insertFunction(id, msg, new SetExtraArgs() { + @Override + public void set(PreparedStatement on) throws SQLException { + on.setDate(3, date1); + on.setNull(4, Types.TIMESTAMP); + } + }); + } + + /** + * This test case is special - we're only inserting into a subset of the + * columns in the table. + */ + public static void insertFunctiontestLessColumnsInFileThanInTableInputNullStringPassed(int id, String msg, + final String str1) throws SQLException { + insertFunction(id, msg, new SetExtraArgs() { + @Override + public void set(PreparedStatement on) throws SQLException { + on.setString(3, str1); + on.setNull(4, Types.VARCHAR); + } + }); + } + + /** + * This test case is special - we're only inserting into a subset of the + * columns in the table. + */ + public static void insertFunctiontestLessColumnsInFileThanInTableInputNullIntPassed(int id, String msg, + final int int1) throws SQLException { + insertFunction(id, msg, new SetExtraArgs() { + @Override + public void set(PreparedStatement on) throws SQLException { + on.setInt(3, int1); + on.setNull(4, Types.INTEGER); + } + }); + } }
