Repository: sqoop Updated Branches: refs/heads/trunk af277ae3e -> 2c199a7f4
SQOOP-3206: Make sqoop fail if user uses --direct connector and tries to encode a null value when using a MySQL database (Zach Berkowitz via Anna Szonyi) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2c199a7f Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2c199a7f Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2c199a7f Branch: refs/heads/trunk Commit: 2c199a7f4fc744f653f036d5afb6f49b0d4c445c Parents: af277ae Author: Anna Szonyi <[email protected]> Authored: Thu Jun 29 18:44:14 2017 +0200 Committer: Anna Szonyi <[email protected]> Committed: Thu Jun 29 18:44:14 2017 +0200 ---------------------------------------------------------------------- src/java/org/apache/sqoop/tool/ExportTool.java | 14 ++++++++++++ src/java/org/apache/sqoop/tool/ImportTool.java | 11 +++++++-- .../sqoop/manager/DirectMySQLExportTest.java | 18 +++++++++++++++ .../cloudera/sqoop/manager/DirectMySQLTest.java | 24 ++++++++++++++------ .../tool/TestExportToolValidateOptions.java | 16 +++++++++++++ .../sqoop/tool/TestValidateImportOptions.java | 17 +++++++++++++- 6 files changed, 90 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/java/org/apache/sqoop/tool/ExportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java index 5512fa7..cd6cdf3 100644 --- a/src/java/org/apache/sqoop/tool/ExportTool.java +++ b/src/java/org/apache/sqoop/tool/ExportTool.java @@ -33,6 +33,7 @@ import com.cloudera.sqoop.cli.RelatedOptions; import com.cloudera.sqoop.cli.ToolOptions; import com.cloudera.sqoop.manager.ExportJobContext; import com.cloudera.sqoop.util.ExportException; +import static org.apache.sqoop.manager.SupportedManagers.MYSQL; /** * Tool that performs HDFS exports to databases. @@ -385,8 +386,21 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { void vaildateDirectExportOptions(SqoopOptions options) throws InvalidOptionsException { if (options.isDirect()) { validateHasDirectConnectorOption(options); + validateDirectMysqlOptions(options); } } + + public void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException { + if (!MYSQL.isTheManagerTypeOf(options)) { + return; + } + if (options.getInNullStringValue() != null || options.getInNullNonStringValue() != null) { + throw new InvalidOptionsException( + "The --direct option is not compatible with the --input-null-string or " + + "--input-null-non-string command for MySQL exports"); + } + } + private void applyNewUpdateOptions(CommandLine in, SqoopOptions out) throws InvalidOptionsException { if (in.hasOption(UPDATE_MODE_ARG)) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index 4b1b12d..78c7758 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -1120,13 +1120,20 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { } void validateDirectMysqlOptions(SqoopOptions options) throws InvalidOptionsException { - if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile - && MYSQL.isTheManagerTypeOf(options)) { + if (!MYSQL.isTheManagerTypeOf(options)) { + return; + } + if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) { throw new InvalidOptionsException( "MySQL direct import currently supports only text output format. " + "Parameters --as-sequencefile --as-avrodatafile and --as-parquetfile are not " + "supported with --direct params in MySQL case."); } + if (options.getNullStringValue() != null || options.getNullNonStringValue() != null) { + throw new InvalidOptionsException( + "The --direct option is not compatible with the --null-string or " + + "--null-non-string command for MySQL imports"); + } } /** * Validate the incremental import options. http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java index 9fa8816..f9e3cde 100644 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java +++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java @@ -303,6 +303,24 @@ public class DirectMySQLExportTest extends TestExport { } } + @Test(expected = IOException.class) + public void testExportInputNullStringFailsValidate() throws IOException { + runExport(getArgv(true, 10, 10, + "--username", MySQLAuthTest.AUTH_TEST_USER, + "--password", MySQLAuthTest.AUTH_TEST_PASS, + "--connect", MySQLAuthTest.AUTH_CONNECT_STRING, + "--input-null-string", "null")); + } + + @Test(expected = IOException.class) + public void testExportInputNullNonStringFailsValidate() throws IOException { + runExport(getArgv(true, 10, 10, + "--username", MySQLAuthTest.AUTH_TEST_USER, + "--password", MySQLAuthTest.AUTH_TEST_PASS, + "--connect", MySQLAuthTest.AUTH_CONNECT_STRING, + "--input-null-non-string", "null")); + } + @Ignore("Ignoring this test as staging is not supported in direct mode.") @Override @Test http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java index a58fa17..247ce0b 100644 --- a/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java +++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java @@ -192,13 +192,7 @@ public class DirectMySQLTest extends ImportJobTestCase { } String [] argv = getArgv(mysqlOutputDelims, isDirect, tableName, extraArgs); - try { - runImport(argv); - } catch (IOException ioe) { - LOG.error("Got IOException during import: " + ioe.toString()); - ioe.printStackTrace(); - fail(ioe.toString()); - } + runImport(argv); File f = new File(filePath.toString()); assertTrue("Could not find imported data file: " + f, f.exists()); @@ -358,6 +352,22 @@ public class DirectMySQLTest extends ImportJobTestCase { } + @Test(expected = IOException.class) + public void testSqoopNullStringValueFailsValidate() throws Exception { + String [] expectedResults = {}; + String [] extraArgs = {"--null-string", "abc"}; + + doImport(false, true, getTableName(), expectedResults, extraArgs); + } + + @Test(expected = IOException.class) + public void testSqoopNullNonStringValueFailsValidate() throws Exception { + String [] expectedResults = {}; + String [] extraArgs = {"--null-non-string", "abc"}; + + doImport(false, true, getTableName(), expectedResults, extraArgs); + } + @Test public void testJdbcEscapedColumnName() throws Exception { // Test a JDBC-based import of a table with a column whose name is http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java index dfe1952..0018fb1 100644 --- a/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java +++ b/src/test/org/apache/sqoop/tool/TestExportToolValidateOptions.java @@ -66,6 +66,22 @@ public class TestExportToolValidateOptions { exportTool.vaildateDirectExportOptions(options); } + @Test(expected = SqoopOptions.InvalidOptionsException.class) + public void givenDirectImportInputNullStringThrows() throws SqoopOptions.InvalidOptionsException { + SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL); + when(options.getInNullNonStringValue()).thenReturn("abc"); + + exportTool.validateDirectMysqlOptions(options); + } + + @Test(expected = SqoopOptions.InvalidOptionsException.class) + public void givenDirectImportInputNullNonStringThrows() throws SqoopOptions.InvalidOptionsException { + SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL); + when(options.getInNullNonStringValue()).thenReturn("abc"); + + exportTool.validateDirectMysqlOptions(options); + } + private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) { return stubOptions(supportedManagers, true); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c199a7f/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java index acf4fcf..d4084ed 100644 --- a/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java +++ b/src/test/org/apache/sqoop/tool/TestValidateImportOptions.java @@ -44,7 +44,6 @@ public class TestValidateImportOptions { when(options.getConnectString()).thenReturn(mysqlConnectionString); importTool.validateDirectMysqlOptions(options); verify(options, times(1)).getFileLayout(); - verifyNoMoreInteractions(options); } @@ -107,6 +106,22 @@ public class TestValidateImportOptions { importTool.validateDirectImportOptions(options); } + @Test(expected = SqoopOptions.InvalidOptionsException.class) + public void givenDirectImportNullNonStringThrows() throws SqoopOptions.InvalidOptionsException { + SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL); + when(options.getNullNonStringValue()).thenReturn("abc"); + + importTool.validateDirectMysqlOptions(options); + } + + @Test(expected = SqoopOptions.InvalidOptionsException.class) + public void givenDirectImportNullStringThrows() throws SqoopOptions.InvalidOptionsException { + SqoopOptions options = stubDirectOptions(SupportedManagers.MYSQL); + when(options.getNullStringValue()).thenReturn("abc"); + + importTool.validateDirectMysqlOptions(options); + } + private SqoopOptions stubDirectOptions(SupportedManagers supportedManagers) { return stubOptions(supportedManagers, true); }
