Repository: sqoop Updated Branches: refs/heads/trunk 8e45d2b38 -> b790c606a
SQOOP-3332: Extend Documentation of --resilient flag and add warning message when detected (Fero Szabo via Boglarka Egyed) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b790c606 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b790c606 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b790c606 Branch: refs/heads/trunk Commit: b790c606afd17067f563a107cde9faa733b242b6 Parents: 8e45d2b Author: Boglarka Egyed <[email protected]> Authored: Thu Jun 28 17:30:26 2018 +0200 Committer: Boglarka Egyed <[email protected]> Committed: Thu Jun 28 17:30:26 2018 +0200 ---------------------------------------------------------------------- src/docs/user/connectors.txt | 23 ++++++++++++++++---- .../apache/sqoop/manager/SQLServerManager.java | 8 +++++++ .../SqlServerManagerContextConfigurator.java | 2 +- .../sqlserver/SQLServerManagerImportTest.java | 8 ++++++- 4 files changed, 35 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/b790c606/src/docs/user/connectors.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt index f1c7aeb..59e3e00 100644 --- a/src/docs/user/connectors.txt +++ b/src/docs/user/connectors.txt @@ -147,17 +147,32 @@ $ sqoop export ... --export-dir custom_dir --table custom_table -- --identity-in Resilient operations ^^^^^^^^^^^^^^^^^^^^ -You can override the default and use resilient operations during export. +You can override the default and use resilient operations during export or import. This will retry failed operations, i.e. if the connection gets dropped by SQL Server, the mapper will try to reconnect and continue from where it was before. -The split-by column has to be specified and it is also required to be unique -and in ascending order. -For example: +In case of export, the +\--resilient+ option will ensure that Sqoop will try to recover +from connection resets. + +In case of import, however, one has to use both the +\--resilient+ option and specify +the +\--split-by+ column to trigger the retry mechanism. An important requirement is +that the data must be unique and ordered ascending by the split-by column, otherwise +records could be either lost or duplicated. + +Example commands using resilient operations: ---- $ sqoop export ... --export-dir custom_dir --table custom_table -- --resilient ---- +Importing from a table: +---- +$ sqoop import ... --table custom_table --split-by id -- --resilient +---- + +Importing via a query: +---- +$ sqoop import ... --query "SELECT ... WHERE $CONDITIONS" --split-by ordered_column -- --resilient + Schema support ^^^^^^^^^^^^^^ http://git-wip-us.apache.org/repos/asf/sqoop/blob/b790c606/src/java/org/apache/sqoop/manager/SQLServerManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/SQLServerManager.java b/src/java/org/apache/sqoop/manager/SQLServerManager.java index c98ad2d..33c57bc 100644 --- a/src/java/org/apache/sqoop/manager/SQLServerManager.java +++ b/src/java/org/apache/sqoop/manager/SQLServerManager.java @@ -333,6 +333,11 @@ public class SQLServerManager extends InformationSchemaManager { this.tableHints = hints; } + if (cmdLine.hasOption(SqlServerManagerContextConfigurator.RESILIENT_OPTION)) { + LOG.warn("Sqoop will use resilient operations! In case of import, " + + "the split-by column also has to be specified, unique, and in ascending order."); + } + identityInserts = cmdLine.hasOption(IDENTITY_INSERT); } @@ -359,6 +364,9 @@ public class SQLServerManager extends InformationSchemaManager { .withDescription("Allow identity inserts") .withLongOpt(IDENTITY_INSERT).create()); + extraOptions.addOption(OptionBuilder + .withLongOpt(SqlServerManagerContextConfigurator.RESILIENT_OPTION).create()); + return extraOptions; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/b790c606/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java index cf58f63..eab8a42 100644 --- a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java +++ b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java @@ -28,7 +28,7 @@ import org.apache.sqoop.mapreduce.sqlserver.SqlServerExportBatchOutputFormat; public class SqlServerManagerContextConfigurator { - private static final String RESILIENT_OPTION = "resilient"; + public static final String RESILIENT_OPTION = "resilient"; /** * Check if the user has requested the operation to be resilient. http://git-wip-us.apache.org/repos/asf/sqoop/blob/b790c606/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java index fc1c489..79e37f0 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java @@ -28,9 +28,11 @@ import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.manager.SQLServerManager; import org.apache.sqoop.testutil.ArgumentArrayBuilder; import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.util.ExpectedLogMessage; import org.apache.sqoop.util.FileListing; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -129,7 +131,7 @@ public class SQLServerManagerImportTest extends ImportJobTestCase { private final String tableName; public SQLServerManagerImportTest(ArgumentArrayBuilder builder, String tableName) { - this.builder = builder; + this.builder = new ArgumentArrayBuilder().with(builder); this.tableName = tableName; } @@ -266,6 +268,9 @@ public class SQLServerManagerImportTest extends ImportJobTestCase { } } + @Rule + public ExpectedLogMessage logMessage = new ExpectedLogMessage(); + @Test public void testImportSimple() throws IOException { doImportAndVerify(builder, tableName); @@ -285,6 +290,7 @@ public class SQLServerManagerImportTest extends ImportJobTestCase { @Test public void testImportTableResilient() throws IOException { + logMessage.expectWarn("Sqoop will use resilient operations! In case of import, the split-by column also has to be specified, unique, and in ascending order."); builder.withToolOption("resilient"); doImportAndVerify(builder, tableName); }
