From b790c606afd17067f563a107cde9faa733b242b6 Mon Sep 17 00:00:00 2001 From: Boglarka Egyed Date: Thu, 28 Jun 2018 17:30:26 +0200 Subject: [PATCH] SQOOP-3332: Extend Documentation of --resilient flag and add warning message when detected (Fero Szabo via Boglarka Egyed) --- src/docs/user/connectors.txt | 23 +++++++++++++++---- .../sqoop/manager/SQLServerManager.java | 8 +++++++ .../SqlServerManagerContextConfigurator.java | 2 +- .../sqlserver/SQLServerManagerImportTest.java | 8 ++++++- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt index f1c7aebe..59e3e00b 100644 --- a/src/docs/user/connectors.txt +++ b/src/docs/user/connectors.txt @@ -147,17 +147,32 @@ $ sqoop export ... --export-dir custom_dir --table custom_table -- --identity-in Resilient operations ^^^^^^^^^^^^^^^^^^^^ -You can override the default and use resilient operations during export. +You can override the default and use resilient operations during export or import. This will retry failed operations, i.e. if the connection gets dropped by SQL Server, the mapper will try to reconnect and continue from where it was before. -The split-by column has to be specified and it is also required to be unique -and in ascending order. -For example: +In case of export, the +\--resilient+ option will ensure that Sqoop will try to recover +from connection resets. + +In case of import, however, one has to use both the +\--resilient+ option and specify +the +\--split-by+ column to trigger the retry mechanism. An important requirement is +that the data must be unique and ordered ascending by the split-by column, otherwise +records could be either lost or duplicated. + +Example commands using resilient operations: ---- $ sqoop export ... --export-dir custom_dir --table custom_table -- --resilient ---- +Importing from a table: +---- +$ sqoop import ... --table custom_table --split-by id -- --resilient +---- + +Importing via a query: +---- +$ sqoop import ... --query "SELECT ... WHERE $CONDITIONS" --split-by ordered_column -- --resilient + Schema support ^^^^^^^^^^^^^^ diff --git a/src/java/org/apache/sqoop/manager/SQLServerManager.java b/src/java/org/apache/sqoop/manager/SQLServerManager.java index c98ad2db..33c57bcc 100644 --- a/src/java/org/apache/sqoop/manager/SQLServerManager.java +++ b/src/java/org/apache/sqoop/manager/SQLServerManager.java @@ -333,6 +333,11 @@ void parseExtraArgs(String[] args) throws ParseException { this.tableHints = hints; } + if (cmdLine.hasOption(SqlServerManagerContextConfigurator.RESILIENT_OPTION)) { + LOG.warn("Sqoop will use resilient operations! In case of import, " + + "the split-by column also has to be specified, unique, and in ascending order."); + } + identityInserts = cmdLine.hasOption(IDENTITY_INSERT); } @@ -359,6 +364,9 @@ private RelatedOptions getExtraOptions() { .withDescription("Allow identity inserts") .withLongOpt(IDENTITY_INSERT).create()); + extraOptions.addOption(OptionBuilder + .withLongOpt(SqlServerManagerContextConfigurator.RESILIENT_OPTION).create()); + return extraOptions; } diff --git a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java index cf58f631..eab8a421 100644 --- a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java +++ b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java @@ -28,7 +28,7 @@ public class SqlServerManagerContextConfigurator { - private static final String RESILIENT_OPTION = "resilient"; + public static final String RESILIENT_OPTION = "resilient"; /** * Check if the user has requested the operation to be resilient. diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java index fc1c4895..79e37f08 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java @@ -28,9 +28,11 @@ import org.apache.sqoop.manager.SQLServerManager; import org.apache.sqoop.testutil.ArgumentArrayBuilder; import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.util.ExpectedLogMessage; import org.apache.sqoop.util.FileListing; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -129,7 +131,7 @@ public static Iterable testConfigurations() { private final String tableName; public SQLServerManagerImportTest(ArgumentArrayBuilder builder, String tableName) { - this.builder = builder; + this.builder = new ArgumentArrayBuilder().with(builder); this.tableName = tableName; } @@ -266,6 +268,9 @@ public void tearDown() { } } + @Rule + public ExpectedLogMessage logMessage = new ExpectedLogMessage(); + @Test public void testImportSimple() throws IOException { doImportAndVerify(builder, tableName); @@ -285,6 +290,7 @@ public void testImportTableHintsMultiple() throws IOException { @Test public void testImportTableResilient() throws IOException { + logMessage.expectWarn("Sqoop will use resilient operations! In case of import, the split-by column also has to be specified, unique, and in ascending order."); builder.withToolOption("resilient"); doImportAndVerify(builder, tableName); }