Repository: sqoop Updated Branches: refs/heads/trunk 3fba26419 -> b0b7b4779
SQOOP-2349: Add command line option for setting transaction isolation levels for metadata queries Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b0b7b477 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b0b7b477 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b0b7b477 Branch: refs/heads/trunk Commit: b0b7b4779c13101930f402fe3c7d832ea0a503f2 Parents: 3fba264 Author: Attila Szabo <[email protected]> Authored: Wed Nov 23 21:45:03 2016 +0100 Committer: Attila Szabo <[email protected]> Committed: Wed Nov 23 21:45:03 2016 +0100 ---------------------------------------------------------------------- src/java/org/apache/sqoop/SqoopOptions.java | 28 ++++++++--- .../org/apache/sqoop/manager/SqlManager.java | 2 +- .../org/apache/sqoop/tool/BaseSqoopTool.java | 20 ++++++++ .../sqoop/tool/JDBCTransactionLevels.java | 39 ++++++++++++++ .../org/apache/sqoop/tool/TestImportTool.java | 53 ++++++++++++++++++++ 5 files changed, 134 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 30b4705..ef26f16 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.reflect.Field; import java.net.URLDecoder; +import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -340,6 +341,8 @@ public class SqoopOptions implements Cloneable { // explicit split by cols @StoredAsProperty("reset.onemapper") private boolean autoResetToOneMapper; + @StoredAsProperty("sqlconnection.metadata.transaction.isolation.level") private int metadataTransactionIsolationLevel; + // These next two fields are not serialized to the metastore. // If this SqoopOptions is created by reading a saved job, these will // be populated by the JobStorage to facilitate updating the same @@ -1033,9 +1036,12 @@ public class SqoopOptions implements Cloneable { // Relaxed isolation will not enabled by default which is the behavior // of sqoop until now. this.relaxedIsolation = false; - + // set default mainframe data set type to partitioned data set this.mainframeInputDatasetType = MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_PARTITIONED; + + // set default transaction isolation level to TRANSACTION_READ_UNCOMMITED + this.metadataTransactionIsolationLevel = Connection.TRANSACTION_READ_COMMITTED; } /** @@ -2659,11 +2665,19 @@ public class SqoopOptions implements Cloneable { this.customToolOptions = customToolOptions; } - public String getToolName() { - return this.toolName; - } + public String getToolName() { + return this.toolName; + } - public void setToolName(String toolName) { - this.toolName = toolName; - } + public void setToolName(String toolName) { + this.toolName = toolName; + } + + public int getMetadataTransactionIsolationLevel() { + return this.metadataTransactionIsolationLevel; + } + + public void setMetadataTransactionIsolationLevel(int transactionIsolationLevel) { + this.metadataTransactionIsolationLevel = transactionIsolationLevel; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/manager/SqlManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/SqlManager.java b/src/java/org/apache/sqoop/manager/SqlManager.java index 768507b..808e330 100644 --- a/src/java/org/apache/sqoop/manager/SqlManager.java +++ b/src/java/org/apache/sqoop/manager/SqlManager.java @@ -918,7 +918,7 @@ public abstract class SqlManager * (queries executed by the ConnManager itself). */ protected int getMetadataIsolationLevel() { - return Connection.TRANSACTION_READ_COMMITTED; + return options.getMetadataTransactionIsolationLevel(); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 235f67a..3ed0f77 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -58,6 +58,8 @@ import com.cloudera.sqoop.metastore.JobData; */ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { + public static final String METADATA_TRANSACTION_ISOLATION_LEVEL = "metadata-transaction-isolation-level"; + public static final Log LOG = LogFactory.getLog( BaseSqoopTool.class.getName()); @@ -478,6 +480,13 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { .hasArg() .withArgName("rootdir") .create()); + commonOpts.addOption(OptionBuilder + .withDescription("Defines the transaction isolation level for metadata queries. " + + "For more details check java.sql.Connection javadoc or the JDBC specificaiton") + .withLongOpt(METADATA_TRANSACTION_ISOLATION_LEVEL) + .hasArg() + .withArgName("isolationlevel") + .create()); // relax isolation requirements commonOpts.addOption(OptionBuilder .withDescription("Use read-uncommitted isolation for imports") @@ -1029,6 +1038,17 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { if (in.hasOption(RELAXED_ISOLATION)) { out.setRelaxedIsolation(true); } + + if (in.hasOption(METADATA_TRANSACTION_ISOLATION_LEVEL)) { + String transactionLevel = in.getOptionValue(METADATA_TRANSACTION_ISOLATION_LEVEL); + try { + out.setMetadataTransactionIsolationLevel(JDBCTransactionLevels.valueOf(transactionLevel).getTransactionIsolationLevelValue()); + } catch (IllegalArgumentException e) { + throw new RuntimeException("Only transaction isolation levels defined by " + + "java.sql.Connection class are supported. Check the " + + "java.sql.Connection javadocs for more details", e); + } + } } private void applyCredentialsOptions(CommandLine in, SqoopOptions out) http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java b/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java new file mode 100644 index 0000000..c141497 --- /dev/null +++ b/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.tool; + +import java.sql.Connection; + +enum JDBCTransactionLevels { + TRANSACTION_NONE(Connection.TRANSACTION_NONE), + TRANSACTION_READ_UNCOMMITTED(Connection.TRANSACTION_READ_UNCOMMITTED), + TRANSACTION_READ_COMMITTED(Connection.TRANSACTION_READ_COMMITTED), + TRANSACTION_REPEATABLE_READ(Connection.TRANSACTION_REPEATABLE_READ), + TRANSACTION_SERIALIZABLE(Connection.TRANSACTION_SERIALIZABLE); + + private final int transactionLevelValue; + + private JDBCTransactionLevels(int transactionLevelValue) { + this.transactionLevelValue = transactionLevelValue; + } + + public int getTransactionIsolationLevelValue() { + return transactionLevelValue; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/test/org/apache/sqoop/tool/TestImportTool.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/tool/TestImportTool.java b/src/test/org/apache/sqoop/tool/TestImportTool.java new file mode 100644 index 0000000..4136e9f --- /dev/null +++ b/src/test/org/apache/sqoop/tool/TestImportTool.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.tool; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import java.sql.Connection; + +import org.apache.sqoop.SqoopOptions; +import org.junit.experimental.theories.DataPoints; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; + +@RunWith(Theories.class) +public class TestImportTool { + @DataPoints + public static final Object[][] TRANSACTION_ISOLATION_LEVEL_NAMES_AND_VALUES = { + {"TRANSACTION_NONE", Connection.TRANSACTION_NONE}, + {"TRANSACTION_READ_COMMITTED",Connection.TRANSACTION_READ_COMMITTED}, + {"TRANSACTION_READ_UNCOMMITTED",Connection.TRANSACTION_READ_UNCOMMITTED}, + {"TRANSACTION_REPEATABLE_READ",Connection.TRANSACTION_REPEATABLE_READ}, + {"TRANSACTION_SERIALIZABLE",Connection.TRANSACTION_SERIALIZABLE} + }; + + @Theory + public void esnureTransactionIsolationLevelsAreMappedToTheRightValues(Object[] values) + throws Exception { + ImportTool importTool = new ImportTool(); + String[] args = { "--" + BaseSqoopTool.METADATA_TRANSACTION_ISOLATION_LEVEL, values[0].toString() }; + SqoopOptions options = importTool.parseArguments(args, null, null, true); + assertThat(options.getMetadataTransactionIsolationLevel(), is(equalTo(values[1]))); + } + +}
