Repository: sqoop Updated Branches: refs/heads/trunk 739bbce48 -> b3e941be0
SQOOP-3346: Upgrade Hadoop version to 2.8.0 (Boglarka Egyed via Szabolcs Vasas) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b3e941be Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b3e941be Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b3e941be Branch: refs/heads/trunk Commit: b3e941be050d615f6aae5b4fafb9e0754c8c6a72 Parents: 739bbce Author: Szabolcs Vasas <[email protected]> Authored: Fri Jul 20 10:54:40 2018 +0200 Committer: Szabolcs Vasas <[email protected]> Committed: Fri Jul 20 10:54:40 2018 +0200 ---------------------------------------------------------------------- ivy/libraries.properties | 2 +- .../sqoop/config/ConfigurationConstants.java | 15 +++---------- .../sqoop/config/ConfigurationHelper.java | 23 +++++--------------- .../org/apache/sqoop/mapreduce/JobBase.java | 4 +--- src/java/org/apache/sqoop/tool/ImportTool.java | 5 ++--- src/test/org/apache/sqoop/TestSqoopOptions.java | 1 + .../sqlserver/SQLServerParseMethodsTest.java | 2 -- .../org/apache/sqoop/orm/TestParseMethods.java | 4 ---- 8 files changed, 13 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/ivy/libraries.properties ---------------------------------------------------------------------- diff --git a/ivy/libraries.properties b/ivy/libraries.properties index c506ca8..d679b12 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -55,7 +55,7 @@ accumulo.version=1.6.2 slf4j.version=1.7.7 -hadoop.version=2.6.0 +hadoop.version=2.8.0 hbase.version=1.2.4 hcatalog.version=1.2.1 http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationConstants.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java index 666852c..3724f25 100644 --- a/src/java/org/apache/sqoop/config/ConfigurationConstants.java +++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java @@ -55,23 +55,14 @@ public final class ConfigurationConstants { "mapred.reduce.tasks.speculative.execution"; /** - * The Configuration property identifying the job tracker address. - */ - public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS = - "mapred.job.tracker"; - - /** - * The Configuration property identifying the job tracker address (new). - */ - public static final String PROP_MAPREDUCE_JOB_TRACKER_ADDRESS = - "mapreduce.jobtracker.address"; - - /** * The Configuration property identifying the framework name. If set to YARN * then we will not be in local mode. */ public static final String PROP_MAPREDUCE_FRAMEWORK_NAME = "mapreduce.framework.name"; + + public static final String MAPREDUCE_FRAMEWORK_LOCAL = "local"; + /** * The group name of task counters. */ http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationHelper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/config/ConfigurationHelper.java b/src/java/org/apache/sqoop/config/ConfigurationHelper.java index fb2ab03..5a3d8ff 100644 --- a/src/java/org/apache/sqoop/config/ConfigurationHelper.java +++ b/src/java/org/apache/sqoop/config/ConfigurationHelper.java @@ -31,6 +31,9 @@ import org.apache.sqoop.mapreduce.db.DBConfiguration; import org.apache.hadoop.util.ReflectionUtils; +import static org.apache.sqoop.config.ConfigurationConstants.MAPREDUCE_FRAMEWORK_LOCAL; +import static org.apache.sqoop.config.ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME; + /** * This class provides static helper methods that allow access and manipulation * of job configuration. It is convenient to keep such access in one place in @@ -121,13 +124,6 @@ public final class ConfigurationHelper { } /** - * Sets the Jobtracker address to use for a job. - */ - public static void setJobtrackerAddr(Configuration conf, String addr) { - conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr); - } - - /** * @return the Configuration property identifying a DBWritable to use. */ public static String getDbInputClassProperty() { @@ -233,17 +229,8 @@ public final class ConfigurationHelper { return config.getInt(ConfigurationConstants.PROP_SPLIT_LIMIT, -1); } public static boolean isLocalJobTracker(Configuration conf) { - // If framework is set to YARN, then we can't be running in local mode - if ("yarn".equalsIgnoreCase(conf - .get(ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME))) { - return false; - } - String jtAddr = conf - .get(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS); - String jtAddr2 = conf - .get(ConfigurationConstants.PROP_MAPREDUCE_JOB_TRACKER_ADDRESS); - return (jtAddr != null && jtAddr.equals("local")) - || (jtAddr2 != null && jtAddr2.equals("local")); + String frameworkName = conf.get(PROP_MAPREDUCE_FRAMEWORK_NAME); + return MAPREDUCE_FRAMEWORK_LOCAL.equals(frameworkName); } private ConfigurationHelper() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/mapreduce/JobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java index 6d1e049..f7b1b46 100644 --- a/src/java/org/apache/sqoop/mapreduce/JobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java @@ -272,9 +272,7 @@ public class JobBase { protected void loadJars(Configuration conf, String ormJarFile, String tableClassName) throws IOException { - boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address")) - || "local".equals(conf.get("mapred.job.tracker")); - if (isLocal) { + if (ConfigurationHelper.isLocalJobTracker(conf)) { // If we're using the LocalJobRunner, then instead of using the compiled // jar file as the job source, we're running in the current thread. Push // on another classloader that loads from that jar in addition to http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index f7310b9..ccded65 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -42,6 +42,7 @@ import org.apache.sqoop.SqoopOptions; import org.apache.sqoop.SqoopOptions.InvalidOptionsException; import org.apache.sqoop.cli.RelatedOptions; import org.apache.sqoop.cli.ToolOptions; +import org.apache.sqoop.config.ConfigurationHelper; import org.apache.sqoop.hive.HiveClient; import org.apache.sqoop.hive.HiveClientFactory; import org.apache.sqoop.manager.ImportJobContext; @@ -118,9 +119,7 @@ public class ImportTool extends BaseSqoopTool { private void loadJars(Configuration conf, String ormJarFile, String tableClassName) throws IOException { - boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address")) - || "local".equals(conf.get("mapred.job.tracker")); - if (isLocal) { + if (ConfigurationHelper.isLocalJobTracker(conf)) { // If we're using the LocalJobRunner, then instead of using the compiled // jar file as the job source, we're running in the current thread. Push // on another classloader that loads from that jar in addition to http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/TestSqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/TestSqoopOptions.java b/src/test/org/apache/sqoop/TestSqoopOptions.java index ba4a4d4..d0591ad 100644 --- a/src/test/org/apache/sqoop/TestSqoopOptions.java +++ b/src/test/org/apache/sqoop/TestSqoopOptions.java @@ -858,6 +858,7 @@ public class TestSqoopOptions { @Test public void testSqoopOptionsCloneIsEqual() throws Exception { SqoopOptions options = createSqoopOptionsFilledWithRandomData(); + options.getConf().setAllowNullValueProperties(false); // always false in cloned conf SqoopOptions clonedOptions = (SqoopOptions) options.clone(); assertThat(options).isEqualToComparingFieldByFieldRecursively(clonedOptions); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java index 833ebe8..fbd8d96 100644 --- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java +++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java @@ -172,8 +172,6 @@ public class SQLServerParseMethodsTest extends ImportJobTestCase { // Tell the job what class we're testing. job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName); - // use local mode in the same JVM. - ConfigurationHelper.setJobtrackerAddr(job, "local"); job.set("fs.default.name", "file:///"); String warehouseDir = getWarehouseDir(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/orm/TestParseMethods.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/orm/TestParseMethods.java b/src/test/org/apache/sqoop/orm/TestParseMethods.java index 46bb52d..0d34f5a 100644 --- a/src/test/org/apache/sqoop/orm/TestParseMethods.java +++ b/src/test/org/apache/sqoop/orm/TestParseMethods.java @@ -135,8 +135,6 @@ public class TestParseMethods extends ImportJobTestCase { // Tell the job what class we're testing. job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName); - // use local mode in the same JVM. - ConfigurationHelper.setJobtrackerAddr(job, "local"); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } @@ -263,8 +261,6 @@ public class TestParseMethods extends ImportJobTestCase { job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0"); job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test"); - // use local mode in the same JVM. - ConfigurationHelper.setJobtrackerAddr(job, "local"); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); }
