TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces. Removes Tez configuration for in-degree and out-degree. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c2732c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c2732c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c2732c9 Branch: refs/heads/master Commit: 4c2732c991a4cc76011acedbf9c17726c06bc145 Parents: 268f3ac Author: Siddharth Seth <[email protected]> Authored: Thu May 23 13:40:22 2013 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu May 23 13:40:22 2013 -0700 ---------------------------------------------------------------------- .../java/org/apache/tez/common/TezJobConfig.java | 16 +--------- .../org/apache/hadoop/mapred/LocalJobRunner.java | 1 - .../tez/mapreduce/hadoop/DeprecatedKeys.java | 17 ++++------ .../processor/reduce/ReduceProcessor.java | 1 - .../apache/tez/mapreduce/task/MRRuntimeTask.java | 4 ++ .../tez/mapreduce/hadoop/TestDeprecatedKeys.java | 23 +------------- 6 files changed, 14 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java index 0d6f6be..c0b0ae6 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java @@ -66,7 +66,7 @@ public class TezJobConfig { */ public static final String LOCAL_DIRS = "tez.engine.local.dirs"; public static final String DEFAULT_LOCAL_DIRS = "/tmp"; - + /** * One local dir for the speicfic job. */ @@ -80,20 +80,6 @@ public class TezJobConfig { public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp"; public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir"; - - /** - * - */ - public static final String TEZ_ENGINE_TASK_INDEGREE = - "tez.engine.task.in-degree"; - public static final int DEFAULT_TEZ_ENGINE_TASK_INDEGREE = 1; - - /** - * - */ - public static final String TEZ_ENGINE_TASK_OUTDEGREE = - "tez.engine.task.out-degree"; - public static final int DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE = 1; /** * http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java index 5c181e2..5dade2b 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java @@ -445,7 +445,6 @@ public class LocalJobRunner implements ClientProtocol { JobConf localConf = new JobConf(job); localConf.setUser(user); localConf.set("mapreduce.jobtracker.address", "local"); - localConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, mapIds.size()); setupChildMapredLocalDirs(reduceId, user, localConf); // FIXME invalid task context http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 01d4fe0..499159f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -136,8 +136,6 @@ public class DeprecatedKeys { registerMRToEngineKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_ENGINE_IFILE_READAHEAD_BYTES); registerMRToEngineKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.RECORDS_BEFORE_PROGRESS); - - registerMRToEngineKeyTranslation(MRJobConfig.JOB_LOCAL_DIR, MRConfig.LOCAL_DIR); registerMRToEngineKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR); @@ -149,6 +147,12 @@ public class DeprecatedKeys { registerMRToEngineKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS); + // Counter replacement will work in this manner, as long as TezCounters + // extends MRCounters and is used directly by the Mapper/Reducer. + // When these counters are eventually translated over to MRCounters, this + // may break. + // Framework counters, like FILESYSTEM will likely be incompatible since + // they enum key belongs to a different package. registerMRToEngineKeyTranslation(MRJobConfig.COUNTERS_MAX_KEY, TezJobConfig.COUNTERS_MAX_KEY); registerMRToEngineKeyTranslation(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY); @@ -180,7 +184,7 @@ public class DeprecatedKeys { registerMRToEngineKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM); registerMRToEngineKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_ENGINE_INPUT_BUFFER_PERCENT); - + registerMRToEngineKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.DAG_CREDENTIALS_BINARY); registerMRToEngineKeyTranslation("map.sort.class", TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS); @@ -188,17 +192,10 @@ public class DeprecatedKeys { registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_GROUP_COMPARATOR_CLASS); registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS); - - registerMRToEngineKeyTranslation(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE); - registerMRToEngineKeyTranslation(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE); } private static void addDeprecatedKeys() { - // Adding deprecation for num_maps, num_reduces - in case some component in - // Hadoop MR land uses these. Tez components already use tez-in/out-degree. - _(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE); - _(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE); } private static void registerMRToEngineKeyTranslation(String mrKey, http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java index ce49726..9a6fb5b 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java @@ -338,7 +338,6 @@ implements Processor { throws IOException, InterruptedException { super.localizeConfiguration(jobConf); jobConf.setBoolean(JobContext.TASK_ISMAP, false); - jobConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, numMapTasks); } @Override http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java index 34aaa1b..a3d5684 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java @@ -170,6 +170,10 @@ public class MRRuntimeTask extends RuntimeTask { job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR)); job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS)); + if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) { + job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, + job.get(TezJobConfig.DAG_CREDENTIALS_BINARY)); + } // setup the child's attempt directories // Do the task-type specific localization http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index 16e783e..99aad74 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -18,28 +18,9 @@ package org.apache.tez.mapreduce.hadoop; -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobConf; -import org.apache.tez.common.TezJobConfig; -import org.junit.Test; public class TestDeprecatedKeys { - @Test - public void testNumTasksDeprecation() { - - Configuration conf = new Configuration(false); - conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, 6); - conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 3); - - - DeprecatedKeys.init(); - JobConf jobConf = new JobConf(conf); - - Assert.assertEquals(6, jobConf.getNumMapTasks()); - Assert.assertEquals(3, jobConf.getNumReduceTasks()); - } - + // TODO Add tests + }
