Repository: tez Updated Branches: refs/heads/master 980385ab6 -> 41ac55155
TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout (jlowe) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/41ac5515 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/41ac5515 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/41ac5515 Branch: refs/heads/master Commit: 41ac55155a59ba60b3da64b6b627c5760d8d6dcf Parents: 980385a Author: Jason Lowe <[email protected]> Authored: Fri Feb 26 21:56:17 2016 +0000 Committer: Jason Lowe <[email protected]> Committed: Fri Feb 26 21:56:17 2016 +0000 ---------------------------------------------------------------------- CHANGES.txt | 4 +++- .../java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 4 ++-- .../org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java | 6 ------ 3 files changed, 5 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/41ac5515/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 9800072..706a305 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-3029. Add an onError method to service plugin contexts. ALL CHANGES: + TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout TEZ-3128. Avoid stopping containers on the AM shutdown thread. TEZ-3129. Tez task and task attempt UI needs application fails with NotFoundException TEZ-3114. Shuffle OOM due to EventMetaData flood @@ -388,7 +389,8 @@ INCOMPATIBLE CHANGES TEZ-2679. Admin forms of launch env settings TEZ-2949. Allow duplicate dag names within session for Tez. -ALL CHANGES +ALL CHANGES: + TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout TEZ-3129. Tez task and task attempt UI needs application fails with NotFoundException TEZ-3114. Shuffle OOM due to EventMetaData flood TEZ-3102. Fetch failure of a speculated task causes job hang http://git-wip-us.apache.org/repos/asf/tez/blob/41ac5515/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 345f7aa..1335aa2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -82,6 +82,8 @@ public class DeprecatedKeys { TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH); mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); + mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT, + TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. @@ -153,8 +155,6 @@ public class DeprecatedKeys { registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS); registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC); - - registerMRToRuntimeKeyTranslation(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); } private static void addDeprecatedKeys() { http://git-wip-us.apache.org/repos/asf/tez/blob/41ac5515/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index 06d2990..ae218a0 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -43,7 +43,6 @@ public class TestDeprecatedKeys { jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f); jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true); jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f); - jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); MRHelpers.translateMRConfToTez(jobConf); @@ -65,7 +64,6 @@ public class TestDeprecatedKeys { assertEquals(0.33f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0), 0.01f); - assertEquals(1000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); } @Test(timeout = 5000) @@ -78,7 +76,6 @@ public class TestDeprecatedKeys { jobConf.setInt(MRJobConfig.IO_SORT_MB, 100); jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100); jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f); - jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200); @@ -103,7 +100,6 @@ public class TestDeprecatedKeys { jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator"); - jobConf.setInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000); jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true); @@ -135,7 +131,6 @@ public class TestDeprecatedKeys { assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "")); assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f); - assertEquals(2000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES)); @@ -161,7 +156,6 @@ public class TestDeprecatedKeys { assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS)); assertNull(jobConf.get("map.sort.class")); assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART)); - assertNull(jobConf.get(MRJobConfig.TASK_TIMEOUT)); } }
