Repository: tez Updated Branches: refs/heads/master b03faf1ad -> 460e0b2a0
TEZ-2966. Tez does not honor mapreduce.task.timeout (jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/460e0b2a Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/460e0b2a Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/460e0b2a Branch: refs/heads/master Commit: 460e0b2a01c00748bb80ad2314b2168dfa1e9615 Parents: b03faf1 Author: Jonathan Eagles <[email protected]> Authored: Fri Dec 11 10:16:25 2015 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Fri Dec 11 10:16:25 2015 -0600 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 3 ++- .../tez/mapreduce/hadoop/TestDeprecatedKeys.java | 13 ++++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 60fb502..ac8af72 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES: + TEZ-2966. Tez does not honor mapreduce.task.timeout TEZ-2979. FlakyTest: org.apache.tez.history.TestHistoryParser. TEZ-1491. Tez reducer-side merge's counter update is slow. TEZ-2943. Change shuffle vertex manager to use per vertex data for auto @@ -278,6 +279,7 @@ INCOMPATIBLE CHANGES TEZ-2949. Allow duplicate dag names within session for Tez. ALL CHANGES + TEZ-2966. Tez does not honor mapreduce.task.timeout TEZ-2346. TEZ-UI: Lazy load other info / counter data TEZ-2975. Bump up apache commons dependency. TEZ-2970. Re-localization in TezChild does not use correct UGI. http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 49f95c0..345f7aa 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -84,7 +84,7 @@ public class DeprecatedKeys { TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); } - // TODO TEZAM4 Sometime, make sure this gets loaded by default. Insteaf of the current initialization in MRAppMaster, TezChild. + // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. // Maybe define in an TEZConfiguration / TEZ JobConf variant. public static void init() { @@ -154,6 +154,7 @@ public class DeprecatedKeys { registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC); + registerMRToRuntimeKeyTranslation(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); } private static void addDeprecatedKeys() { http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index 2414743..06d2990 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; import org.apache.hadoop.mapred.JobConf; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.apache.tez.runtime.library.common.Constants; @@ -42,6 +43,7 @@ public class TestDeprecatedKeys { jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f); jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true); jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f); + jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); MRHelpers.translateMRConfToTez(jobConf); @@ -63,11 +65,12 @@ public class TestDeprecatedKeys { assertEquals(0.33f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0), 0.01f); + assertEquals(1000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); } @Test(timeout = 5000) /** - * Set of keys that can be overriden at tez runtime + * Set of keys that can be overridden at tez runtime */ public void verifyTezOverridenKeys() { JobConf jobConf = new JobConf(); @@ -75,7 +78,8 @@ public class TestDeprecatedKeys { jobConf.setInt(MRJobConfig.IO_SORT_MB, 100); jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100); jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f); - + jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); + jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true); @@ -99,7 +103,8 @@ public class TestDeprecatedKeys { jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator"); - + jobConf.setInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000); + jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true); @@ -130,6 +135,7 @@ public class TestDeprecatedKeys { assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "")); assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f); + assertEquals(2000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES)); @@ -155,6 +161,7 @@ public class TestDeprecatedKeys { assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS)); assertNull(jobConf.get("map.sort.class")); assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART)); + assertNull(jobConf.get(MRJobConfig.TASK_TIMEOUT)); } }
