Repository: tez Updated Branches: refs/heads/master 3a1dcb6a3 -> bbcafcb81
TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bbcafcb8 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bbcafcb8 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bbcafcb8 Branch: refs/heads/master Commit: bbcafcb81a548e1e3bca301e9b1d328c97e36384 Parents: 3a1dcb6 Author: Jonathan Eagles <[email protected]> Authored: Thu Nov 12 13:50:20 2015 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Thu Nov 12 13:50:20 2015 -0600 ---------------------------------------------------------------------- .../dag/library/vertexmanager/ShuffleVertexManager.java | 6 +++--- .../library/vertexmanager/TestShuffleVertexManager.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index d9c4941..c5016aa 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -949,12 +949,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin { ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT); - if (slowStartMinSrcCompletionFraction < 0 + if (slowStartMinSrcCompletionFraction < 0 || slowStartMaxSrcCompletionFraction > 1 || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) { throw new IllegalArgumentException( "Invalid values for slowStartMinSrcCompletionFraction" - + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and " - + "max cannot be < min."); + + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be > 1," + + " and max cannot be < min."); } enableAutoParallelism = conf http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index 18d4bc1..fec08b2 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -589,7 +589,16 @@ public class TestShuffleVertexManager { Assert.assertTrue(e.getMessage().contains( "Invalid values for slowStartMinSrcCompletionFraction")); } - + + try { + // source vertex have some tasks. max > 1. + manager = createManager(conf, mockContext, 0.0f, 95.0f); + Assert.assertTrue(false); // should not come here + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Invalid values for slowStartMinSrcCompletionFraction")); + } + try { // source vertex have some tasks. min > max manager = createManager(conf, mockContext, 0.5f, 0.3f);
