Repository: tez Updated Branches: refs/heads/branch-0.7 aacad1866 -> 7121c3aed
TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles) (cherry picked from commit bbcafcb81a548e1e3bca301e9b1d328c97e36384) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0a9bad95 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0a9bad95 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0a9bad95 Branch: refs/heads/branch-0.7 Commit: 0a9bad95cd3069f40b1d29201883af48ac52256f Parents: aacad18 Author: Jonathan Eagles <[email protected]> Authored: Thu Nov 12 13:50:20 2015 -0600 Committer: Jonathan Eagles <[email protected]> Committed: Thu Nov 12 13:53:53 2015 -0600 ---------------------------------------------------------------------- .../dag/library/vertexmanager/ShuffleVertexManager.java | 6 +++--- .../library/vertexmanager/TestShuffleVertexManager.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/0a9bad95/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index d177460..4dfd411 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -790,12 +790,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin { ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT); - if (slowStartMinSrcCompletionFraction < 0 + if (slowStartMinSrcCompletionFraction < 0 || slowStartMaxSrcCompletionFraction > 1 || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) { throw new IllegalArgumentException( "Invalid values for slowStartMinSrcCompletionFraction" - + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and " - + "max cannot be < min."); + + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be > 1," + + " and max cannot be < min."); } enableAutoParallelism = conf http://git-wip-us.apache.org/repos/asf/tez/blob/0a9bad95/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index 8807674..4f93049 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -535,7 +535,16 @@ public class TestShuffleVertexManager { Assert.assertTrue(e.getMessage().contains( "Invalid values for slowStartMinSrcCompletionFraction")); } - + + try { + // source vertex have some tasks. max > 1. + manager = createManager(conf, mockContext, 0.0f, 95.0f); + Assert.assertTrue(false); // should not come here + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Invalid values for slowStartMinSrcCompletionFraction")); + } + try { // source vertex have some tasks. min > max manager = createManager(conf, mockContext, 0.5f, 0.3f);
