Repository: tez Updated Branches: refs/heads/master c636dc220 -> 936ff8de0
TEZ-2290. Scale memory for Default Sorter down to a max of 2047 MB if configured higher (rbalamohan) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/936ff8de Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/936ff8de Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/936ff8de Branch: refs/heads/master Commit: 936ff8de0c02b0b1e5c94b8c31598c8d40a9d62d Parents: c636dc2 Author: Rajesh Balamohan <[email protected]> Authored: Thu Apr 9 06:41:13 2015 +0530 Committer: Rajesh Balamohan <[email protected]> Committed: Thu Apr 9 06:41:13 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../common/sort/impl/dflt/DefaultSorter.java | 27 +++++++++++++++----- .../sort/impl/dflt/TestDefaultSorter.java | 20 +++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 45c1541..cce0919 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly ALL CHANGES: + TEZ-2290. Scale memory for Default Sorter down to a max of 2047 MB if configured higher. TEZ-2233. Allow EdgeProperty of an edge to be changed by VertexManager TEZ-2293. When running in "mr" mode, always use MR config settings. TEZ-2273. Tez UI: Support client side searching & sorting for dag tasks page http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java index a6db0c6..2cbb70a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -128,13 +129,8 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable { final float spillper = this.conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT); - final int sortmb = (int) availableMemoryMb; - if (sortmb <= 0) { - throw new RuntimeException("InitialMemoryAssigned is <= 0: " + initialMemoryAvailable); - } - Preconditions.checkArgument(sortmb > 0 && sortmb <= 2047, - TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB - + " for DefaultSorter should be larger than 0 and less than or equal to 2047"); + final int sortmb = computeSortBufferSize((int) availableMemoryMb); + Preconditions.checkArgument(spillper <= (float) 1.0 && spillper > (float) 0.0, TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT + " should be greater than 0 and less than or equal to 1"); @@ -207,6 +203,23 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable { } } + @VisibleForTesting + static int computeSortBufferSize(int availableMemoryMB) { + + if (availableMemoryMB <= 0) { + throw new RuntimeException(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + + "=" + availableMemoryMB + ". It should be > 0"); + } + + if (availableMemoryMB > 2047) { + LOG.warn("Scaling down " + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + + "=" + availableMemoryMB + " to 2047 (max sort buffer size supported for DefaultSorter)"); + } + + //cap sort buffer to 2047 for DefaultSorter. + return Math.min(2047, availableMemoryMB); + } + @Override public void write(Object key, Object value) throws IOException { http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java index c9c215d..408ec3b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java @@ -124,6 +124,26 @@ public class TestDefaultSorter { } } + @Test(timeout = 5000) + public void testSortMBLimits() throws Exception { + + assertTrue("Expected 2047", DefaultSorter.computeSortBufferSize(4096) == 2047); + assertTrue("Expected 2047", DefaultSorter.computeSortBufferSize(2047) == 2047); + assertTrue("Expected 1024", DefaultSorter.computeSortBufferSize(1024) == 1024); + + try { + DefaultSorter.computeSortBufferSize(0); + fail("Should have thrown error for setting buffer size to 0"); + } catch(RuntimeException re) { + } + + try { + DefaultSorter.computeSortBufferSize(-100); + fail("Should have thrown error for setting buffer size to negative value"); + } catch(RuntimeException re) { + } + } + @Test(timeout = 30000) //Test TEZ-1977 public void basicTest() throws IOException {
