TEZ-3647. Add a setting which lets Tez determine Xmx. Contributed by Zhiyuan Yang.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1b1eb1d4 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1b1eb1d4 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1b1eb1d4 Branch: refs/heads/TEZ-1190 Commit: 1b1eb1d4b995c221d2eece2c12fca6ca52b94f27 Parents: 4ce6ea6 Author: Siddharth Seth <[email protected]> Authored: Fri Mar 3 13:45:05 2017 -0800 Committer: Siddharth Seth <[email protected]> Committed: Fri Mar 3 13:45:05 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../main/java/org/apache/tez/client/TezClientUtils.java | 10 +++++++++- .../java/org/apache/tez/dag/api/TezConfiguration.java | 4 ++++ .../main/java/org/apache/tez/dag/api/TezConstants.java | 6 ++++++ .../java/org/apache/tez/client/TestTezClientUtils.java | 10 ++++++++++ 5 files changed, 30 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 07841bf..763acd8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3647. Add a setting which lets Tez determine Xmx. TEZ-3644. Cleanup container list stored in AMNode. TEZ-3646. IFile.Writer has an extra output stream flush call TEZ-3643. Long running AMs can go out of memory due to retained AMContainer instances. http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index ab37b17..b16d1ca 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -39,6 +39,7 @@ import java.util.Map.Entry; import com.google.common.base.Strings; import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Precision; import org.apache.tez.common.JavaOptsChecker; import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto; import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; @@ -963,9 +964,16 @@ public class TezClientUtils { || (resource.getMemory() <= 0)) { return javaOpts; } - if (maxHeapFactor <= 0 || maxHeapFactor >= 1) { + + if ((maxHeapFactor <= 0 && !Precision.equals(maxHeapFactor, -1, 0.01)) || maxHeapFactor >= 1) { return javaOpts; } + + if (Precision.equals(maxHeapFactor, -1, 0.01)) { + maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB + ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB + : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB; + } int maxMemory = (int)(resource.getMemory() * maxHeapFactor); maxMemory = maxMemory <= 0 ? 1 : maxMemory; http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 94f40bb..5900f5e 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -356,6 +356,10 @@ public class TezConfiguration extends Configuration { * actual allocation of memory to tasks the cluster. The value if used as a * fraction that is applied to the memory allocated Factor to size Xmx based * on container memory size. Value should be greater than 0 and less than 1. + * + * Set this value to -1 to allow Tez to use different default max heap fraction + * for different container memory size. Current policy is to use 0.7 for container + * smaller than 4GB and use 0.8 for larger container. */ @ConfigurationScope(Scope.AM) @ConfigurationProperty(type="float") http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java index 06b9cb7..b75269a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java @@ -123,5 +123,11 @@ public class TezConstants { */ public static final long TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS_MINIMUM = 1000; + /** + * For container JVM Xmx auto determination + */ + public static final int TEZ_CONTAINER_SMALL_SLAB_BOUND_MB = 4096; + public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB = 0.7; + public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB = 0.8; } http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index a5e9d3c..7ff8125 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -629,6 +629,16 @@ public class TestTezClientUtils { javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(355, 1), 100); Assert.assertEquals(origJavaOpts, javaOpts); + + origJavaOpts = ""; + javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, + Resource.newInstance(1000, 1), -1); + Assert.assertTrue(javaOpts.contains("-Xmx700m")); + + origJavaOpts = ""; + javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, + Resource.newInstance(5000, 1), -1); + Assert.assertTrue(javaOpts.contains("-Xmx4000m")); } @Test (timeout=5000)
