This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new c41283f [GOBBLIN-770] Add JVM configuration to avoid exhausting YARN
container memory
c41283f is described below
commit c41283fb39a1e58799923eb5e188df1fbbf7bd8d
Author: Hung Tran <[email protected]>
AuthorDate: Wed May 15 14:08:53 2019 -0700
[GOBBLIN-770] Add JVM configuration to avoid exhausting YARN container
memory
Closes #2634 from htran1/yarn_memory_config
---
.../gobblin/yarn/GobblinYarnAppLauncher.java | 30 +++++++++++++++++++---
.../gobblin/yarn/GobblinYarnConfigurationKeys.java | 14 ++++++++++
.../java/org/apache/gobblin/yarn/YarnService.java | 25 ++++++++++++++++--
.../gobblin/yarn/GobblinYarnAppLauncherTest.java | 8 ++++++
.../org/apache/gobblin/yarn/YarnServiceTest.java | 21 +++++++++++++++
.../test/resources/GobblinYarnAppLauncherTest.conf | 3 +++
6 files changed, 96 insertions(+), 5 deletions(-)
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 5370375..8288604 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -71,6 +71,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
@@ -201,6 +202,10 @@ public class GobblinYarnAppLauncher {
private final boolean emailNotificationOnShutdown;
+ private final int appMasterMemoryMbs;
+ private final int jvmMemoryOverheadMbs;
+ private final double jvmMemoryXmxRatio;
+
public GobblinYarnAppLauncher(Config config, YarnConfiguration
yarnConfiguration) throws IOException {
this.config = config;
@@ -238,6 +243,24 @@ public class GobblinYarnAppLauncher {
this.emailNotificationOnShutdown =
config.getBoolean(GobblinYarnConfigurationKeys.EMAIL_NOTIFICATION_ON_SHUTDOWN_KEY);
+
+ this.appMasterMemoryMbs =
this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY);
+
+ this.jvmMemoryXmxRatio = ConfigUtils.getDouble(this.config,
+ GobblinYarnConfigurationKeys.APP_MASTER_JVM_MEMORY_XMX_RATIO_KEY,
+ GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_JVM_MEMORY_XMX_RATIO);
+
+ Preconditions.checkArgument(this.jvmMemoryXmxRatio >= 0 &&
this.jvmMemoryXmxRatio <= 1,
+ GobblinYarnConfigurationKeys.APP_MASTER_JVM_MEMORY_XMX_RATIO_KEY + "
must be between 0 and 1 inclusive");
+
+ this.jvmMemoryOverheadMbs = ConfigUtils.getInt(this.config,
+ GobblinYarnConfigurationKeys.APP_MASTER_JVM_MEMORY_OVERHEAD_MBS_KEY,
+
GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_JVM_MEMORY_OVERHEAD_MBS);
+
+ Preconditions.checkArgument(this.jvmMemoryOverheadMbs <
this.appMasterMemoryMbs * this.jvmMemoryXmxRatio,
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + "
cannot be more than "
+ + GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
+ + GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);
}
/**
@@ -509,7 +532,7 @@ public class GobblinYarnAppLauncher {
}
private Resource prepareContainerResource(GetNewApplicationResponse
newApplicationResponse) {
- int memoryMbs =
this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY);
+ int memoryMbs = this.appMasterMemoryMbs;
int maximumMemoryCapacity =
newApplicationResponse.getMaximumResourceCapability().getMemory();
if (memoryMbs > maximumMemoryCapacity) {
LOGGER.info(String.format("Specified AM memory [%d] is above the maximum
memory capacity [%d] of the "
@@ -635,11 +658,12 @@ public class GobblinYarnAppLauncher {
YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath,
LocalResourceType.ARCHIVE, resourceMap);
}
- private String buildApplicationMasterCommand(int memoryMbs) {
+ @VisibleForTesting
+ protected String buildApplicationMasterCommand(int memoryMbs) {
String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
return new StringBuilder()
.append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
- .append(" -Xmx").append(memoryMbs).append("M")
+ .append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) -
this.jvmMemoryOverheadMbs).append("M")
.append(" ").append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs))
.append(" ").append(GobblinApplicationMaster.class.getName())
.append("
--").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 6098eae..7c80063 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -45,6 +45,13 @@ public class GobblinYarnConfigurationKeys {
public static final String APP_MASTER_WORK_DIR_NAME = "appmaster";
public static final String APP_MASTER_JVM_ARGS_KEY = GOBBLIN_YARN_PREFIX +
"app.master.jvm.args";
public static final String APP_MASTER_SERVICE_CLASSES = GOBBLIN_YARN_PREFIX
+ "app.master.serviceClasses";
+ // Amount of overhead to subtract when computing the Xmx value. This is to
account for non-heap memory, like metaspace
+ // and stack memory
+ public static final String APP_MASTER_JVM_MEMORY_OVERHEAD_MBS_KEY =
GOBBLIN_YARN_PREFIX + "app.master.jvmMemoryOverheadMbs";
+ public static final int DEFAULT_APP_MASTER_JVM_MEMORY_OVERHEAD_MBS = 0;
+ // The ratio of the amount of Xmx to carve out of the container memory
before adjusting for jvm memory overhead
+ public static final String APP_MASTER_JVM_MEMORY_XMX_RATIO_KEY =
GOBBLIN_YARN_PREFIX + "app.master.jvmMemoryXmxRatio";
+ public static final double DEFAULT_APP_MASTER_JVM_MEMORY_XMX_RATIO = 1.0;
// Gobblin Yarn container configuration properties.
public static final String INITIAL_CONTAINERS_KEY = GOBBLIN_YARN_PREFIX +
"initial.containers";
@@ -56,6 +63,13 @@ public class GobblinYarnConfigurationKeys {
public static final String CONTAINER_WORK_DIR_NAME = "container";
public static final String CONTAINER_JVM_ARGS_KEY = GOBBLIN_YARN_PREFIX +
"container.jvm.args";
public static final String CONTAINER_HOST_AFFINITY_ENABLED =
GOBBLIN_YARN_PREFIX + "container.affinity.enabled";
+ // Amount of overhead to subtract when computing the Xmx value. This is to
account for non-heap memory, like metaspace
+ // and stack memory
+ public static final String CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY =
GOBBLIN_YARN_PREFIX + "container.jvmMemoryOverheadMbs";
+ public static final int DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS = 0;
+ // The ratio of the amount of Xmx to carve out of the container memory
before adjusting for jvm memory overhead
+ public static final String CONTAINER_JVM_MEMORY_XMX_RATIO_KEY =
GOBBLIN_YARN_PREFIX + "container.jvmMemoryXmxRatio";
+ public static final double DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO = 1.0;
// Helix configuration properties.
public static final String HELIX_INSTANCE_MAX_RETRIES = GOBBLIN_YARN_PREFIX
+ "helix.instance.max.retries";
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index 614ac1f..f3d4e53 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
@@ -139,6 +140,8 @@ public class YarnService extends AbstractIdleService {
private final int initialContainers;
private final int requestedContainerMemoryMbs;
private final int requestedContainerCores;
+ private final int jvmMemoryOverheadMbs;
+ private final double jvmMemoryXmxRatio;
private final boolean containerHostAffinityEnabled;
private final int helixInstanceMaxRetries;
@@ -231,6 +234,22 @@ public class YarnService extends AbstractIdleService {
this.releasedContainerCache =
CacheBuilder.newBuilder().expireAfterAccess(ConfigUtils.getInt(config,
GobblinYarnConfigurationKeys.RELEASED_CONTAINERS_CACHE_EXPIRY_SECS,
GobblinYarnConfigurationKeys.DEFAULT_RELEASED_CONTAINERS_CACHE_EXPIRY_SECS),
TimeUnit.SECONDS).build();
+
+ this.jvmMemoryXmxRatio = ConfigUtils.getDouble(this.config,
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
+ GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO);
+
+ Preconditions.checkArgument(this.jvmMemoryXmxRatio >= 0 &&
this.jvmMemoryXmxRatio <= 1,
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY + "
must be between 0 and 1 inclusive");
+
+ this.jvmMemoryOverheadMbs = ConfigUtils.getInt(this.config,
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
+
GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS);
+
+ Preconditions.checkArgument(this.jvmMemoryOverheadMbs <
this.requestedContainerMemoryMbs * this.jvmMemoryXmxRatio,
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + "
cannot be more than "
+ + GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
+ + GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);
}
@SuppressWarnings("unused")
@@ -521,11 +540,13 @@ public class YarnService extends AbstractIdleService {
}
}
- private String buildContainerCommand(Container container, String
helixInstanceName) {
+ @VisibleForTesting
+ protected String buildContainerCommand(Container container, String
helixInstanceName) {
String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName();
return new StringBuilder()
.append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
- .append("
-Xmx").append(container.getResource().getMemory()).append("M")
+ .append(" -Xmx").append((int) (container.getResource().getMemory() *
this.jvmMemoryXmxRatio) -
+ this.jvmMemoryOverheadMbs).append("M")
.append(" ").append(JvmUtils.formatJvmArguments(this.containerJvmArgs))
.append(" ").append(GobblinYarnTaskRunner.class.getName())
.append("
--").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
diff --git
a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
index f409f5f..c9ec435 100644
---
a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
+++
b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
@@ -182,6 +182,14 @@ public class GobblinYarnAppLauncherTest implements
HelixMessageTestBase {
}
@Test
+ public void testBuildApplicationMasterCommand() {
+ String command =
this.gobblinYarnAppLauncher.buildApplicationMasterCommand(64);
+
+ // 41 is from 64 * 0.8 - 10
+ Assert.assertTrue(command.contains("-Xmx41"));
+ }
+
+ @Test
public void testCreateHelixCluster() throws Exception {
// This is tested here instead of in HelixUtilsTest to avoid setting up
yet another testing ZooKeeper server.
HelixUtils.createGobblinHelixCluster(
diff --git
a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnServiceTest.java
b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnServiceTest.java
index 60aad14..29cd68d 100644
--- a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnServiceTest.java
+++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnServiceTest.java
@@ -250,6 +250,27 @@ public class YarnServiceTest {
Assert.assertTrue(yarnService.getReleasedContainerCache().getIfPresent(containerId1)
== null);
}
+ @Test(groups = {"gobblin.yarn", "disabledOnTravis"}, dependsOnMethods =
"testReleasedContainerCache")
+ public void testBuildContainerCommand() throws Exception {
+ Config modifiedConfig = this.config
+
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
ConfigValueFactory.fromAnyRef("10"))
+
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
ConfigValueFactory.fromAnyRef("0.8"));
+
+ TestYarnService yarnService =
+ new TestYarnService(modifiedConfig, "testApp2", "appId2",
+ this.clusterConf, FileSystem.getLocal(new Configuration()),
this.eventBus);
+
+ ContainerId containerId =
ContainerId.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1,
0),
+ 0), 0);
+ Resource resource = Resource.newInstance(2048, 1);
+ Container container = Container.newInstance(containerId, null, null,
resource, null, null);
+
+ String command = yarnService.buildContainerCommand(container,
"helixInstance1");
+
+ // 1628 is from 2048 * 0.8 - 10
+ Assert.assertTrue(command.contains("-Xmx1628"));
+ }
+
private static class TestYarnService extends YarnService {
public TestYarnService(Config config, String applicationName, String
applicationId, YarnConfiguration yarnConfiguration,
diff --git a/gobblin-yarn/src/test/resources/GobblinYarnAppLauncherTest.conf
b/gobblin-yarn/src/test/resources/GobblinYarnAppLauncherTest.conf
index d5d6bb3..89edb97 100644
--- a/gobblin-yarn/src/test/resources/GobblinYarnAppLauncherTest.conf
+++ b/gobblin-yarn/src/test/resources/GobblinYarnAppLauncherTest.conf
@@ -21,3 +21,6 @@ include "reference"
gobblin.cluster.helix.cluster.name=GobblinYarnAppLauncherTest
gobblin.yarn.app.name=GobblinYarnAppLauncherTest
gobblin.yarn.work.dir=GobblinYarnAppLauncherTest
+
+gobblin.yarn.app.master.jvmMemoryOverheadMbs=10
+gobblin.yarn.app.master.jvmMemoryXmxRatio=0.8
