[gobblin] branch master updated: GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode (#3379)

[suvasude]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new a18d4e0  GOBBLIN-1528: Reduce overshoot of Yarn containers in 
Gobblin-on-Yarn mode (#3379)
a18d4e0 is described below

commit a18d4e01f9999426c541441ea4d7f88f82c4b2b0
Author: Sudarshan Vasudevan <sudarsh...@gmail.com>
AuthorDate: Mon Aug 30 11:08:46 2021 -0700

    GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode 
(#3379)
    
    * GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode
    
    * Clarify comment
---
 .../java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java   | 4 ++++
 gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java  | 5 ++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 512d24f..6e374d3 100644
--- 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -131,4 +131,8 @@ public class GobblinYarnConfigurationKeys {
   //Container classpaths properties
   public static final String GOBBLIN_YARN_ADDITIONAL_CLASSPATHS = 
GOBBLIN_YARN_PREFIX + "additional.classpaths";
   public static final String GOBBLIN_YARN_CLASSPATHS = GOBBLIN_YARN_PREFIX + 
"classpaths";
+
+  //Config to control Heartbeat interval for Yarn AMRM client.
+  public static final String AMRM_HEARTBEAT_INTERVAL_SECS = 
GOBBLIN_YARN_PREFIX + "amRmHeartbeatIntervalSecs";
+  public static final Integer DEFAULT_AMRM_HEARTBEAT_INTERVAL_SECS = 15;
 }
diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index d1bce22..9fadc94 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -221,8 +221,11 @@ public class YarnService extends AbstractIdleService {
     this.yarnConfiguration = yarnConfiguration;
     this.fs = fs;
 
+    int amRmHeartbeatIntervalMillis = Long.valueOf(TimeUnit.SECONDS.toMillis(
+        ConfigUtils.getInt(config, 
GobblinYarnConfigurationKeys.AMRM_HEARTBEAT_INTERVAL_SECS,
+            
GobblinYarnConfigurationKeys.DEFAULT_AMRM_HEARTBEAT_INTERVAL_SECS))).intValue();
     this.amrmClientAsync = closer.register(
-        AMRMClientAsync.createAMRMClientAsync(1000, new 
AMRMClientCallbackHandler()));
+        AMRMClientAsync.createAMRMClientAsync(amRmHeartbeatIntervalMillis, new 
AMRMClientCallbackHandler()));
     this.amrmClientAsync.init(this.yarnConfiguration);
     this.nmClientAsync = 
closer.register(NMClientAsync.createNMClientAsync(getNMClientCallbackHandler()));
     this.nmClientAsync.init(this.yarnConfiguration);