[incubator-gobblin] branch master updated: [GOBBLIN-1165] Add config to enable user to set additional yarn classpathes

[suvasude]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 4325920  [GOBBLIN-1165] Add config to enable user to set additional 
yarn classpathes
4325920 is described below

commit 4325920dc72b82d2a96e2ba52ff0363d6f8575b4
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Mon Jun 1 17:06:36 2020 -0700

    [GOBBLIN-1165] Add config to enable user to set additional yarn classpathes
    
    Closes #3017 from ZihanLi58/GOBBLIN-1165-n
---
 .../java/org/apache/gobblin/yarn/GobblinApplicationMaster.java    | 6 +++++-
 .../org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java     | 2 ++
 .../src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java     | 8 +++++++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
index c1f7cc0..898d96b 100644
--- 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
+++ 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
@@ -26,6 +26,7 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.gobblin.util.logs.LogCopier;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -97,7 +98,10 @@ public class GobblinApplicationMaster extends 
GobblinClusterManager {
       this.applicationLauncher
           .addService(logCopier);
     }
-
+    if (!ConfigUtils.emptyIfNotPresent(config, 
GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS).equals(StringUtils.EMPTY)){
+      LOGGER.info("set additional yarn class path to yarnConfig");
+      
yarnConfiguration.setStrings(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS,
 
config.getString(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS));
+    }
     this.yarnService = buildYarnService(this.config, applicationName, 
this.applicationId, yarnConfiguration, this.fs);
     this.applicationLauncher.addService(this.yarnService);
 
diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 16c82a5..ed19740 100644
--- 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -113,4 +113,6 @@ public class GobblinYarnConfigurationKeys {
 
   //Configuration properties relating to container mode of execution e.g. 
Gobblin cluster runs on Yarn
   public static final String CONTAINER_NUM_KEY = "container.num";
+  //Container classpaths properties
+  public static final String GOBBLIN_YARN_ADDITIONAL_CLASSPATHS = 
GOBBLIN_YARN_PREFIX + "additional.classpaths";
 }
diff --git 
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java 
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
index 7b7f8f5..0b58aea 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
@@ -47,7 +47,6 @@ import com.google.common.collect.Maps;
  * @author Yinan Li
  */
 public class YarnHelixUtils {
-
   /**
    * Write a {@link Token} to a given file.
    *
@@ -128,6 +127,13 @@ public class YarnHelixUtils {
             environmentVariableMap, 
ApplicationConstants.Environment.CLASSPATH.key(), classpath.trim());
       }
     }
+    String[] additionalClassPath = 
yarnConfiguration.getStrings(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS);
+    if (additionalClassPath != null) {
+      for (String classpath : additionalClassPath) {
+        Apps.addToEnvironment(
+            environmentVariableMap, 
ApplicationConstants.Environment.CLASSPATH.key(), classpath.trim());
+      }
+    }
 
     return environmentVariableMap;
   }