[samza] branch master updated: SAMZA-2628: Include the localized resource lib directory in the classpath of AM container (#1469)

[atoomula]

This is an automated email from the ASF dual-hosted git repository.

atoomula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git


The following commit(s) were added to refs/heads/master by this push:
     new 66b2efd  SAMZA-2628: Include the localized resource lib directory in 
the classpath of AM container (#1469)
66b2efd is described below

commit 66b2efd1ee96436dd4f4af798da458bc5928d824
Author: Aditya Toomula <atoom...@linkedin.com>
AuthorDate: Wed Mar 3 12:29:19 2021 -0800

    SAMZA-2628: Include the localized resource lib directory in the classpath 
of AM container (#1469)
    
    * SAMZA-2628: Include the localized resource lib directory in the classpath 
of AM container
    
    * SAMZA-2628: Include the localized resource lib directory in the classpath 
of AM container
    
    * Empty commit to trigger build.
---
 .../scala/org/apache/samza/job/yarn/YarnJob.scala  |  2 ++
 .../org/apache/samza/job/yarn/TestYarnJob.java     | 34 +++++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala 
b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
index ca681f5..7e4565b 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala
@@ -194,6 +194,8 @@ object YarnJob extends Logging {
     Option.apply(yarnConfig.getAMJavaHome).foreach {
       amJavaHome => envMapBuilder += ShellCommandConfig.ENV_JAVA_HOME -> 
amJavaHome
     }
+    envMapBuilder += ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR ->
+      
Util.envVarEscape(config.get(ShellCommandConfig.ADDITIONAL_CLASSPATH_DIR, ""))
     envMapBuilder.result()
   }
 
diff --git 
a/samza-yarn/src/test/java/org/apache/samza/job/yarn/TestYarnJob.java 
b/samza-yarn/src/test/java/org/apache/samza/job/yarn/TestYarnJob.java
index 4858d76..f068800 100644
--- a/samza-yarn/src/test/java/org/apache/samza/job/yarn/TestYarnJob.java
+++ b/samza-yarn/src/test/java/org/apache/samza/job/yarn/TestYarnJob.java
@@ -65,7 +65,8 @@ public class TestYarnJob {
     Map<String, String> expected = ImmutableMap.of(
         ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG, 
expectedCoordinatorStreamConfigStringValue,
         ShellCommandConfig.ENV_JAVA_OPTS, Util.envVarEscape(amJvmOptions),
-        ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "false");
+        ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "false",
+        ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR, "");
     assertEquals(expected, JavaConverters.mapAsJavaMapConverter(
         YarnJob$.MODULE$.buildEnvironment(config, new YarnConfig(config), new 
JobConfig(config))).asJava());
   }
@@ -85,7 +86,8 @@ public class TestYarnJob {
         ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG, 
expectedCoordinatorStreamConfigStringValue,
         ShellCommandConfig.ENV_JAVA_OPTS, "",
         ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "true",
-        ShellCommandConfig.ENV_APPLICATION_LIB_DIR, "./__package/lib");
+        ShellCommandConfig.ENV_APPLICATION_LIB_DIR, "./__package/lib",
+        ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR, "");
     assertEquals(expected, JavaConverters.mapAsJavaMapConverter(
         YarnJob$.MODULE$.buildEnvironment(config, new YarnConfig(config), new 
JobConfig(config))).asJava());
   }
@@ -106,7 +108,8 @@ public class TestYarnJob {
         ShellCommandConfig.ENV_COORDINATOR_SYSTEM_CONFIG, 
expectedCoordinatorStreamConfigStringValue,
         ShellCommandConfig.ENV_JAVA_OPTS, "",
         ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "false",
-        ShellCommandConfig.ENV_JAVA_HOME, "/some/path/to/java/home");
+        ShellCommandConfig.ENV_JAVA_HOME, "/some/path/to/java/home",
+        ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR, "");
     assertEquals(expected, JavaConverters.mapAsJavaMapConverter(
         YarnJob$.MODULE$.buildEnvironment(config, new YarnConfig(config), new 
JobConfig(config))).asJava());
   }
@@ -126,7 +129,30 @@ public class TestYarnJob {
         ShellCommandConfig.ENV_SUBMISSION_CONFIG, expectedSubmissionConfig,
         ShellCommandConfig.ENV_JAVA_OPTS, "",
         ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "true",
-        ShellCommandConfig.ENV_APPLICATION_LIB_DIR, "./__package/lib");
+        ShellCommandConfig.ENV_APPLICATION_LIB_DIR, "./__package/lib",
+        ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR, "");
+    assertEquals(expected, JavaConverters.mapAsJavaMapConverter(
+        YarnJob$.MODULE$.buildEnvironment(config, new YarnConfig(config), new 
JobConfig(config))).asJava());
+  }
+
+  @Test
+  public void testBuildJobWithAdditionalClassPath() throws IOException {
+    Config config = new MapConfig(new ImmutableMap.Builder<String, String>()
+        .put(JobConfig.JOB_NAME, "jobName")
+        .put(JobConfig.JOB_ID, "jobId")
+        .put(JobConfig.CONFIG_LOADER_FACTORY, 
"org.apache.samza.config.loaders.PropertiesConfigLoaderFactory")
+        .put(YarnConfig.AM_JVM_OPTIONS, "")
+        .put(JobConfig.JOB_SPLIT_DEPLOYMENT_ENABLED, "true")
+        .put(ShellCommandConfig.ADDITIONAL_CLASSPATH_DIR, "./sqlapp/lib/*")
+        .build());
+    String expectedSubmissionConfig = 
Util.envVarEscape(SamzaObjectMapper.getObjectMapper()
+        .writeValueAsString(config));
+    Map<String, String> expected = ImmutableMap.of(
+        ShellCommandConfig.ENV_SUBMISSION_CONFIG, expectedSubmissionConfig,
+        ShellCommandConfig.ENV_JAVA_OPTS, "",
+        ShellCommandConfig.ENV_SPLIT_DEPLOYMENT_ENABLED, "true",
+        ShellCommandConfig.ENV_APPLICATION_LIB_DIR, "./__package/lib",
+        ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR, "./sqlapp/lib/*");
     assertEquals(expected, JavaConverters.mapAsJavaMapConverter(
         YarnJob$.MODULE$.buildEnvironment(config, new YarnConfig(config), new 
JobConfig(config))).asJava());
   }