[samza] branch master updated: SAMZA-2499: Add submission config file support (#1336)

[cameronlee]

This is an automated email from the ASF dual-hosted git repository.

cameronlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git


The following commit(s) were added to refs/heads/master by this push:
     new a086ae0  SAMZA-2499: Add submission config file support (#1336)
a086ae0 is described below

commit a086ae0a88b114109b8f3da6d487009505132c82
Author: Ke Wu <ke.wu...@icloud.com>
AuthorDate: Thu Apr 2 14:13:02 2020 -0700

    SAMZA-2499: Add submission config file support (#1336)
    
    API Changes:
    1. Add --config-path option support when running run-app.sh
    
    Upgrade Instructions:
    None
    
    Usage Instructions:
    Alternative way when submitting job,
    
    deploy/samza/bin/run-app.sh \
      --config app.class=A \
      --config job.stream.factory=B \
      --config yarn.package.path=C \
      --config job.config.loader.factory=D \
      --config job.config.loader.properties.xx=E \
      --config job.name=F
    
    can be simplied to
    
    deploy/samza/bin/run-app.sh 
--config-path=/path/to/submission/properties/file/submission.properties
    
    where submission.properties contains
    
    app.class=A
    job.stream.factory=B
    yarn.package.path=C
    job.config.loader.factory=D
    job.config.loader.properties.xx=E
    job.name=F
---
 .../samza/runtime/ApplicationRunnerMain.java       | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git 
a/samza-core/src/main/java/org/apache/samza/runtime/ApplicationRunnerMain.java 
b/samza-core/src/main/java/org/apache/samza/runtime/ApplicationRunnerMain.java
index 220b68b..b917d4d 100644
--- 
a/samza-core/src/main/java/org/apache/samza/runtime/ApplicationRunnerMain.java
+++ 
b/samza-core/src/main/java/org/apache/samza/runtime/ApplicationRunnerMain.java
@@ -19,10 +19,14 @@
 
 package org.apache.samza.runtime;
 
+import java.util.HashMap;
+import java.util.Map;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.samza.config.Config;
+import org.apache.samza.config.ConfigLoader;
 import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.loaders.PropertiesConfigLoader;
 import org.apache.samza.util.CommandLine;
 
 
@@ -40,6 +44,12 @@ public class ApplicationRunnerMain {
             .describedAs("operation=run")
             .defaultsTo("run");
 
+    OptionSpec<String> configPathOpt =
+        parser().accepts("config-path", "File path to submission properties 
file.")
+            .withOptionalArg()
+            .ofType(String.class)
+            .describedAs("path");
+
     ApplicationRunnerOperation getOperation(OptionSet options) {
       String rawOp = options.valueOf(operationOpt);
       return ApplicationRunnerOperation.fromString(rawOp);
@@ -47,8 +57,16 @@ public class ApplicationRunnerMain {
 
     @Override
     public Config loadConfig(OptionSet options) {
-      // ConfigLoader is not supposed to be invoked to load full job config 
during job submission.
-      return new MapConfig(getConfigOverrides(options));
+      Map<String, String> submissionConfig = new HashMap<>();
+
+      if (options.has(configPathOpt)) {
+        ConfigLoader loader = new 
PropertiesConfigLoader(options.valueOf(configPathOpt));
+        submissionConfig.putAll(loader.getConfig());
+      }
+
+      submissionConfig.putAll(getConfigOverrides(options));
+
+      return new MapConfig(submissionConfig);
     }
   }