This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 1ca3192a2 [GOBBLIN-1914] Configurable Gobblin Application Master class
for Yarn (#3781)
1ca3192a2 is described below
commit 1ca3192a29a3fd45045ca5bcddbdf58763ae1f79
Author: Matthew Ho <[email protected]>
AuthorDate: Thu Sep 21 16:21:52 2023 -0700
[GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn
(#3781)
* [GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn
* Configurable logs
---
.../gobblin/yarn/GobblinYarnAppLauncher.java | 23 ++++++++++++++++------
.../gobblin/yarn/GobblinYarnConfigurationKeys.java | 3 +++
.../gobblin/yarn/GobblinYarnAppLauncherTest.java | 3 +++
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 48ac89479..391b160fd 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -801,23 +801,34 @@ public class GobblinYarnAppLauncher {
@VisibleForTesting
protected String buildApplicationMasterCommand(String applicationId, int
memoryMbs) {
- String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
+ Class appMasterClass;
+ try {
+ String appMasterClassName = ConfigUtils.getString(
+ config, GobblinYarnConfigurationKeys.APP_MASTER_CLASS,
GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_CLASS);
+ appMasterClass = Class.forName(appMasterClassName);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ String logFileName = ConfigUtils.getString(config,
+ GobblinYarnConfigurationKeys.APP_MASTER_LOG_FILE_NAME,
appMasterClass.getSimpleName());
+
return new StringBuilder()
.append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
.append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) -
this.jvmMemoryOverheadMbs).append("M")
.append("
-D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=").append(this.containerTimezone)
.append("
-D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
- .append("
-D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+ .append("
-D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(logFileName).append(".").append(ApplicationConstants.STDOUT)
.append(" ").append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs))
- .append(" ").append(GobblinApplicationMaster.class.getName())
+ .append(" ").append(appMasterClass.getName())
.append("
--").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
.append(" ").append(this.applicationName)
.append("
--").append(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)
.append(" ").append(applicationId)
.append("
1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
- appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+ logFileName).append(".").append(ApplicationConstants.STDOUT)
.append("
2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
- appMasterClassName).append(".").append(ApplicationConstants.STDERR)
+ logFileName).append(".").append(ApplicationConstants.STDERR)
.toString();
}
@@ -1071,4 +1082,4 @@ public class GobblinYarnAppLauncher {
gobblinYarnAppLauncher.launch();
}
-}
\ No newline at end of file
+}
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 7088dfa99..f1cc4ada1 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -30,6 +30,9 @@ public class GobblinYarnConfigurationKeys {
public static final String GOBBLIN_YARN_PREFIX = "gobblin.yarn.";
// General Gobblin Yarn application configuration properties.
+ public static final String APP_MASTER_CLASS = GOBBLIN_YARN_PREFIX +
"app.master.class";
+ public static final String DEFAULT_APP_MASTER_CLASS =
GobblinApplicationMaster.class.getName();
+ public static final String APP_MASTER_LOG_FILE_NAME = GOBBLIN_YARN_PREFIX +
"app.master.log.file.name";
public static final String APPLICATION_NAME_KEY = GOBBLIN_YARN_PREFIX +
"app.name";
public static final String APP_QUEUE_KEY = GOBBLIN_YARN_PREFIX + "app.queue";
public static final String APP_REPORT_INTERVAL_MINUTES_KEY =
GOBBLIN_YARN_PREFIX + "app.report.interval.minutes";
diff --git
a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
index 65e9dc269..756e88241 100644
---
a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
+++
b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
@@ -229,6 +229,9 @@ public class GobblinYarnAppLauncherTest implements
HelixMessageTestBase {
// 41 is from 64 * 0.8 - 10
Assert.assertTrue(command.contains("-Xmx41"));
+
Assert.assertTrue(command.contains("org.apache.gobblin.yarn.GobblinApplicationMaster"));
+ Assert.assertTrue(command.contains("GobblinApplicationMaster.stdout"));
+ Assert.assertTrue(command.contains("GobblinApplicationMaster.stderr"));
}
@Test
