Repository: incubator-gobblin Updated Branches: refs/heads/master 7e1d8433a -> 0795fa7a0
[GOBBLIN-429] Pass jvm options as a list to child process Closes #2306 from yukuai518/jvm Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/0795fa7a Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/0795fa7a Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/0795fa7a Branch: refs/heads/master Commit: 0795fa7a07d17957288a3c9cde75dde91e7649be Parents: 7e1d843 Author: Kuai Yu <[email protected]> Authored: Fri Mar 16 13:21:13 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Fri Mar 16 13:21:13 2018 -0700 ---------------------------------------------------------------------- .../cluster/GobblinClusterConfigurationKeys.java | 4 ++-- .../org/apache/gobblin/cluster/SingleTaskLauncher.java | 12 ++++++++---- gobblin-cluster/src/test/resources/BasicCluster.conf | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java index 5719b23..76382e7 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java @@ -37,8 +37,8 @@ public class GobblinClusterConfigurationKeys { GOBBLIN_CLUSTER_PREFIX + "task.classpath"; public static final String TASK_LOG_CONFIG = GOBBLIN_CLUSTER_PREFIX + "task.log.config"; - public static final String TASK_JAVA_OPTIONS = - GOBBLIN_CLUSTER_PREFIX + "task.java.options"; + public static final String TASK_JVM_OPTIONS = + GOBBLIN_CLUSTER_PREFIX + "task.jvm.options"; // General Gobblin Cluster application configuration properties. public static final String APPLICATION_NAME_OPTION_NAME = "app_name"; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java index 078f122..8cbbc00 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTaskLauncher.java @@ -23,6 +23,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.text.StrTokenizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +61,7 @@ class SingleTaskLauncher { // The -cp parameter list can be very long. final String completeCmdLine = String.join(" ", command); - logger.debug("cmd line:\n{}", completeCmdLine); + logger.info("cmd line:\n{}", completeCmdLine); final Process taskProcess = this.processBuilder.start(command); @@ -88,9 +89,12 @@ class SingleTaskLauncher { } private void addJavaOptions() { - if (sysConfig.hasPath(GobblinClusterConfigurationKeys.TASK_JAVA_OPTIONS)) { - final String javaOptions = sysConfig.getString(GobblinClusterConfigurationKeys.TASK_JAVA_OPTIONS); - this.cmd.add(javaOptions); + if (sysConfig.hasPath(GobblinClusterConfigurationKeys.TASK_JVM_OPTIONS)) { + final String taskJvmOptions = sysConfig.getString(GobblinClusterConfigurationKeys.TASK_JVM_OPTIONS); + StrTokenizer tokenizer = new StrTokenizer(taskJvmOptions, ' ', '"'); + while(tokenizer.hasNext()) { + this.cmd.add(tokenizer.next()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0795fa7a/gobblin-cluster/src/test/resources/BasicCluster.conf ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/test/resources/BasicCluster.conf b/gobblin-cluster/src/test/resources/BasicCluster.conf index a42b41d..92016f8 100644 --- a/gobblin-cluster/src/test/resources/BasicCluster.conf +++ b/gobblin-cluster/src/test/resources/BasicCluster.conf @@ -21,4 +21,6 @@ gobblin.cluster.workDir=/tmp/gobblinClusterBasicTest/ gobblin.cluster.job.conf.path=${gobblin.cluster.workDir}/jobs gobblin.cluster.standaloneMode=true gobblin.cluster.job.executeInSchedulingThread=false +gobblin.cluster.enableTaskInSeparateProcess=false +gobblin.cluster.task.jvm.options="-Xms10m -Xmx1g -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20" jobexecutor.threadpool.size=20
