Repository: incubator-gobblin Updated Branches: refs/heads/master 49974214a -> d75d8fbb5
[GOBBLIN-625] Add distributd job launcher support Closes #2495 from kyuamazon/disttag Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/d75d8fbb Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/d75d8fbb Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/d75d8fbb Branch: refs/heads/master Commit: d75d8fbb5ebe2ada80e11dc69f612eaa05bed143 Parents: 4997421 Author: Kuai Yu <[email protected]> Authored: Mon Nov 5 13:21:20 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Mon Nov 5 13:21:20 2018 -0800 ---------------------------------------------------------------------- .../gobblin/cluster/GobblinClusterConfigurationKeys.java | 1 + .../cluster/GobblinHelixDistributeJobExecutionLauncher.java | 7 +++++++ 2 files changed, 8 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d75d8fbb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java index 3fb665e..00a2e91 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java @@ -71,6 +71,7 @@ public class GobblinClusterConfigurationKeys { // Helix related tagging public static final String HELIX_JOB_TAG_KEY = GOBBLIN_CLUSTER_PREFIX + "helixJobTag"; + public static final String HELIX_PLANNING_JOB_TAG_KEY = GOBBLIN_CLUSTER_PREFIX + "helixPlanningJobTag"; public static final String HELIX_INSTANCE_TAGS_KEY = GOBBLIN_CLUSTER_PREFIX + "helixInstanceTags"; // Planning job properties http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d75d8fbb/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java index f7b0a83..707bdca 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java @@ -209,6 +209,13 @@ class GobblinHelixDistributeJobExecutionLauncher implements JobExecutionLauncher // Planning job never timeout (Helix defaults 1h timeout, set a large number '1 month') jobConfigBuilder.setTimeoutPerTask(JobConfig.DEFAULT_TIMEOUT_PER_TASK * 24 * 30); + // Planning job should have its own tag support + if (jobProps.containsKey(GobblinClusterConfigurationKeys.HELIX_PLANNING_JOB_TAG_KEY)) { + String jobTag = jobProps.getProperty(GobblinClusterConfigurationKeys.HELIX_JOB_TAG_KEY); + log.info("PlanningJob {} has tags associated : {}", planningId, jobTag); + jobConfigBuilder.setInstanceGroupTag(jobTag); + } + jobConfigBuilder.setNumConcurrentTasksPerInstance(PropertiesUtils.getPropAsInt(jobProps, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY, GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT));
