Repository: incubator-gobblin Updated Branches: refs/heads/master 6403d247e -> c0c77ba54
[GOBBLIN-476] Add helix task timeout in GobblinHelixJobLauncher Closes #2347 from yukuai518/time Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/c0c77ba5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/c0c77ba5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/c0c77ba5 Branch: refs/heads/master Commit: c0c77ba5470494d200c394697cbceecf41302db8 Parents: 6403d24 Author: Kuai Yu <[email protected]> Authored: Wed Apr 25 10:23:06 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Wed Apr 25 10:23:06 2018 -0700 ---------------------------------------------------------------------- .../org/apache/gobblin/configuration/ConfigurationKeys.java | 3 +++ .../org/apache/gobblin/cluster/GobblinHelixJobLauncher.java | 5 +++++ 2 files changed, 8 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c0c77ba5/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index a0eeca3..2291d72 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -117,6 +117,9 @@ public class ConfigurationKeys { public static final String HELIX_JOB_TIMEOUT_SECONDS = "job.timeout.seconds"; public static final String DEFAULT_HELIX_JOB_TIMEOUT_SECONDS = "10800"; + public static final String HELIX_TASK_TIMEOUT_SECONDS = "task.timeout.seconds"; + public static final long DEFAULT_HELIX_TASK_TIMEOUT_SECONDS = 60 * 60; + /** * Task executor and state tracker configuration properties. */ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c0c77ba5/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java index 5035216..8c7bbe1 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java @@ -253,6 +253,11 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { JobConfig.Builder jobConfigBuilder = new JobConfig.Builder(); jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt( ConfigurationKeys.MAX_TASK_RETRIES_KEY, ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES)); + + jobConfigBuilder.setTimeoutPerTask(this.jobContext.getJobState().getPropAsLong( + ConfigurationKeys.HELIX_TASK_TIMEOUT_SECONDS, + ConfigurationKeys.DEFAULT_HELIX_TASK_TIMEOUT_SECONDS) * 1000); + jobConfigBuilder.setFailureThreshold(workUnits.size()); jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME); jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig,
