Repository: incubator-gobblin Updated Branches: refs/heads/master db8ee526f -> d63141a03
[GOBBLIN-377] Add debug logging to print out job configuration in gobblin cluster Closes #2251 from htran1/cluster_debug Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/d63141a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/d63141a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/d63141a0 Branch: refs/heads/master Commit: d63141a0385df848a417b8b9fdf95f500905bda4 Parents: db8ee52 Author: Hung Tran <[email protected]> Authored: Wed Jan 17 17:54:34 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Wed Jan 17 17:54:45 2018 -0800 ---------------------------------------------------------------------- .../org/apache/gobblin/cluster/GobblinHelixJobLauncher.java | 6 ++++++ .../java/org/apache/gobblin/cluster/GobblinTaskRunner.java | 3 +++ .../src/main/java/org/apache/gobblin/cluster/SingleTask.java | 3 +++ 3 files changed, 12 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d63141a0/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java index 79f3b9e..fc78053 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java @@ -132,6 +132,7 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { List<? extends Tag<?>> metadataTags, ConcurrentHashMap<String, Boolean> runningMap) throws Exception { super(jobProps, addAdditionalMetadataTags(jobProps, metadataTags)); + LOGGER.debug("GobblinHelixJobLauncher: jobProps {}, appWorkDir {}", jobProps, appWorkDir); this.helixManager = helixManager; this.helixTaskDriver = new TaskDriver(this.helixManager); @@ -269,6 +270,9 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { Path jobStateFilePath = new Path(this.appWorkDir, this.jobContext.getJobId() + "." + JOB_STATE_FILE_NAME); SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState()); + + LOGGER.debug("GobblinHelixJobLauncher.createJob: jobStateFilePath {}, jobState {} jobProperties {}", + jobStateFilePath, this.jobContext.getJobState().toString(), this.jobContext.getJobState().getProperties()); } JobConfig.Builder jobConfigBuilder = new JobConfig.Builder(); @@ -440,6 +444,8 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY, ""))); metadataTags.add(new Tag<>(GobblinClusterMetricTagNames.JOB_EXECUTION_ID, jobExecutionId)); + LOGGER.debug("GobblinHelixJobLauncher.addAdditionalMetadataTags: metadataTags {}", metadataTags); + return metadataTags; } } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d63141a0/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java index 2580b0e..dead73b 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinTaskRunner.java @@ -165,6 +165,9 @@ public class GobblinTaskRunner { services.addAll(getServices()); this.serviceManager = new ServiceManager(services); + + logger.debug("GobblinTaskRunner: applicationName {}, helixInstanceName {}, applicationId {}, taskRunnerId {}, config {}, appWorkDir {}", + applicationName, helixInstanceName, applicationId, taskRunnerId, config, appWorkDirOptional); } private Path initAppWorkDir(Config config, Optional<Path> appWorkDirOptional) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d63141a0/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTask.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTask.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTask.java index da0c633..3b69e0c 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTask.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/SingleTask.java @@ -73,6 +73,9 @@ public class SingleTask { JobState jobState = getJobState(); Config jobConfig = getConfigFromJobState(jobState); + _logger.debug("SingleTask.run: jobId {} workUnitFilePath {} jobStateFilePath {} jobState {} jobConfig {}", + _jobId, _workUnitFilePath, _jobStateFilePath, jobState, jobConfig); + try (SharedResourcesBroker<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(jobConfig, GobblinScopeTypes.GLOBAL.defaultScopeInstance())) { SharedResourcesBroker<GobblinScopeTypes> jobBroker = getJobBroker(jobState, globalBroker);
