Repository: incubator-gobblin Updated Branches: refs/heads/master 396fc40de -> bb7f6b372
[GOBBLIN-229] Cleanup job.state file upon job completion in cluster mode Closes #2083 from yukuai518/jobstateclean Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/bb7f6b37 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/bb7f6b37 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/bb7f6b37 Branch: refs/heads/master Commit: bb7f6b372c450526fe7a29621afe0e7ebf33114e Parents: 396fc40 Author: Kuai Yu <[email protected]> Authored: Mon Aug 28 17:17:46 2017 -0700 Committer: Hung Tran <[email protected]> Committed: Mon Aug 28 17:17:46 2017 -0700 ---------------------------------------------------------------------- .../org/apache/gobblin/cluster/GobblinHelixJobLauncher.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/bb7f6b37/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java ---------------------------------------------------------------------- diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java index 80c0667..1e645d9 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java @@ -221,7 +221,7 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { } finally { // The last iteration of output TaskState collecting will run when the collector service gets stopped this.taskStateCollectorService.stopAsync().awaitTerminated(); - deletePersistedWorkUnitsForJob(); + cleanupWorkingDirectory(); } } @@ -352,11 +352,14 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { } /** - * Delete persisted {@link WorkUnit}s upon job completion. + * Delete persisted {@link WorkUnit}s and {@link JobState} upon job completion. */ - private void deletePersistedWorkUnitsForJob() throws IOException { + private void cleanupWorkingDirectory() throws IOException { LOGGER.info("Deleting persisted work units for job " + this.jobContext.getJobId()); stateStores.wuStateStore.delete(this.jobContext.getJobId()); + LOGGER.info("Deleting job state file for job " + this.jobContext.getJobId()); + Path jobStateFilePath = new Path(this.appWorkDir, this.jobContext.getJobId() + "." + JOB_STATE_FILE_NAME); + this.fs.delete(jobStateFilePath, false); } /**
