[gobblin] branch master updated: use user supplied props to create FileSystem in DatasetCleanerTask (#3483)

[zihanli58]

This is an automated email from the ASF dual-hosted git repository.

zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 8966a11a4 use user supplied props to create FileSystem in 
DatasetCleanerTask (#3483)
8966a11a4 is described below

commit 8966a11a4cd26f85da295c20906f1f4a0a7236b9
Author: Arjun Singh Bora <ab...@linkedin.com>
AuthorDate: Tue Apr 26 11:00:36 2022 -0700

    use user supplied props to create FileSystem in DatasetCleanerTask (#3483)
---
 .../org/apache/gobblin/runtime/retention/DatasetCleanerTask.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git 
a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
 
b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
index 4c5cbd29f..719ecc163 100644
--- 
a/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
+++ 
b/gobblin-data-management/src/main/java/org/apache/gobblin/runtime/retention/DatasetCleanerTask.java
@@ -28,6 +28,7 @@ import org.apache.gobblin.configuration.WorkUnitState;
 import org.apache.gobblin.data.management.retention.DatasetCleaner;
 import org.apache.gobblin.runtime.TaskContext;
 import org.apache.gobblin.runtime.task.BaseAbstractTask;
+import org.apache.gobblin.util.JobConfigurationUtils;
 
 
 /**
@@ -48,8 +49,9 @@ public class DatasetCleanerTask extends BaseAbstractTask {
   @Override
   public void run() {
     try {
-      DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(new 
Configuration()),
-          this.taskContext.getTaskState().getProperties());
+      Configuration conf = new Configuration();
+      
JobConfigurationUtils.putStateIntoConfiguration(this.taskContext.getTaskState(),
 conf);
+      DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(conf), 
this.taskContext.getTaskState().getProperties());
       datasetCleaner.clean();
       this.workingState = WorkUnitState.WorkingState.SUCCESSFUL;
     } catch (IOException e) {