[incubator-gobblin] branch master updated: [GOBBLIN-961] Bypass locked directories when calculating src watermark

[suvasude]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 1b3a16e  [GOBBLIN-961] Bypass locked directories when calculating src 
watermark
1b3a16e is described below

commit 1b3a16e1cb9e0494e3d875ca66cbd01c7ea33517
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Fri Nov 15 10:54:42 2019 -0800

    [GOBBLIN-961] Bypass locked directories when calculating src watermark
    
    Closes #2810 from ZihanLi58/GOBBLIN-961
---
 .../data/management/copy/replication/SourceHadoopFsEndPoint.java   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git 
a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
 
b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
index 2a56f2e..2137d77 100644
--- 
a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
+++ 
b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/replication/SourceHadoopFsEndPoint.java
@@ -75,8 +75,11 @@ public class SourceHadoopFsEndPoint extends HadoopFsEndPoint 
{
 
       Collection<Path> validPaths = 
ReplicationDataValidPathPicker.getValidPaths(this);
       for (Path p : validPaths) {
-        this.allFileStatus.addAll(
-            FileListUtils.listFilesRecursively(fs, p, super.getPathFilter(), 
super.isApplyFilterToDirectories()));
+        try {
+          this.allFileStatus.addAll(FileListUtils.listFilesRecursively(fs, p, 
super.getPathFilter(), super.isApplyFilterToDirectories()));
+        } catch (Exception e) {
+          log.error(String.format("Error while try read file in directory %s 
to get watermark", p.toString()));
+        }
       }
 
       for (FileStatus f : this.allFileStatus) {