Repository: incubator-apex-malhar Updated Branches: refs/heads/devel-3 f070c33bf -> 480d6ed6c
MLHR-1931 #resolve added exists and listFiles to FileAccess Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/bc9a51fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/bc9a51fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/bc9a51fb Branch: refs/heads/devel-3 Commit: bc9a51fbc467de4843a487d8f9781600f181a045 Parents: 54106d4 Author: Chandni Singh <[email protected]> Authored: Sun Dec 6 01:20:25 2015 -0800 Committer: Chandni Singh <[email protected]> Committed: Sun Dec 6 01:20:25 2015 -0800 ---------------------------------------------------------------------- .../datatorrent/lib/fileaccess/FileAccess.java | 22 ++++++++++++++++++++ .../lib/fileaccess/FileAccessFSImpl.java | 18 ++++++++++++++++ 2 files changed, 40 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/bc9a51fb/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java ---------------------------------------------------------------------- diff --git a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java index 4b7f6e5..16d6d72 100644 --- a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java +++ b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccess.java @@ -24,6 +24,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.TreeMap; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.RemoteIterator; + import com.datatorrent.netlet.util.Slice; /** @@ -52,6 +55,25 @@ public interface FileAccess extends Closeable long getFileSize(long bucketKey, String s) throws IOException; /** + * Checks if a file exists under a bucket. + * + * @param bucketKey bucket key + * @param fileName file name + * @return true if file exists; false otherwise. + * @throws IOException + */ + boolean exists(long bucketKey, String fileName) throws IOException; + + /** + * Lists the files in the bucket path. + * + * @param bucketKey bucket key + * @return status of all the files in the bucket path if the bucket exists; null otherwise. + * @throws IOException + */ + RemoteIterator<LocatedFileStatus> listFiles(long bucketKey) throws IOException; + + /** * Data File Format Reader */ interface FileReader extends Closeable http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/bc9a51fb/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java ---------------------------------------------------------------------- diff --git a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java index 80a201a..5c7e8d3 100644 --- a/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java +++ b/library/src/main/java/com/datatorrent/lib/fileaccess/FileAccessFSImpl.java @@ -27,8 +27,10 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import com.datatorrent.netlet.util.DTThrowable; @@ -122,6 +124,22 @@ public abstract class FileAccessFSImpl implements FileAccess } @Override + public boolean exists(long bucketKey, String fileName) throws IOException + { + return fs.exists(getFilePath(bucketKey, fileName)); + } + + @Override + public RemoteIterator<LocatedFileStatus> listFiles(long bucketKey) throws IOException + { + Path bucketPath = getBucketPath(bucketKey); + if (!fs.exists(bucketPath)) { + return null; + } + return fs.listFiles(bucketPath, true); + } + + @Override public String toString() { return this.getClass().getSimpleName() + "[basePath=" + basePath + "]";
