Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 c9e4fb83c -> 2bc50dd93
HADOOP-14172. S3Guard: import does not import empty directory. Contributed by Sean Mackrory Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bc50dd9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bc50dd9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bc50dd9 Branch: refs/heads/HADOOP-13345 Commit: 2bc50dd937d582202a8b5d54694beac0f9b72d51 Parents: c9e4fb8 Author: Mingliang Liu <[email protected]> Authored: Thu Apr 6 14:11:34 2017 -0700 Committer: Mingliang Liu <[email protected]> Committed: Thu Apr 6 14:11:58 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 15 +++++++++++++-- .../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 2 +- .../hadoop/fs/s3a/s3guard/TestS3GuardTool.java | 6 +++++- 3 files changed, 19 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bc50dd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 129d4ae..f56259c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -2415,6 +2415,18 @@ public class S3AFileSystem extends FileSystem { @Override public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) throws FileNotFoundException, IOException { + return innerListFiles(f, recursive, + new Listing.AcceptFilesOnly(qualify(f))); + } + + public RemoteIterator<LocatedFileStatus> listFilesAndDirectories(Path f, + boolean recursive) throws IOException { + return innerListFiles(f, recursive, + new Listing.AcceptAllButSelfAndS3nDirs(qualify(f))); + } + + private RemoteIterator<LocatedFileStatus> innerListFiles(Path f, boolean + recursive, Listing.FileStatusAcceptor acceptor) throws IOException { incrementStatistic(INVOCATION_LIST_FILES); Path path = qualify(f); LOG.debug("listFiles({}, {})", path, recursive); @@ -2435,8 +2447,7 @@ public class S3AFileSystem extends FileSystem { return listing.createLocatedFileStatusIterator( listing.createFileStatusListingIterator(path, createListObjectsRequest(key, delimiter), - ACCEPT_ALL, - new Listing.AcceptFilesOnly(path))); + ACCEPT_ALL, acceptor)); } } catch (AmazonClientException e) { // TODO s3guard: http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bc50dd9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 9579839..fef8e2d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -445,7 +445,7 @@ public abstract class S3GuardTool extends Configured implements Tool { private void importDir(FileStatus status) throws IOException { Preconditions.checkArgument(status.isDirectory()); RemoteIterator<LocatedFileStatus> it = - s3a.listFiles(status.getPath(), true); + s3a.listFilesAndDirectories(status.getPath(), true); while (it.hasNext()) { LocatedFileStatus located = it.next(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bc50dd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java index 132cf1a..8874511 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java @@ -55,6 +55,8 @@ public class TestS3GuardTool extends S3GuardToolTestBase { fs.mkdirs(parent); Path dir = new Path(parent, "a"); fs.mkdirs(dir); + Path emptyDir = new Path(parent, "emptyDir"); + fs.mkdirs(emptyDir); for (int i = 0; i < 10; i++) { String child = String.format("file-%d", i); try (FSDataOutputStream out = fs.create(new Path(dir, child))) { @@ -66,12 +68,14 @@ public class TestS3GuardTool extends S3GuardToolTestBase { cmd.setMetadataStore(ms); assertEquals("Import command did not exit successfully - see output", - SUCCESS, cmd.run(new String[]{"import", dir.toString()})); + SUCCESS, cmd.run(new String[]{"import", parent.toString()})); DirListingMetadata children = ms.listChildren(dir); assertEquals("Unexpected number of paths imported", 10, children .getListing().size()); + assertEquals("Expected 2 items: empty directory and a parent directory", 2, + ms.listChildren(parent).getListing().size()); // assertTrue(children.isAuthoritative()); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
