[12/44] hadoop git commit: HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a409530
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a409530
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a409530
Branch: refs/heads/YARN-2915
Commit: 8a40953058d50d421d62b71067a13b626b3cba1f
Parents: f6f3a44
Author: Andrew Wang
Authored: Fri Sep 16 15:37:36 2016 -0700
Committer: Andrew Wang
Committed: Fri Sep 16 15:37:36 2016 -0700
--
.../java/org/apache/hadoop/fs/FileSystem.java | 125 ---
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 68 --
.../hadoop/fs/http/client/HttpFSFileSystem.java | 56 +++--
.../hadoop/fs/http/client/HttpFSUtils.java | 2 +
.../hadoop/fs/http/server/FSOperations.java | 62 +
.../http/server/HttpFSParametersProvider.java | 20 +++
.../hadoop/fs/http/server/HttpFSServer.java | 17 +++
.../service/hadoop/FileSystemAccessService.java | 4 +-
.../fs/http/client/BaseTestHttpFSWith.java | 62 -
11 files changed, 340 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a409530/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9bde29d..5939f97 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.Tracer;
import org.apache.htrace.core.TraceScope;
+import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import static com.google.common.base.Preconditions.checkArgument;
@@ -1530,7 +1531,68 @@ public abstract class FileSystem extends Configured
implements Closeable {
*/
public abstract FileStatus[] listStatus(Path f) throws
FileNotFoundException,
IOException;
-
+
+ /**
+ * Represents a batch of directory entries when iteratively listing a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * For internal use by FileSystem subclasses that override
+ * {@link FileSystem#listStatusBatch(Path, byte[])} to implement iterative
+ * listing.
+ */
+ @InterfaceAudience.Private
+ public static class DirectoryEntries {
+private final FileStatus[] entries;
+private final byte[] token;
+private final boolean hasMore;
+
+public DirectoryEntries(FileStatus[] entries, byte[] token, boolean
+hasMore) {
+ this.entries = entries;
+ if (token != null) {
+this.token = token.clone();
+ } else {
+this.token = null;
+ }
+ this.hasMore = hasMore;
+}
+
+public FileStatus[] getEntries() {
+ return entries;
+}
+
+public byte[] getToken() {
+ return token;
+}
+
+public boolean hasMore() {
+ return hasMore;
+}
+ }
+
+ /**
+ * Given an opaque iteration token, return the next batch of entries in a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * This method should be overridden by FileSystem subclasses that want to
+ * use the generic {@link FileSystem#listStatusIterator(Path)}
implementation.
+ * @param f Path to list
+ * @param token opaque iteration token returned by previous call, or null
+ * if this is the first call.
+ * @return
+ * @throws FileNotFoundException
+ * @throws IOException
+ */
+ @InterfaceAudience.Private
+ protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
+ FileNotFoundException, IOException {
+// The default implementation returns the entire listing as a single batch.
+// Thus, there is never a second batch, and no need to respect the passed
+// token or set a token in the returned DirectoryEntries.
+FileStatus[] listing = listStatus(f);
+return new DirectoryEntries(listing, null, false);
+ }
+
/*
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array results.
@@ -1767,6 +1829,49 @@ public abstract class FileSystem extends Configured
implements Closeable {
}
/**
+ * Generic iterator for implementing {@link #listStatusIterator(Path)}.
+
[23/26] hadoop git commit: HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a409530
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a409530
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a409530
Branch: refs/heads/HADOOP-12756
Commit: 8a40953058d50d421d62b71067a13b626b3cba1f
Parents: f6f3a44
Author: Andrew Wang
Authored: Fri Sep 16 15:37:36 2016 -0700
Committer: Andrew Wang
Committed: Fri Sep 16 15:37:36 2016 -0700
--
.../java/org/apache/hadoop/fs/FileSystem.java | 125 ---
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 68 --
.../hadoop/fs/http/client/HttpFSFileSystem.java | 56 +++--
.../hadoop/fs/http/client/HttpFSUtils.java | 2 +
.../hadoop/fs/http/server/FSOperations.java | 62 +
.../http/server/HttpFSParametersProvider.java | 20 +++
.../hadoop/fs/http/server/HttpFSServer.java | 17 +++
.../service/hadoop/FileSystemAccessService.java | 4 +-
.../fs/http/client/BaseTestHttpFSWith.java | 62 -
11 files changed, 340 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a409530/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9bde29d..5939f97 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.Tracer;
import org.apache.htrace.core.TraceScope;
+import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import static com.google.common.base.Preconditions.checkArgument;
@@ -1530,7 +1531,68 @@ public abstract class FileSystem extends Configured
implements Closeable {
*/
public abstract FileStatus[] listStatus(Path f) throws
FileNotFoundException,
IOException;
-
+
+ /**
+ * Represents a batch of directory entries when iteratively listing a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * For internal use by FileSystem subclasses that override
+ * {@link FileSystem#listStatusBatch(Path, byte[])} to implement iterative
+ * listing.
+ */
+ @InterfaceAudience.Private
+ public static class DirectoryEntries {
+private final FileStatus[] entries;
+private final byte[] token;
+private final boolean hasMore;
+
+public DirectoryEntries(FileStatus[] entries, byte[] token, boolean
+hasMore) {
+ this.entries = entries;
+ if (token != null) {
+this.token = token.clone();
+ } else {
+this.token = null;
+ }
+ this.hasMore = hasMore;
+}
+
+public FileStatus[] getEntries() {
+ return entries;
+}
+
+public byte[] getToken() {
+ return token;
+}
+
+public boolean hasMore() {
+ return hasMore;
+}
+ }
+
+ /**
+ * Given an opaque iteration token, return the next batch of entries in a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * This method should be overridden by FileSystem subclasses that want to
+ * use the generic {@link FileSystem#listStatusIterator(Path)}
implementation.
+ * @param f Path to list
+ * @param token opaque iteration token returned by previous call, or null
+ * if this is the first call.
+ * @return
+ * @throws FileNotFoundException
+ * @throws IOException
+ */
+ @InterfaceAudience.Private
+ protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
+ FileNotFoundException, IOException {
+// The default implementation returns the entire listing as a single batch.
+// Thus, there is never a second batch, and no need to respect the passed
+// token or set a token in the returned DirectoryEntries.
+FileStatus[] listing = listStatus(f);
+return new DirectoryEntries(listing, null, false);
+ }
+
/*
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array results.
@@ -1767,6 +1829,49 @@ public abstract class FileSystem extends Configured
implements Closeable {
}
/**
+ * Generic iterator for implementing {@link #listStatusIterator(Path)}.
hadoop git commit: HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 031d5f6c5 -> b03a0be7a
HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
(cherry picked from commit 8a40953058d50d421d62b71067a13b626b3cba1f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b03a0be7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b03a0be7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b03a0be7
Branch: refs/heads/branch-2
Commit: b03a0be7a3830e755155afa170e33ffb80f8acf3
Parents: 031d5f6
Author: Andrew Wang
Authored: Fri Sep 16 15:37:36 2016 -0700
Committer: Andrew Wang
Committed: Fri Sep 16 15:37:49 2016 -0700
--
.../java/org/apache/hadoop/fs/FileSystem.java | 125 ---
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 68 --
.../hadoop/fs/http/client/HttpFSFileSystem.java | 56 +++--
.../hadoop/fs/http/client/HttpFSUtils.java | 2 +
.../hadoop/fs/http/server/FSOperations.java | 62 +
.../http/server/HttpFSParametersProvider.java | 20 +++
.../hadoop/fs/http/server/HttpFSServer.java | 17 +++
.../service/hadoop/FileSystemAccessService.java | 4 +-
.../fs/http/client/BaseTestHttpFSWith.java | 62 -
11 files changed, 340 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b03a0be7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 52b9484..6b06802 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.Tracer;
import org.apache.htrace.core.TraceScope;
+import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import static com.google.common.base.Preconditions.checkArgument;
@@ -1529,7 +1530,68 @@ public abstract class FileSystem extends Configured
implements Closeable {
*/
public abstract FileStatus[] listStatus(Path f) throws
FileNotFoundException,
IOException;
-
+
+ /**
+ * Represents a batch of directory entries when iteratively listing a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * For internal use by FileSystem subclasses that override
+ * {@link FileSystem#listStatusBatch(Path, byte[])} to implement iterative
+ * listing.
+ */
+ @InterfaceAudience.Private
+ public static class DirectoryEntries {
+private final FileStatus[] entries;
+private final byte[] token;
+private final boolean hasMore;
+
+public DirectoryEntries(FileStatus[] entries, byte[] token, boolean
+hasMore) {
+ this.entries = entries;
+ if (token != null) {
+this.token = token.clone();
+ } else {
+this.token = null;
+ }
+ this.hasMore = hasMore;
+}
+
+public FileStatus[] getEntries() {
+ return entries;
+}
+
+public byte[] getToken() {
+ return token;
+}
+
+public boolean hasMore() {
+ return hasMore;
+}
+ }
+
+ /**
+ * Given an opaque iteration token, return the next batch of entries in a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * This method should be overridden by FileSystem subclasses that want to
+ * use the generic {@link FileSystem#listStatusIterator(Path)}
implementation.
+ * @param f Path to list
+ * @param token opaque iteration token returned by previous call, or null
+ * if this is the first call.
+ * @return
+ * @throws FileNotFoundException
+ * @throws IOException
+ */
+ @InterfaceAudience.Private
+ protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
+ FileNotFoundException, IOException {
+// The default implementation returns the entire listing as a single batch.
+// Thus, there is never a second batch, and no need to respect the passed
+// token or set a token in the returned DirectoryEntries.
+FileStatus[] listing = listStatus(f);
+return new DirectoryEntries(listing, null, false);
+ }
+
/*
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array results.
@@ -1767,6 +1829,49 @@ public abstrac
hadoop git commit: HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
Repository: hadoop
Updated Branches:
refs/heads/trunk f6f3a447b -> 8a4095305
HDFS-10823. Implement HttpFSFileSystem#listStatusIterator.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a409530
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a409530
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a409530
Branch: refs/heads/trunk
Commit: 8a40953058d50d421d62b71067a13b626b3cba1f
Parents: f6f3a44
Author: Andrew Wang
Authored: Fri Sep 16 15:37:36 2016 -0700
Committer: Andrew Wang
Committed: Fri Sep 16 15:37:36 2016 -0700
--
.../java/org/apache/hadoop/fs/FileSystem.java | 125 ---
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 68 --
.../hadoop/fs/http/client/HttpFSFileSystem.java | 56 +++--
.../hadoop/fs/http/client/HttpFSUtils.java | 2 +
.../hadoop/fs/http/server/FSOperations.java | 62 +
.../http/server/HttpFSParametersProvider.java | 20 +++
.../hadoop/fs/http/server/HttpFSServer.java | 17 +++
.../service/hadoop/FileSystemAccessService.java | 4 +-
.../fs/http/client/BaseTestHttpFSWith.java | 62 -
11 files changed, 340 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a409530/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9bde29d..5939f97 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.Tracer;
import org.apache.htrace.core.TraceScope;
+import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import static com.google.common.base.Preconditions.checkArgument;
@@ -1530,7 +1531,68 @@ public abstract class FileSystem extends Configured
implements Closeable {
*/
public abstract FileStatus[] listStatus(Path f) throws
FileNotFoundException,
IOException;
-
+
+ /**
+ * Represents a batch of directory entries when iteratively listing a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * For internal use by FileSystem subclasses that override
+ * {@link FileSystem#listStatusBatch(Path, byte[])} to implement iterative
+ * listing.
+ */
+ @InterfaceAudience.Private
+ public static class DirectoryEntries {
+private final FileStatus[] entries;
+private final byte[] token;
+private final boolean hasMore;
+
+public DirectoryEntries(FileStatus[] entries, byte[] token, boolean
+hasMore) {
+ this.entries = entries;
+ if (token != null) {
+this.token = token.clone();
+ } else {
+this.token = null;
+ }
+ this.hasMore = hasMore;
+}
+
+public FileStatus[] getEntries() {
+ return entries;
+}
+
+public byte[] getToken() {
+ return token;
+}
+
+public boolean hasMore() {
+ return hasMore;
+}
+ }
+
+ /**
+ * Given an opaque iteration token, return the next batch of entries in a
+ * directory. This is a private API not meant for use by end users.
+ *
+ * This method should be overridden by FileSystem subclasses that want to
+ * use the generic {@link FileSystem#listStatusIterator(Path)}
implementation.
+ * @param f Path to list
+ * @param token opaque iteration token returned by previous call, or null
+ * if this is the first call.
+ * @return
+ * @throws FileNotFoundException
+ * @throws IOException
+ */
+ @InterfaceAudience.Private
+ protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
+ FileNotFoundException, IOException {
+// The default implementation returns the entire listing as a single batch.
+// Thus, there is never a second batch, and no need to respect the passed
+// token or set a token in the returned DirectoryEntries.
+FileStatus[] listing = listStatus(f);
+return new DirectoryEntries(listing, null, false);
+ }
+
/*
* Filter files/directories in the given path using the user-supplied path
* filter. Results are added to the given array results.
@@ -1767,6 +1829,49 @@ public abstract class FileSystem extends Configured
implements Closeable {
}
/**
