This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5fee950 HADOOP-17323. S3A getFileStatus("/") to skip IO (#2479)
5fee950 is described below
commit 5fee95076b578e0e5d4fc0c8c92af8221e2d0efa
Author: Mukund Thakur <[email protected]>
AuthorDate: Tue Nov 24 16:36:56 2020 +0530
HADOOP-17323. S3A getFileStatus("/") to skip IO (#2479)
Contributed by Mukund Thakur.
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +++
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 31 ++++++++++++++++++++++
.../hadoop/fs/s3a/performance/OperationCost.java | 5 ++++
3 files changed, 40 insertions(+)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 945a0ad..f2cd2cd 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -3144,6 +3144,10 @@ public class S3AFileSystem extends FileSystem implements
StreamCapabilities,
"s3GetFileStatus(%s) wants to know if a directory is empty but"
+ " does not request a list probe", path);
+ if (key.isEmpty() && !needEmptyDirectoryFlag) {
+ return new S3AFileStatus(Tristate.UNKNOWN, path, username);
+ }
+
if (!key.isEmpty() && !key.endsWith("/")
&& probes.contains(StatusProbeEnum.Head)) {
try {
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
index 6461ecd..ca8e49c 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -29,6 +29,7 @@ import
org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.assertj.core.api.Assertions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -259,6 +260,36 @@ public class ITestS3AFileOperationCost extends
AbstractS3ACostTest {
}
@Test
+ public void testCostOfRootFileStatus() throws Throwable {
+ Path root = path("/");
+ S3AFileStatus rootStatus = verifyRawInnerGetFileStatus(
+ root,
+ false,
+ StatusProbeEnum.ALL,
+ ROOT_FILE_STATUS_PROBE);
+ String rootStatusContent = rootStatus.toString();
+ Assertions.assertThat(rootStatus.isDirectory())
+ .describedAs("Status returned should be a directory "
+ + rootStatusContent)
+ .isEqualTo(true);
+ Assertions.assertThat(rootStatus.isEmptyDirectory())
+ .isEqualTo(Tristate.UNKNOWN);
+
+ rootStatus = verifyRawInnerGetFileStatus(
+ root,
+ true,
+ StatusProbeEnum.ALL,
+ FILE_STATUS_DIR_PROBE);
+ Assertions.assertThat(rootStatus.isDirectory())
+ .describedAs("Status returned should be a directory "
+ + rootStatusContent)
+ .isEqualTo(true);
+ Assertions.assertThat(rootStatus.isEmptyDirectory())
+ .isNotEqualByComparingTo(Tristate.UNKNOWN);
+
+ }
+
+ @Test
public void testIsDirIsFileMissingPath() throws Throwable {
describe("performing isDir and isFile on a missing file");
Path path = methodPath();
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
index 0a1438d..8605d7f 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
@@ -77,6 +77,11 @@ public final class OperationCost {
public static final OperationCost FILE_STATUS_FILE_PROBE = HEAD_OPERATION;
/**
+ * Cost of getFileStatus on root directory.
+ */
+ public static final OperationCost ROOT_FILE_STATUS_PROBE = NO_IO;
+
+ /**
* Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#ALL}.
*/
public static final OperationCost FILE_STATUS_ALL_PROBES =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
