This is an automated email from the ASF dual-hosted git repository.
gabota pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new de6b7bc HADOOP-16409. Allow authoritative mode on non-qualified
paths. Contributed by Sean Mackrory
de6b7bc is described below
commit de6b7bc67ace7744adb0320ee7de79cf28259d2d
Author: Sean Mackrory <mackror...@gmail.com>
AuthorDate: Mon Jul 8 11:27:07 2019 -0600
HADOOP-16409. Allow authoritative mode on non-qualified paths. Contributed
by Sean Mackrory
---
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 --
.../java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java | 1 +
.../src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java | 3 +--
.../test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java | 4 ++++
4 files changed, 6 insertions(+), 4 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 9b3a4b1..3ae4f1f 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -2421,7 +2421,6 @@ public class S3AFileSystem extends FileSystem implements
StreamCapabilities,
result.add(files.next());
}
// merge the results. This will update the store as needed
-
return S3Guard.dirListingUnion(metadataStore, path, result, dirMeta,
allowAuthoritative, ttlTimeProvider);
} else {
@@ -3810,7 +3809,6 @@ public class S3AFileSystem extends FileSystem implements
StreamCapabilities,
final PathMetadata pm = metadataStore.get(path, true);
// shouldn't need to check pm.isDeleted() because that will have
// been caught by getFileStatus above.
-
MetadataStoreListFilesIterator metadataStoreListFilesIterator =
new MetadataStoreListFilesIterator(metadataStore, pm,
allowAuthoritative);
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 3329b54..9f06313 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -1321,6 +1321,7 @@ public class DynamoDBMetadataStore implements
MetadataStore,
final DirListingMetadata meta,
@Nullable final BulkOperationState operationState) throws IOException {
LOG.debug("Saving to table {} in region {}: {}", tableName, region, meta);
+
// directory path
Path path = meta.getPath();
DDBPathMetadata ddbPathMeta =
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index 97070f8..85e4d12 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -787,13 +787,12 @@ public final class S3Guard {
public static boolean allowAuthoritative(Path p, S3AFileSystem fs,
boolean authMetadataStore, Collection<String> authPaths) {
- String haystack = fs.maybeAddTrailingSlash(p.toString());
+ String haystack = fs.maybeAddTrailingSlash(fs.qualify(p).toString());
if (authMetadataStore) {
return true;
}
if (!authPaths.isEmpty()) {
for (String needle : authPaths) {
-
if (haystack.startsWith(needle)) {
return true;
}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
index b7c5823..c35a585 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
@@ -294,6 +294,10 @@ public class ITestAuthoritativePath extends
AbstractS3ATestBase {
Path directoryMatch = new Path(testRoot, "/auth/oritative");
assertTrue(S3Guard.allowAuthoritative(directoryMatch, fs,
false, authPaths));
+
+ Path unqualifiedMatch = new Path(testRoot.toUri().getPath(),
"/auth/oritative");
+ assertTrue(S3Guard.allowAuthoritative(unqualifiedMatch, fs,
+ false, authPaths));
} finally {
cleanUpFS(fs);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
