steveloughran commented on a change in pull request #3534:
URL: https://github.com/apache/hadoop/pull/3534#discussion_r765877448
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
##########
@@ -279,82 +205,19 @@ private ObjectListingIterator createObjectListingIterator(
LOG.debug("Requesting all entries under {} with delimiter '{}'",
key, delimiter);
}
- final RemoteIterator<S3AFileStatus> cachedFilesIterator;
- final Set<Path> tombstones;
- boolean allowAuthoritative = listingOperationCallbacks
- .allowAuthoritative(path);
- if (recursive) {
- final PathMetadata pm = getStoreContext()
- .getMetadataStore()
- .get(path, true);
- if (pm != null) {
- if (pm.isDeleted()) {
- OffsetDateTime deletedAt = OffsetDateTime
- .ofInstant(Instant.ofEpochMilli(
- pm.getFileStatus().getModificationTime()),
- ZoneOffset.UTC);
- throw new FileNotFoundException("Path " + path + " is recorded as " +
- "deleted by S3Guard at " + deletedAt);
- }
- }
- MetadataStoreListFilesIterator metadataStoreListFilesIterator =
- new MetadataStoreListFilesIterator(
- getStoreContext().getMetadataStore(),
- pm,
- allowAuthoritative);
- tombstones = metadataStoreListFilesIterator.listTombstones();
- // if all of the below is true
- // - authoritative access is allowed for this metadatastore
- // for this directory,
- // - all the directory listings are authoritative on the client
- // - the caller does not force non-authoritative access
- // return the listing without any further s3 access
- if (!forceNonAuthoritativeMS &&
- allowAuthoritative &&
- metadataStoreListFilesIterator.isRecursivelyAuthoritative()) {
- S3AFileStatus[] statuses = S3AUtils.iteratorToStatuses(
- metadataStoreListFilesIterator, tombstones);
- cachedFilesIterator = createProvidedFileStatusIterator(
- statuses, ACCEPT_ALL, acceptor);
- return createLocatedFileStatusIterator(cachedFilesIterator);
- }
- cachedFilesIterator = metadataStoreListFilesIterator;
- } else {
- DirListingMetadata meta =
- S3Guard.listChildrenWithTtl(
- getStoreContext().getMetadataStore(),
- path,
- listingOperationCallbacks.getUpdatedTtlTimeProvider(),
- allowAuthoritative);
- if (meta != null) {
- tombstones = meta.listTombstones();
- } else {
- tombstones = null;
- }
- cachedFilesIterator = createProvidedFileStatusIterator(
- S3Guard.dirMetaToStatuses(meta), ACCEPT_ALL, acceptor);
- if (allowAuthoritative && meta != null && meta.isAuthoritative()) {
- // metadata listing is authoritative, so return it directly
- return createLocatedFileStatusIterator(cachedFilesIterator);
- }
- }
- return createTombstoneReconcilingIterator(
- createLocatedFileStatusIterator(
- createFileStatusListingIterator(path,
- listingOperationCallbacks
- .createListObjectsRequest(key,
- delimiter,
- span),
- ACCEPT_ALL,
- acceptor,
- cachedFilesIterator,
- span)),
- collectTombstones ? tombstones : null);
+ return createLocatedFileStatusIterator(
Review comment:
😀
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
