This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 96c43b3354b HDDS-16362. GetObjectAttributes ObjectParts should return
Part entries for FSO buckets (#11242).
96c43b3354b is described below
commit 96c43b3354bad6659eb1f7d0c9a35cdc65f3dd41
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Tue Oct 6 10:04:36 2026 +0530
HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for
FSO buckets (#11242).
---
.../ozone/client/S3HeadObjectAttributes.java | 10 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 16 ++-
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 98 +++++++++++---
.../src/main/proto/OmClientProtocol.proto | 1 +
.../apache/hadoop/ozone/om/OmMetadataReader.java | 6 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 1 +
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 38 ++++--
.../hadoop/ozone/client/ClientProtocolStub.java | 2 +-
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 146 ++++++++++++++++++++-
10 files changed, 304 insertions(+), 39 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java
index 91a5ae70193..fd2a8e214ea 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java
@@ -20,6 +20,7 @@
import java.util.Collections;
import java.util.NavigableMap;
import java.util.Objects;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
/**
* Head metadata and completed multipart part sizes from a single S3 {@code
GetKeyInfo} call.
@@ -28,13 +29,16 @@ public final class S3HeadObjectAttributes {
private final OzoneKey key;
private final NavigableMap<Integer, Long> completedMultipartPartSizes;
+ private final BucketLayout bucketLayout;
public S3HeadObjectAttributes(OzoneKey key,
- NavigableMap<Integer, Long> completedMultipartPartSizes) {
+ NavigableMap<Integer, Long> completedMultipartPartSizes,
+ BucketLayout bucketLayout) {
this.key = Objects.requireNonNull(key, "key == null");
this.completedMultipartPartSizes = completedMultipartPartSizes == null
? Collections.emptyNavigableMap()
: completedMultipartPartSizes;
+ this.bucketLayout = bucketLayout == null ? BucketLayout.DEFAULT :
bucketLayout;
}
public OzoneKey getKey() {
@@ -44,4 +48,8 @@ public OzoneKey getKey() {
public NavigableMap<Integer, Long> getCompletedMultipartPartSizes() {
return completedMultipartPartSizes;
}
+
+ public BucketLayout getBucketLayout() {
+ return bucketLayout;
+ }
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 899cf921e4c..522fb17e1fb 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1955,7 +1955,9 @@ public OzoneKey headS3Object(String bucketName, String
keyName,
@Override
public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName,
String keyName)
throws IOException {
- OmKeyInfo keyInfo = getS3KeyInfo(bucketName, keyName, true);
+ KeyInfoWithVolumeContext keyInfoWithS3Context =
+ getS3KeyInfoWithVolumeContext(bucketName, keyName, true);
+ OmKeyInfo keyInfo = keyInfoWithS3Context.getKeyInfo();
OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations();
NavigableMap<Integer, Long> partSizes = Collections.emptyNavigableMap();
if (locationGroup != null && locationGroup.isMultipartKey()) {
@@ -1967,7 +1969,9 @@ public S3HeadObjectAttributes
headS3ObjectAttributes(String bucketName, String k
}
}
}
- return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo),
partSizes);
+ BucketLayout bucketLayout = keyInfoWithS3Context.getBucketLayout()
+ .orElse(BucketLayout.DEFAULT);
+ return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo),
partSizes, bucketLayout);
}
private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName,
@@ -1993,6 +1997,12 @@ private OmKeyInfo getS3PartOmKeyInfo(String bucketName,
String keyName,
@Nonnull
private OmKeyInfo getS3KeyInfo(
String bucketName, String keyName, boolean isHeadOp) throws IOException {
+ return getS3KeyInfoWithVolumeContext(bucketName, keyName,
isHeadOp).getKeyInfo();
+ }
+
+ @Nonnull
+ private KeyInfoWithVolumeContext getS3KeyInfoWithVolumeContext(
+ String bucketName, String keyName, boolean isHeadOp) throws IOException {
verifyBucketName(bucketName);
Objects.requireNonNull(keyName, "keyName == null");
@@ -2010,7 +2020,7 @@ private OmKeyInfo getS3KeyInfo(
KeyInfoWithVolumeContext keyInfoWithS3Context =
ozoneManagerClient.getKeyInfo(keyArgs, true);
keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal);
- return keyInfoWithS3Context.getKeyInfo();
+ return keyInfoWithS3Context;
}
@Nonnull
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
index d6d54d3c174..bc2a70942ac 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
@@ -37,12 +37,22 @@ public class KeyInfoWithVolumeContext {
private final OmKeyInfo keyInfo;
+ private final Optional<BucketLayout> bucketLayout;
+
public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs,
String userPrincipal,
OmKeyInfo keyInfo) {
+ this(volumeArgs, userPrincipal, keyInfo, null);
+ }
+
+ public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs,
+ String userPrincipal,
+ OmKeyInfo keyInfo,
+ BucketLayout bucketLayout) {
this.volumeArgs = Optional.ofNullable(volumeArgs);
this.userPrincipal = Optional.ofNullable(userPrincipal);
this.keyInfo = keyInfo;
+ this.bucketLayout = Optional.ofNullable(bucketLayout);
}
public static KeyInfoWithVolumeContext fromProtobuf(
@@ -52,6 +62,8 @@ public static KeyInfoWithVolumeContext fromProtobuf(
OmVolumeArgs.getFromProtobuf(proto.getVolumeInfo()) : null)
.setUserPrincipal(proto.getUserPrincipal())
.setKeyInfo(OmKeyInfo.getFromProtobuf(proto.getKeyInfo()))
+ .setBucketLayout(proto.hasBucketLayout()
+ ? BucketLayout.fromProto(proto.getBucketLayout()) : null)
.build();
}
@@ -60,6 +72,7 @@ public GetKeyInfoResponse toProtobuf(int clientVersion) {
volumeArgs.ifPresent(v -> builder.setVolumeInfo(v.getProtobuf()));
userPrincipal.ifPresent(builder::setUserPrincipal);
builder.setKeyInfo(keyInfo.getProtobuf(clientVersion));
+ bucketLayout.ifPresent(layout ->
builder.setBucketLayout(layout.toProto()));
return builder.build();
}
@@ -75,6 +88,10 @@ public Optional<String> getUserPrincipal() {
return userPrincipal;
}
+ public Optional<BucketLayout> getBucketLayout() {
+ return bucketLayout;
+ }
+
public static Builder newBuilder() {
return new Builder();
}
@@ -86,6 +103,7 @@ public static class Builder {
private OmVolumeArgs volumeArgs;
private String userPrincipal;
private OmKeyInfo keyInfo;
+ private BucketLayout bucketLayout;
public Builder setVolumeArgs(OmVolumeArgs volumeArgs) {
this.volumeArgs = volumeArgs;
@@ -102,8 +120,13 @@ public Builder setKeyInfo(OmKeyInfo keyInfo) {
return this;
}
+ public Builder setBucketLayout(BucketLayout layout) {
+ this.bucketLayout = layout;
+ return this;
+ }
+
public KeyInfoWithVolumeContext build() {
- return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo);
+ return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo,
bucketLayout);
}
}
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index bb3efb1d304..6e88f707672 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -2243,6 +2243,72 @@ public void
testGetObjectAttributesMultipartObjectParts(@TempDir Path tempDir) t
assertTrue(secondPage.objectParts().parts().isEmpty());
}
+ /**
+ * Directory (FSO layout) buckets return per-part {@code Part} elements even
when no additional
+ * checksum was stored at upload time, matching AWS S3 directory-bucket
behavior.
+ */
+ @Test
+ public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir)
+ throws Exception {
+ final String bucketName = uniqueObjectName();
+ final String keyName = getKeyName();
+ final int partSize = (int) (5 * MB);
+
+ createFsoBucket(bucketName);
+ try {
+ File multipartUploadFile =
+
Files.createFile(tempDir.resolve("fso-get-object-attributes-mpu.txt")).toFile();
+ createFile(multipartUploadFile, (int) (15 * MB));
+ multipartUpload(bucketName, keyName, multipartUploadFile, partSize, new
HashMap<>(),
+ Collections.emptyList());
+
+ GetObjectAttributesResponse attributesResponse =
s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.OBJECT_PARTS,
ObjectAttributes.OBJECT_SIZE)
+ .build());
+
+ assertNotNull(attributesResponse.objectParts());
+ assertEquals(3, attributesResponse.objectParts().totalPartsCount());
+ assertFalse(attributesResponse.objectParts().isTruncated());
+ assertEquals(multipartUploadFile.length(),
attributesResponse.objectSize());
+ assertEquals(3, attributesResponse.objectParts().parts().size());
+ for (int i = 0; i < 3; i++) {
+ assertEquals(i + 1,
attributesResponse.objectParts().parts().get(i).partNumber());
+ assertEquals((long) partSize,
attributesResponse.objectParts().parts().get(i).size());
+ }
+
+ GetObjectAttributesResponse firstPage = s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+ .maxParts(2)
+ .partNumberMarker(0)
+ .build());
+ assertTrue(firstPage.objectParts().isTruncated());
+ assertEquals(2, firstPage.objectParts().parts().size());
+ assertEquals(1, firstPage.objectParts().parts().get(0).partNumber());
+ assertEquals(2, firstPage.objectParts().parts().get(1).partNumber());
+
+ GetObjectAttributesResponse secondPage = s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+ .maxParts(2)
+ .partNumberMarker(2)
+ .build());
+ assertFalse(secondPage.objectParts().isTruncated());
+ assertEquals(1, secondPage.objectParts().parts().size());
+ assertEquals(3, secondPage.objectParts().parts().get(0).partNumber());
+ } finally {
+ s3Client.deleteObject(b -> b.bucket(bucketName).key(keyName));
+ deleteFsoBucket(bucketName);
+ }
+ }
+
@Test
public void testGetObjectAttributesNonContiguousMultipartObjectParts()
throws Exception {
final String bucketName = getBucketName();
@@ -3443,6 +3509,22 @@ private static void assertNotImplemented(Executable
request) {
assertEquals(S3ErrorTable.NOT_IMPLEMENTED.getErrorMessage(),
exception.awsErrorDetails().errorMessage());
}
+ private void createFsoBucket(String bucketName) throws Exception {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+ volume.createBucket(bucketName, BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .build());
+ }
+ }
+
+ private void deleteFsoBucket(String bucketName) throws Exception {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+ volume.deleteBucket(bucketName);
+ }
+ }
+
private String multipartUpload(String bucketName, String key, File file, int
partSize,
Map<String, String> userMetadata, List<Tag>
tags) throws Exception {
String uploadId = initiateMultipartUpload(bucketName, key, userMetadata,
tags);
@@ -4591,21 +4673,5 @@ public void testListBucketsIncludesFSOBuckets() throws
Exception {
deleteFsoBucket(fsoBucketName);
}
}
-
- private void createFsoBucket(String bucketName) throws Exception {
- try (OzoneClient ozoneClient = cluster.newClient()) {
- OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
- volume.createBucket(bucketName, BucketArgs.newBuilder()
- .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
- .build());
- }
- }
-
- private void deleteFsoBucket(String bucketName) throws Exception {
- try (OzoneClient ozoneClient = cluster.newClient()) {
- OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
- volume.deleteBucket(bucketName);
- }
- }
}
}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index ddc4cd80aab..43f7c6aae51 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1419,6 +1419,7 @@ message GetKeyInfoResponse {
optional KeyInfo keyInfo = 1;
optional VolumeInfo volumeInfo = 2;
optional string UserPrincipal = 3;
+ optional BucketLayoutProto bucketLayout = 4;
}
message RenameKeysRequest {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index d6db1cb35d8..c3121599566 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -31,6 +31,7 @@
import java.net.InetAddress;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -46,6 +47,7 @@
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
@@ -199,7 +201,9 @@ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs
args,
OmMetadataReader.getClientAddress());
KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext
.newBuilder()
- .setKeyInfo(keyInfo);
+ .setKeyInfo(keyInfo)
+ .setBucketLayout(Objects.requireNonNullElse(
+ bucket.bucketLayout(), BucketLayout.DEFAULT));
s3VolumeContext.ifPresent(context -> {
builder.setVolumeArgs(context.getOmVolumeArgs());
builder.setUserPrincipal(context.getUserPrincipal());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
index 3e59092c74f..acba22ce721 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
@@ -323,6 +323,7 @@ private KeyInfoWithVolumeContext
denormalizeKeyInfoWithVolumeContext(
.setKeyInfo(denormalizeOmKeyInfo(k.getKeyInfo()))
.setVolumeArgs(k.getVolumeArgs().orElse(null))
.setUserPrincipal(k.getUserPrincipal().orElse(null))
+ .setBucketLayout(k.getBucketLayout().orElse(null))
.build();
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
index 171fabde56b..45755f22de4 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
@@ -57,10 +57,10 @@
* <p>Returns selected metadata about an object without transferring the
object body.
* Supported attributes: {@code ETag}, {@code ObjectSize}, {@code
StorageClass}, {@code ObjectParts}.
*
- * <p>The {@code Checksum} attribute is not yet supported because Ozone does
not store
- * non-MD5 checksum algorithms in key metadata. For general-purpose buckets,
{@code Part}
- * elements under {@code ObjectParts} are omitted unless an additional
checksum is stored
- * on the object, matching AWS S3 behavior. Object versioning ({@code
versionId}) and
+ * <p>For general-purpose buckets, individual {@code Part} elements under
{@code ObjectParts}
+ * are omitted unless an additional checksum is stored on the object, matching
AWS S3 behavior.
+ * For FSO layout buckets (Ozone directory buckets), {@code Part} elements are
always returned,
+ * matching AWS S3 directory-bucket behavior. Object versioning ({@code
versionId}) and
* SSE-C encryption headers are also not supported and are silently ignored.
*
* <p>See
https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html
@@ -101,6 +101,15 @@ static boolean hasStoredAdditionalChecksum(OzoneKey key) {
return false;
}
+ /**
+ * Whether {@code ObjectParts} should include per-part {@code Part} elements
in the response.
+ * Directory (FSO layout) buckets always include them; general-purpose
buckets include them
+ * only when an additional checksum was stored at upload time.
+ */
+ static boolean shouldIncludePartElements(OzoneKey key, boolean
directoryBucketLayout) {
+ return directoryBucketLayout || hasStoredAdditionalChecksum(key);
+ }
+
@Override
S3GAction getAction() {
return queryParams().get(QueryParams.ATTRIBUTES) == null ? null :
S3GAction.NOT_IMPLEMENTED;
@@ -123,12 +132,15 @@ Response handleGetRequest(ObjectRequestContext context,
String keyPath)
OzoneKey key;
NavigableMap<Integer, Long> completedPartSizes = null;
+ boolean directoryBucketLayout = false;
try {
if (requestedAttributes.contains(ATTR_OBJECT_PARTS)) {
S3HeadObjectAttributes headAttributes =
getClientProtocol().headS3ObjectAttributes(bucketName, keyPath);
key = headAttributes.getKey();
completedPartSizes = headAttributes.getCompletedMultipartPartSizes();
+ directoryBucketLayout =
+ headAttributes.getBucketLayout().isFileSystemOptimized();
} else {
key = getClientProtocol().headS3Object(bucketName, keyPath);
}
@@ -143,7 +155,8 @@ Response handleGetRequest(ObjectRequestContext context,
String keyPath)
}
GetObjectAttributesResponse response =
- buildResponse(keyPath, key, requestedAttributes, completedPartSizes);
+ buildResponse(keyPath, key, requestedAttributes, completedPartSizes,
+ directoryBucketLayout);
Response.ResponseBuilder rb = Response.ok(response,
MediaType.APPLICATION_XML_TYPE);
ObjectEndpoint.addLastModifiedDate(rb, key);
@@ -182,7 +195,8 @@ private Set<String> parseAttributesHeader(String keyPath)
throws OS3Exception {
}
private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey
key,
- Set<String> requested, NavigableMap<Integer, Long> completedPartSizes)
+ Set<String> requested, NavigableMap<Integer, Long> completedPartSizes,
+ boolean directoryBucketLayout)
throws IOException, OS3Exception {
GetObjectAttributesResponse resp = new GetObjectAttributesResponse();
@@ -210,7 +224,7 @@ private GetObjectAttributesResponse buildResponse(String
keyPath, OzoneKey key,
String partsCountStr = extractPartsCount(eTag);
if (partsCountStr != null && completedPartSizes != null) {
resp.setObjectParts(buildObjectParts(keyPath,
Integer.parseInt(partsCountStr),
- completedPartSizes, key));
+ completedPartSizes, key, directoryBucketLayout));
}
}
}
@@ -233,10 +247,12 @@ private GetObjectAttributesResponse buildResponse(String
keyPath, OzoneKey key,
*
* <p>For general-purpose buckets, individual {@code Part} elements are
returned only when the
* object has a stored AWS additional checksum in key metadata; otherwise
only {@code ObjectParts}
- * summary and pagination fields are returned.
+ * summary and pagination fields are returned. For FSO layout (directory)
buckets, {@code Part}
+ * elements are always returned.
*/
private GetObjectAttributesResponse.ObjectParts buildObjectParts(String
keyPath,
- int totalPartsCount, NavigableMap<Integer, Long> partSizes, OzoneKey key)
+ int totalPartsCount, NavigableMap<Integer, Long> partSizes, OzoneKey key,
+ boolean directoryBucketLayout)
throws OS3Exception {
int maxParts = parseMaxPartsHeader(keyPath);
int marker = parsePartNumberMarkerHeader(keyPath);
@@ -257,9 +273,7 @@ private GetObjectAttributesResponse.ObjectParts
buildObjectParts(String keyPath,
Iterator<Map.Entry<Integer, Long>> partIterator =
partSizes.tailMap(marker, false).entrySet().iterator();
- // TODO: For FSO (directory) buckets, always include Part entries per AWS
- // directory-bucket GetObjectAttributes behavior, regardless of checksum
metadata.
- boolean includePartEntries = hasStoredAdditionalChecksum(key);
+ boolean includePartEntries = shouldIncludePartElements(key,
directoryBucketLayout);
Integer lastPartReturned = null;
int partsOnPage = 0;
while (partIterator.hasNext() && partsOnPage < maxParts) {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 6e57ae2f604..c8c326eb736 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -149,7 +149,7 @@ public S3HeadObjectAttributes headS3ObjectAttributes(String
bucketName, String k
NavigableMap<Integer, Long> partSizes = bucket instanceof OzoneBucketStub
? ((OzoneBucketStub) bucket).getCompletedMultipartPartSizes(keyName)
: Collections.emptyNavigableMap();
- return new S3HeadObjectAttributes(key, partSizes);
+ return new S3HeadObjectAttributes(key, partSizes,
bucket.getBucketLayout());
}
@Override
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
index 6c39ac89e1a..343924f29ff 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
@@ -35,17 +35,38 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneBucketStub;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.S3HeadObjectAttributes;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.util.S3Consts;
@@ -60,6 +81,7 @@ public class TestObjectAttributesGet {
private static final String CONTENT = "0123456789";
private static final String BUCKET_NAME = "b1";
+ private static final String FSO_BUCKET_NAME = "fso-b1";
private static final String KEY_NAME = "key1";
private ObjectEndpoint rest;
private OzoneBucket bucket;
@@ -299,13 +321,129 @@ public void
testGetObjectAttributesNonContiguousMultipartParts() throws IOExcept
assertEquals(partThreeContent.length(),
paginatedParts.getParts().get(0).getSize());
}
- private void completeMultipartUploadWithParts(String key, String...
partContents)
+ @Test
+ public void testGetObjectAttrFsoMPUPartsWithoutChecksum()
throws IOException, OS3Exception {
- String uploadID = initiateMultipartUpload(rest, BUCKET_NAME, key);
+ OzoneClient client = rest.getClient();
+ String volumeName =
rest.getOzoneConfiguration().get(OzoneConfigKeys.OZONE_S3_VOLUME_NAME,
+ OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT);
+ OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
+ volume.createBucket(FSO_BUCKET_NAME, BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .build());
+
+ final String key = "fso-mpu-key";
+ completeMultipartUploadWithPartsInBucket(FSO_BUCKET_NAME, key, "part-one",
"part-two");
+
+ Response response = getObjectAttributes(rest, FSO_BUCKET_NAME, key,
"ObjectParts");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ GetObjectAttributesResponse.ObjectParts objectParts =
+ ((GetObjectAttributesResponse) response.getEntity()).getObjectParts();
+ assertNotNull(objectParts);
+ assertEquals(2, objectParts.getPartsCount().intValue());
+ assertEquals(2, objectParts.getParts().size());
+ assertEquals(1, objectParts.getParts().get(0).getPartNumber());
+ assertEquals("part-one".length(), objectParts.getParts().get(0).getSize());
+ assertEquals(2, objectParts.getParts().get(1).getPartNumber());
+ assertEquals("part-two".length(), objectParts.getParts().get(1).getSize());
+ }
+
+ /**
+ * Regression: multipart {@code ObjectParts} must not call {@code
context.getBucket()}
+ * (bucket READ). Layout and part sizes come from {@code
headS3ObjectAttributes} only.
+ */
+ @Test
+ public void testGetObjectAttributesObjectPartsWithoutBucketRead()
+ throws IOException, OS3Exception {
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration.set(OzoneConfigKeys.OZONE_S3_VOLUME_NAME,
+ OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT);
+
+ ClientProtocol clientProtocol = mock(ClientProtocol.class);
+ OzoneClient client = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+
+ when(client.getConfiguration()).thenReturn(ozoneConfiguration);
+ when(client.getObjectStore()).thenReturn(objectStore);
+ when(client.getProxy()).thenReturn(clientProtocol);
+ when(objectStore.getClientProxy()).thenReturn(clientProtocol);
+ when(objectStore.getS3Volume()).thenReturn(volume);
+ when(volume.getBucket(anyString())).thenThrow(
+ new OMException("bucket read denied", ResultCodes.ACCESS_DENIED));
+
+ final String mpuKey = "mpu-key";
+ OzoneKey ozoneKey = new OzoneKey("s3v", BUCKET_NAME, mpuKey, 10, 0, 0,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ true, "owner");
+ ozoneKey.setMetadata(Collections.singletonMap(OzoneConsts.ETAG,
"\"abc-2\""));
+
+ NavigableMap<Integer, Long> partSizes = new TreeMap<>();
+ partSizes.put(1, 5L);
+ partSizes.put(2, 5L);
+ S3HeadObjectAttributes headAttributes = new
S3HeadObjectAttributes(ozoneKey, partSizes,
+ BucketLayout.OBJECT_STORE);
+ when(clientProtocol.headS3ObjectAttributes(BUCKET_NAME,
mpuKey)).thenReturn(headAttributes);
+
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD");
+
+ ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+
+ Response response = getObjectAttributes(endpoint, BUCKET_NAME, mpuKey,
"ObjectParts");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ GetObjectAttributesResponse.ObjectParts objectParts =
+ ((GetObjectAttributesResponse) response.getEntity()).getObjectParts();
+ assertNotNull(objectParts);
+ assertEquals(2, objectParts.getPartsCount().intValue());
+ assertTrue(objectParts.getParts().isEmpty());
+
+ verify(volume, never()).getBucket(anyString());
+ verify(clientProtocol).headS3ObjectAttributes(BUCKET_NAME, mpuKey);
+ }
+
+ @Test
+ public void testShouldIncludePartElements() {
+ assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null,
false));
+ assertTrue(ObjectAttributesHandler.shouldIncludePartElements(null, true));
+
+ OzoneKey obsKeyNoChecksum = testOzoneKey(false);
+
assertFalse(ObjectAttributesHandler.shouldIncludePartElements(obsKeyNoChecksum,
false));
+
+ OzoneKey obsKeyWithChecksum = testOzoneKey(true);
+
assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyWithChecksum,
false));
+
+
assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyNoChecksum,
true));
+
assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyWithChecksum,
true));
+ }
+
+ private static OzoneKey testOzoneKey(boolean withAdditionalChecksum) {
+ OzoneKey key = new OzoneKey("vol", "b", "k", 0, 0, 0,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ true, "owner");
+ if (withAdditionalChecksum) {
+ key.setMetadata(Collections.singletonMap("x-amz-checksum-crc32", "abc"));
+ }
+ return key;
+ }
+
+ private void completeMultipartUploadWithPartsInBucket(String bucketName,
String key,
+ String... partContents) throws IOException, OS3Exception {
+ String uploadID = initiateMultipartUpload(rest, bucketName, key);
List<Part> partsList = new ArrayList<>();
for (int i = 0; i < partContents.length; i++) {
- partsList.add(uploadPart(rest, BUCKET_NAME, key, i + 1, uploadID,
partContents[i]));
+ partsList.add(uploadPart(rest, bucketName, key, i + 1, uploadID,
partContents[i]));
}
- completeMultipartUpload(rest, BUCKET_NAME, key, uploadID, partsList);
+ completeMultipartUpload(rest, bucketName, key, uploadID, partsList);
+ }
+
+ private void completeMultipartUploadWithParts(String key, String...
partContents)
+ throws IOException, OS3Exception {
+ completeMultipartUploadWithPartsInBucket(BUCKET_NAME, key, partContents);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
