This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1db063b63ce HDDS-16072. Implement GetObjectAttributes: ETag,
ObjectSize, StorageClass, and ObjectParts count (#10930).
1db063b63ce is described below
commit 1db063b63ceed0d2946970752d3d6eea529db1aa
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Wed Aug 12 11:25:30 2026 +0530
HDDS-16072. Implement GetObjectAttributes: ETag, ObjectSize, StorageClass,
and ObjectParts count (#10930).
---
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 90 +++++++++
.../org/apache/hadoop/ozone/audit/S3GAction.java | 3 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 19 ++
.../s3/endpoint/GetObjectAttributesResponse.java | 206 +++++++++++++++++++++
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 173 +++++++++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 21 +--
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 22 +++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 +
.../ozone/s3/endpoint/EndpointTestUtils.java | 14 ++
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 176 ++++++++++++++++++
10 files changed, 709 insertions(+), 19 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index ac54d20aa49..ec80b9c4de0 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -137,6 +137,8 @@
import
software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse;
import software.amazon.awssdk.services.s3.model.GetBucketTaggingRequest;
import software.amazon.awssdk.services.s3.model.GetBucketTaggingResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectAttributesRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectAttributesResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
@@ -160,6 +162,7 @@
import software.amazon.awssdk.services.s3.model.ListPartsRequest;
import software.amazon.awssdk.services.s3.model.MetadataDirective;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectAttributes;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutBucketAclRequest;
import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest;
@@ -1712,6 +1715,93 @@ public void testHeadObjectReturnsTaggingCount() {
assertEquals(tags.size(), head.tagCount().intValue());
}
+ @Test
+ public void testGetObjectAttributes() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "get-object-attributes-content";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b ->
b.bucket(bucketName).key(keyName),
+ RequestBody.fromString(content));
+
+ GetObjectAttributesResponse attributesResponse =
s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.E_TAG,
ObjectAttributes.OBJECT_SIZE,
+ ObjectAttributes.STORAGE_CLASS)
+ .build());
+
+ assertNotNull(attributesResponse.lastModified());
+ assertEquals(
+ putObjectResponse.eTag().replace("\"", ""),
+ attributesResponse.eTag());
+ assertEquals((long) content.length(), attributesResponse.objectSize());
+ assertEquals("STANDARD", attributesResponse.storageClassAsString());
+ assertNull(attributesResponse.objectParts());
+ }
+
+ @Test
+ public void testGetObjectAttributesMultipartObjectParts(@TempDir Path
tempDir) throws Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ File multipartUploadFile =
Files.createFile(tempDir.resolve("get-object-attributes-mpu.txt")).toFile();
+ createFile(multipartUploadFile, (int) (15 * MB));
+ multipartUpload(bucketName, keyName, multipartUploadFile, (int) (5 * MB),
new HashMap<>(),
+ Collections.emptyList());
+
+ GetObjectAttributesResponse attributesResponse =
s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.OBJECT_PARTS,
ObjectAttributes.E_TAG,
+ ObjectAttributes.OBJECT_SIZE)
+ .build());
+
+ assertNotNull(attributesResponse.eTag());
+ assertTrue(attributesResponse.eTag().contains("-"));
+ assertEquals(multipartUploadFile.length(),
attributesResponse.objectSize());
+ assertNotNull(attributesResponse.objectParts());
+ assertEquals(3, attributesResponse.objectParts().totalPartsCount());
+ assertFalse(attributesResponse.objectParts().isTruncated());
+ }
+
+ @Test
+ public void testGetObjectAttributesNoSuchKey() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ assertThrows(NoSuchKeyException.class, () -> s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.E_TAG)
+ .build()));
+ }
+
+ @Test
+ public void testGetObjectAttributesChecksumOmitted() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+ s3Client.putObject(b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString("checksum-test"));
+
+ GetObjectAttributesResponse attributesResponse =
s3Client.getObjectAttributes(
+ GetObjectAttributesRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .objectAttributes(ObjectAttributes.CHECKSUM,
ObjectAttributes.E_TAG)
+ .build());
+
+ assertNotNull(attributesResponse.eTag());
+ assertNull(attributesResponse.checksum());
+ }
+
@Test
public void testResumableDownloadWithEtagMismatch() throws Exception {
// Arrange
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
index 11efd4e9064..16f5ceb1e4b 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
@@ -62,7 +62,8 @@ public enum S3GAction implements AuditAction {
GET_BUCKET_TAGGING,
PUT_BUCKET_TAGGING,
DELETE_BUCKET_TAGGING,
- PUT_OBJECT_ACL;
+ PUT_OBJECT_ACL,
+ GET_OBJECT_ATTRIBUTES;
@Override
public String getAction() {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 124fed461ff..854dcf80e27 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -28,6 +28,8 @@
import static org.apache.hadoop.ozone.OzoneConsts.KB;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
@@ -810,6 +812,23 @@ protected static CheckedRunnable<IOException>
validateContentLength(
};
}
+ /**
+ * Rejects FSO directory keys when the request path omits the trailing slash.
+ *
+ * <p>Necessary for directories in buckets with FSO layout. Intended for
apps which use Hadoop S3A
+ * (e.g. Trino through the Hive connector).
+ */
+ protected void validateFileKey(String keyPath, OzoneKey key) throws
OMException {
+ boolean isFsoDirCreationEnabled = getOzoneConfiguration()
+ .getBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED,
+ OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT);
+ if (isFsoDirCreationEnabled &&
+ !key.isFile() &&
+ !keyPath.endsWith("/")) {
+ throw new OMException(ResultCodes.KEY_NOT_FOUND);
+ }
+ }
+
protected static String extractPartsCount(String eTag) {
if (eTag.contains("-")) {
String[] parts = eTag.replace("\"", "").split("-");
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/GetObjectAttributesResponse.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/GetObjectAttributesResponse.java
new file mode 100644
index 00000000000..612ade64a8a
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/GetObjectAttributesResponse.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import java.util.ArrayList;
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+
+/**
+ * XML response for the GetObjectAttributes S3 API.
+ *
+ * <p>See
https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html
+ *
+ * <p>The {@code Checksum} field is intentionally omitted: Ozone does not yet
store
+ * non-MD5 checksum algorithms in key metadata. When checksum storage is
implemented,
+ * this field can be added here without any API contract change.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "GetObjectAttributesResponse", namespace =
S3Consts.S3_XML_NAMESPACE)
+public class GetObjectAttributesResponse {
+
+ @XmlElement(name = "ETag")
+ private String eTag;
+
+ @XmlElement(name = "ObjectSize")
+ private Long objectSize;
+
+ @XmlElement(name = "StorageClass")
+ private String storageClass;
+
+ @XmlElement(name = "ObjectParts")
+ private ObjectParts objectParts;
+
+ public String getETag() {
+ return eTag;
+ }
+
+ public void setETag(String tag) {
+ this.eTag = tag;
+ }
+
+ public Long getObjectSize() {
+ return objectSize;
+ }
+
+ public void setObjectSize(Long objectSize) {
+ this.objectSize = objectSize;
+ }
+
+ public String getStorageClass() {
+ return storageClass;
+ }
+
+ public void setStorageClass(String storageClass) {
+ this.storageClass = storageClass;
+ }
+
+ public ObjectParts getObjectParts() {
+ return objectParts;
+ }
+
+ public void setObjectParts(ObjectParts objectParts) {
+ this.objectParts = objectParts;
+ }
+
+ /**
+ * Represents the ObjectParts element in the GetObjectAttributes response.
+ *
+ * <p>For completed multipart-uploaded objects, {@code partsCount} is
derived from
+ * the composite ETag suffix (e.g. {@code "hash-15"} → 15 parts). Per-part
sizes
+ * are not stored for completed multipart uploads in Ozone and are therefore
omitted
+ * from the part list in this response.
+ * TODO: Will support completed multipart uploads in this ticket: HDDS-16073
+ */
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlRootElement(name = "ObjectParts")
+ public static class ObjectParts {
+
+ @XmlElement(name = "IsTruncated")
+ private boolean truncated;
+
+ @XmlElement(name = "MaxParts")
+ private Integer maxParts;
+
+ @XmlElement(name = "PartNumberMarker")
+ private Integer partNumberMarker;
+
+ @XmlElement(name = "NextPartNumberMarker")
+ private Integer nextPartNumberMarker;
+
+ @XmlElement(name = "PartsCount")
+ private Integer partsCount;
+
+ @XmlElement(name = "Part")
+ private List<Part> parts = new ArrayList<>();
+
+ public boolean isTruncated() {
+ return truncated;
+ }
+
+ public void setTruncated(boolean truncated) {
+ this.truncated = truncated;
+ }
+
+ public Integer getMaxParts() {
+ return maxParts;
+ }
+
+ public void setMaxParts(Integer maxParts) {
+ this.maxParts = maxParts;
+ }
+
+ public Integer getPartNumberMarker() {
+ return partNumberMarker;
+ }
+
+ public void setPartNumberMarker(Integer partNumberMarker) {
+ this.partNumberMarker = partNumberMarker;
+ }
+
+ public Integer getNextPartNumberMarker() {
+ return nextPartNumberMarker;
+ }
+
+ public void setNextPartNumberMarker(Integer nextPartNumberMarker) {
+ this.nextPartNumberMarker = nextPartNumberMarker;
+ }
+
+ public Integer getPartsCount() {
+ return partsCount;
+ }
+
+ public void setPartsCount(Integer partsCount) {
+ this.partsCount = partsCount;
+ }
+
+ public List<Part> getParts() {
+ return parts;
+ }
+
+ public void setParts(List<Part> parts) {
+ this.parts = parts;
+ }
+
+ public void addPart(Part part) {
+ this.parts.add(part);
+ }
+ }
+
+ /**
+ * A single part entry within {@link ObjectParts}.
+ */
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlRootElement(name = "Part")
+ public static class Part {
+
+ @XmlElement(name = "PartNumber")
+ private int partNumber;
+
+ @XmlElement(name = "Size")
+ private long size;
+
+ public Part() {
+ }
+
+ public Part(int partNumber, long size) {
+ this.partNumber = partNumber;
+ this.size = size;
+ }
+
+ public int getPartNumber() {
+ return partNumber;
+ }
+
+ public void setPartNumber(int partNumber) {
+ this.partNumber = partNumber;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
new file mode 100644
index 00000000000..ab4934b3d67
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.OBJECT_ATTRIBUTES_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.audit.S3GAction;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint.ObjectRequestContext;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
+
+/**
+ * Handles the GetObjectAttributes S3 API ({@code GET
/{bucket}/{key}?attributes}).
+ *
+ * <p>Returns selected metadata about an object without transferring the
object body.
+ * Supported attributes: {@code ETag}, {@code ObjectSize}, {@code
StorageClass}, {@code ObjectParts}.
+ *
+ * <p>The {@code Checksum} attribute is not yet supported because Ozone does
not store
+ * non-MD5 checksum algorithms in key metadata. Object versioning ({@code
versionId}) and
+ * SSE-C encryption headers are also not supported and are silently ignored.
+ *
+ * <p>See
https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html
+ */
+class ObjectAttributesHandler extends ObjectOperationHandler {
+
+ /** Valid values for the x-amz-object-attributes request header. */
+ static final String ATTR_ETAG = "ETag";
+ static final String ATTR_CHECKSUM = "Checksum";
+ static final String ATTR_OBJECT_PARTS = "ObjectParts";
+ static final String ATTR_STORAGE_CLASS = "StorageClass";
+ static final String ATTR_OBJECT_SIZE = "ObjectSize";
+
+ private static final Set<String> KNOWN_ATTRIBUTES = new
HashSet<>(Arrays.asList(
+ ATTR_ETAG, ATTR_CHECKSUM, ATTR_OBJECT_PARTS, ATTR_STORAGE_CLASS,
ATTR_OBJECT_SIZE));
+
+ @Override
+ Response handleGetRequest(ObjectRequestContext context, String keyPath)
+ throws IOException, OS3Exception {
+
+ if (queryParams().get(QueryParams.ATTRIBUTES) == null) {
+ return null;
+ }
+
+ context.setAction(S3GAction.GET_OBJECT_ATTRIBUTES);
+
+ final long startNanos = context.getStartNanos();
+ try {
+ Set<String> requestedAttributes = parseAttributesHeader(keyPath);
+ String bucketName = context.getBucketName();
+
+ OzoneKey key;
+ try {
+ key = getClientProtocol().headS3Object(bucketName, keyPath);
+ validateFileKey(keyPath, key);
+ } catch (OMException ex) {
+ if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
+ throw newError(NO_SUCH_KEY, keyPath, ex);
+ } else if (isAccessDenied(ex)) {
+ throw newError(ACCESS_DENIED, bucketName + "/" + keyPath, ex);
+ }
+ throw ex;
+ }
+
+ GetObjectAttributesResponse response = buildResponse(key,
requestedAttributes);
+
+ Response.ResponseBuilder rb = Response.ok(response,
MediaType.APPLICATION_XML_TYPE);
+ ObjectEndpoint.addLastModifiedDate(rb, key);
+ getMetrics().updateGetObjectAttributesSuccessStats(startNanos);
+ return rb.build();
+
+ } catch (OS3Exception | IOException ex) {
+ getMetrics().updateGetObjectAttributesFailureStats(startNanos);
+ throw ex;
+ }
+ }
+
+ /**
+ * Parses and validates the {@code x-amz-object-attributes} request header.
+ *
+ * <p>The header is required. Unknown attribute names that are not in the
S3-defined
+ * set are rejected with {@code InvalidArgument}.
+ */
+ private Set<String> parseAttributesHeader(String keyPath) throws
OS3Exception {
+ String headerValue =
getHeaders().getHeaderString(OBJECT_ATTRIBUTES_HEADER);
+ if (StringUtils.isBlank(headerValue)) {
+ throw newError(INVALID_ARGUMENT, keyPath,
+ new IllegalArgumentException(OBJECT_ATTRIBUTES_HEADER + " is
required"));
+ }
+
+ Set<String> requested = new HashSet<>();
+ for (String token : headerValue.split(",")) {
+ String attr = token.trim();
+ if (!KNOWN_ATTRIBUTES.contains(attr)) {
+ throw newError(INVALID_ARGUMENT, keyPath,
+ new IllegalArgumentException("Invalid value for " +
OBJECT_ATTRIBUTES_HEADER + ": " + attr));
+ }
+ requested.add(attr);
+ }
+ return requested;
+ }
+
+ private GetObjectAttributesResponse buildResponse(OzoneKey key, Set<String>
requested) {
+ GetObjectAttributesResponse resp = new GetObjectAttributesResponse();
+
+ if (requested.contains(ATTR_ETAG)) {
+ String eTag = key.getMetadata().get(OzoneConsts.ETAG);
+ if (eTag != null) {
+ resp.setETag(eTag);
+ }
+ }
+
+ if (requested.contains(ATTR_OBJECT_SIZE)) {
+ resp.setObjectSize(key.getDataSize());
+ }
+
+ if (requested.contains(ATTR_STORAGE_CLASS)) {
+ S3StorageType storageType = key.getReplicationConfig() == null
+ ? S3StorageType.STANDARD
+ : S3StorageType.fromReplicationConfig(key.getReplicationConfig());
+ resp.setStorageClass(storageType.toString());
+ }
+
+ if (requested.contains(ATTR_OBJECT_PARTS)) {
+ String eTag = key.getMetadata().get(OzoneConsts.ETAG);
+ if (eTag != null) {
+ String partsCountStr = extractPartsCount(eTag);
+ if (partsCountStr != null) {
+ GetObjectAttributesResponse.ObjectParts parts = new
GetObjectAttributesResponse.ObjectParts();
+ parts.setPartsCount(Integer.parseInt(partsCountStr));
+ parts.setTruncated(false);
+ resp.setObjectParts(parts);
+ }
+ }
+ }
+
+ // ATTR_CHECKSUM is intentionally not populated: Ozone does not store
non-MD5
+ // checksum algorithms in key metadata. The field is simply omitted from
the
+ // response, which is valid per the S3 spec.
+
+ return resp;
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 6e8260d5e49..21be929e211 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -152,6 +152,7 @@ protected void init() {
.add(new ObjectGetTorrentHandler())
.add(new ObjectAclHandler())
.add(new ObjectTaggingHandler())
+ .add(new ObjectAttributesHandler())
.add(new MultipartKeyHandler())
.add(this)
.build();
@@ -396,7 +397,7 @@ Response handleGetRequest(ObjectRequestContext context,
String keyPath)
getClientProtocol().getS3KeyDetails(bucketName, keyPath, partNumber)
:
getClientProtocol().getS3KeyDetails(bucketName, keyPath);
- isFile(keyPath, keyDetails);
+ validateFileKey(keyPath, keyDetails);
Response conditionalResponse =
S3ConditionalRequest.evaluatePreconditions(
getHeaders(), keyPath, keyDetails,
S3ConditionalRequest.PreconditionContext.READ);
@@ -647,7 +648,7 @@ public Response head(
getClientProtocol().headS3Object(bucketName, keyPath, partNumber) :
getClientProtocol().headS3Object(bucketName, keyPath);
- isFile(keyPath, key);
+ validateFileKey(keyPath, key);
Response conditionalResponse =
S3ConditionalRequest.evaluatePreconditions(
getHeaders(), keyPath, key,
S3ConditionalRequest.PreconditionContext.READ);
if (conditionalResponse != null) {
@@ -689,22 +690,6 @@ public Response head(
return response.build();
}
- private void isFile(String keyPath, OzoneKey key) throws OMException {
- /*
- Necessary for directories in buckets with FSO layout.
- Intended for apps which use Hadoop S3A.
- Example of such app is Trino (through Hive connector).
- */
- boolean isFsoDirCreationEnabled = getOzoneConfiguration()
- .getBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED,
- OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT);
- if (isFsoDirCreationEnabled &&
- !key.isFile() &&
- !keyPath.endsWith("/")) {
- throw new OMException(ResultCodes.KEY_NOT_FOUND);
- }
- }
-
/**
* Delete a specific object from a bucket, if query param uploadId is
* specified, this request is for abort multipart upload.
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index 160102e7290..0c8f8e504ec 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -109,6 +109,8 @@ public final class S3GatewayMetrics implements Closeable,
MetricsSource {
private @Metric MutableCounterLong deleteBucketTaggingFailure;
private @Metric MutableCounterLong putObjectAclSuccess;
private @Metric MutableCounterLong putObjectAclFailure;
+ private @Metric MutableCounterLong getObjectAttributesSuccess;
+ private @Metric MutableCounterLong getObjectAttributesFailure;
// S3 Gateway Latency Metrics
// BucketEndpoint
@@ -304,6 +306,12 @@ public final class S3GatewayMetrics implements Closeable,
MetricsSource {
"in nanoseconds")
private PerformanceMetrics putObjectAclFailureLatencyNs;
+ @Metric(about = "Latency for successfully getting S3 object attributes in
nanoseconds")
+ private PerformanceMetrics getObjectAttributesSuccessLatencyNs;
+
+ @Metric(about = "Latency for failing to get S3 object attributes in
nanoseconds")
+ private PerformanceMetrics getObjectAttributesFailureLatencyNs;
+
private final Map<String, PerformanceMetrics> performanceMetrics;
/**
@@ -467,6 +475,10 @@ public void getMetrics(MetricsCollector collector, boolean
all) {
putObjectAclSuccessLatencyNs.snapshot(recordBuilder, true);
putObjectAclFailure.snapshot(recordBuilder, true);
putObjectAclFailureLatencyNs.snapshot(recordBuilder, true);
+ getObjectAttributesSuccess.snapshot(recordBuilder, true);
+ getObjectAttributesSuccessLatencyNs.snapshot(recordBuilder, true);
+ getObjectAttributesFailure.snapshot(recordBuilder, true);
+ getObjectAttributesFailureLatencyNs.snapshot(recordBuilder, true);
}
// INC and UPDATE
@@ -758,6 +770,16 @@ public void updatePutObjectAclFailureStats(long
startNanos) {
this.putObjectAclFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
}
+ public void updateGetObjectAttributesSuccessStats(long startNanos) {
+ this.getObjectAttributesSuccess.incr();
+ this.getObjectAttributesSuccessLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updateGetObjectAttributesFailureStats(long startNanos) {
+ this.getObjectAttributesFailure.incr();
+ this.getObjectAttributesFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
// GET
public long getListS3BucketsSuccess() {
return listS3BucketsSuccess.value();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index c8a2f7f4809..8477d8c64a1 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -128,6 +128,9 @@ public final class S3Consts {
// tag limit for bucket
public static final int TAG_BUCKET_NUM_LIMIT = 50;
+ /** Request header carrying the list of object attributes to return. */
+ public static final String OBJECT_ATTRIBUTES_HEADER =
"x-amz-object-attributes";
+
//Never Constructed
private S3Consts() {
@@ -170,6 +173,7 @@ public static final class QueryParams {
public static final String UPLOAD_ID_MARKER = "upload-id-marker";
public static final String UPLOADS = "uploads";
public static final String LIFECYCLE = "lifecycle";
+ public static final String ATTRIBUTES = "attributes";
private QueryParams() {
// no instances
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
index 1e48f6c9be9..0bbbf4eb18e 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
@@ -62,6 +62,20 @@ public static Response getTagging(
return subject.get(bucket, key);
}
+ /** Get object attributes (?attributes). */
+ public static Response getObjectAttributes(
+ ObjectEndpoint subject,
+ String bucket,
+ String key,
+ String attributesHeader
+ ) throws IOException, OS3Exception {
+ subject.queryParamsForTest().set(S3Consts.QueryParams.ATTRIBUTES, "");
+ when(subject.getContext().getMethod()).thenReturn(HttpMethod.GET);
+
when(subject.getHeaders().getHeaderString(S3Consts.OBJECT_ATTRIBUTES_HEADER))
+ .thenReturn(attributesHeader);
+ return subject.get(bucket, key);
+ }
+
/** List parts of MPU. */
public static Response listParts(
ObjectEndpoint subject,
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
new file mode 100644
index 00000000000..0f2cca58b3f
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getObjectAttributes;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.IOException;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Tests for GetObjectAttributes.
+ */
+public class TestObjectAttributesGet {
+
+ private static final String CONTENT = "0123456789";
+ private static final String BUCKET_NAME = "b1";
+ private static final String KEY_NAME = "key1";
+ private ObjectEndpoint rest;
+ private OzoneBucket bucket;
+
+ @BeforeEach
+ public void init() throws Exception {
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(BUCKET_NAME);
+ bucket = client.getObjectStore().getS3Bucket(BUCKET_NAME);
+
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("UNSIGNED-PAYLOAD");
+
+ rest = EndpointBuilder.newObjectEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+ }
+
+ @Test
+ public void testGetObjectAttributesAllSupportedFields() throws IOException,
OS3Exception {
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+
+ Response response = getObjectAttributes(rest, BUCKET_NAME, KEY_NAME,
+ "ETag,ObjectSize,StorageClass");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ assertNotNull(response.getHeaderString(HttpHeaders.LAST_MODIFIED));
+
+ GetObjectAttributesResponse attributes = (GetObjectAttributesResponse)
response.getEntity();
+ assertNotNull(attributes);
+ assertNotNull(attributes.getETag());
+ assertFalse(attributes.getETag().startsWith("\""));
+ assertEquals(CONTENT.length(), attributes.getObjectSize().longValue());
+ assertEquals("STANDARD", attributes.getStorageClass());
+ assertNull(attributes.getObjectParts());
+ }
+
+ @Test
+ public void testGetObjectAttributesEtagOnly() throws IOException,
OS3Exception {
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+
+ Response response = getObjectAttributes(rest, BUCKET_NAME, KEY_NAME,
"ETag");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ GetObjectAttributesResponse attributes = (GetObjectAttributesResponse)
response.getEntity();
+ assertNotNull(attributes.getETag());
+ assertNull(attributes.getObjectSize());
+ assertNull(attributes.getStorageClass());
+ }
+
+ @Test
+ public void testGetObjectAttributesChecksumRequestedButOmitted() throws
IOException, OS3Exception {
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+
+ Response response = getObjectAttributes(rest, BUCKET_NAME, KEY_NAME,
"Checksum,ETag");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ GetObjectAttributesResponse attributes = (GetObjectAttributesResponse)
response.getEntity();
+ assertNotNull(attributes.getETag());
+ }
+
+ @Test
+ public void testGetObjectAttributesMissingHeader() {
+ assertErrorResponse(INVALID_ARGUMENT,
+ () -> getObjectAttributes(rest, BUCKET_NAME, KEY_NAME, null));
+ }
+
+ @Test
+ public void testGetObjectAttributesInvalidAttribute() {
+ assertErrorResponse(INVALID_ARGUMENT,
+ () -> getObjectAttributes(rest, BUCKET_NAME, KEY_NAME,
"NotAValidAttribute"));
+ }
+
+ @Test
+ public void testGetObjectAttributesNoKeyFound() {
+ assertErrorResponse(NO_SUCH_KEY,
+ () -> getObjectAttributes(rest, BUCKET_NAME, "nonexistent", "ETag"));
+ }
+
+ @Test
+ public void testGetObjectAttributesNoBucketFound() {
+ assertErrorResponse(NO_SUCH_BUCKET,
+ () -> getObjectAttributes(rest, "nonexistent", "any", "ETag"));
+ }
+
+ @Test
+ public void testGetObjectAttributesNonMultipartObjectHasNoParts() throws
IOException, OS3Exception {
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+
+ Response response = getObjectAttributes(rest, BUCKET_NAME, KEY_NAME,
"ObjectParts");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ GetObjectAttributesResponse attributes = (GetObjectAttributesResponse)
response.getEntity();
+ assertNull(attributes.getObjectParts());
+ }
+
+ @Test
+ public void testWhenKeyIsDirectoryAndKeyPathDoesNotEndWithASlash() throws
Exception {
+ final String keyPath = "keyDir";
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.setBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, true);
+ rest.setOzoneConfiguration(config);
+ bucket.createDirectory(keyPath);
+
+ assertErrorResponse(NO_SUCH_KEY,
+ () -> getObjectAttributes(rest, BUCKET_NAME, keyPath, "ETag"));
+ }
+
+ @Test
+ public void testWhenKeyIsDirectoryAndKeyPathEndsWithASlash() throws
Exception {
+ final String keyPath = "keyDir/";
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.setBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, true);
+ rest.setOzoneConfiguration(config);
+ bucket.createDirectory(keyPath);
+
+ Response response = getObjectAttributes(rest, BUCKET_NAME, keyPath,
"ObjectSize");
+
+ assertEquals(HTTP_OK, response.getStatus());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
